Add logstash parsing for SDC-BE and AAI-ML

Issue-ID: OOM-378

Change-Id: Iac26f9076b09b815c6cb70643e8c6f384c3b8b9c
Signed-off-by: shanedaniel <shane.daniel@amdocs.com>

diff --git a/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf b/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf
index 1a46375..ed634ed 100644 (file)
--- a/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf
+++ b/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf
@@ -117,27 +117,79 @@ filter {
  }
  # Filter for logback events
  else {
+
+#  mutate { add_field => { "orgmsg" => "%{message}" } }    # Copy of orginal msg for debug
+
   mutate {
     gsub => [
+      'message', ' = ', '=',
       'message', '= ', '=null',
       'message', '=\t', '=null ', #This null is followed by a tab
       'message', '\t$', '\t'
     ]
   }
+  grok {
+    break_on_match => false
+    match => {
+      "message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}",
+                    "(?<MDCs>.*\t)"
+                   ]
+      "source" => ["/var/log/onap/(?<componentName>[^/]+)/",
+                   "/var/log/onap/%{GREEDYDATA:componentLogFile}"
+                  ]
+    }
+    overwrite => ["message"]
+  }
   kv {
+    source => "MDCs"
     field_split => "\t"
     trim_key => "\s"
     trim_value => "\s"
+    remove_field => [ "MDCs" ]
   }
-  grok {
-    break_on_match => false
-    match => {
-      "message" => "%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}"
+
+  date {
+    match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ]
+    target => "Timestamp"
+  }
+
+  if [source] == "/var/log/onap/aai/aai-ml/metrics.log" {
+    csv {
+      source => "message"
+      separator => "|"
+      quote_char => "`"
+      columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Log level", "Unknown8", "Unknown9", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18", "message"]
     }
-    overwrite => ["message"]
   }
- }
-}
+  else if [source] == "/var/log/onap/aai/aai-ml/audit.log" {
+    csv {
+      source => "message"
+      separator => "|"
+      quote_char => "`"
+      columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Log level", "Unknown6", "Unknown7", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "message"]
+    }
+  }
+
+  mutate {
+    remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8", "Unknown9", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18"]
+  }
+
+  if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") {
+    #Parse kvps in message
+    kv {
+      field_split => "\s"
+      trim_key => "\s"
+      trim_value => "\s"
+    }
+
+    #If Request Id is missing and DID is present use as RequestId
+    if (![RequestId] and [DID] =~ /.+/) {
+      mutate { add_field => { "RequestId" => "%{DID}" } }
+    }
+  }
+
+ } #Close else statement for logback events
+} #Close filter
 
 
 output {
@@ -149,7 +201,7 @@ output {
  user => "elastic"
  password => "changeme"
 
- ## The .cer or .pem file to validate the server’s certificate
+ ## The .cer or .pem file to validate the server's certificate
  #cacert => $es_cacert
 
  ## The keystore used to present a certificate to the server. It can be either .jks or .p12
@@ -160,10 +212,10 @@ output {
  ## Default is not set which in that case depends on the protocol specidfied in hosts list
  #ssl => $es_ssl
 
- ## Option to validate the server’s certificate. Default is true
+ ## Option to validate the server's certificate. Default is true
  #ssl_certificate_verification => $es_ssl_certificate_verification
 
- ## The JKS truststore to validate the server’s certificate.
+ ## The JKS truststore to validate the server's certificate.
  #truststore => $es_truststore
  #truststore_password => $es_truststore_password