X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=kubernetes%2Flog%2Fcharts%2Flog-logstash%2Fresources%2Fconfig%2Fonap-pipeline.conf;h=d6b0696b81915d90241d2db9d78a25cd4a2ef4e9;hb=b1ee058f941a9c9ff27467054d98b8a56c988797;hp=3b4fd768c3502ce687b555d098f1421d0b0141d0;hpb=a6d65ef82e17905ca186c42388613afcd416a4ce;p=oom.git diff --git a/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf b/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf index 3b4fd768c3..d6b0696b81 100644 --- a/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf +++ b/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf @@ -1,3 +1,18 @@ +{{/* +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} input { beats { @@ -37,8 +52,20 @@ input { filter { + grok { + break_on_match => false + match => { + "source" => ["/var/log/onap/(?[^/]+)/", + "/var/log/onap/%{GREEDYDATA:componentLogFile}" + ] + } + } + # Filter for log4j xml events if "" in [message] { + + #mutate { add_field => { "orgmsg_log4j" => "%{message}" } } # Copy of orginal msg for debug + #Filter to parse xml event and retrieve data xml { source => "message" @@ -118,7 +145,7 @@ filter { # Filter for logback events else { -# mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug + #mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug mutate { gsub => [ @@ -128,50 +155,44 @@ filter { 'message', '\t$', '\t' ] } + # The grok below parses the message field for all current logback patterns used by oom components. + # Example logback pattern: %d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%msg + # Example grok pattern: %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message} + # Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq grok { - break_on_match => false match => { - "message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}", - "(?.*\t)" - ] - "source" => ["/var/log/onap/(?[^/]+)/", - "/var/log/onap/%{GREEDYDATA:componentLogFile}" - ] + "message" => [ + "%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}", + "\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}" + ] } overwrite => ["message"] } + # The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values. kv { source => "MDCs" - field_split => "\t" + field_split => ",\t" trim_key => "\s" trim_value => "\s" remove_field => [ "MDCs" ] } + if (![Timestamp] and [EndTimestamp]) { + mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } } + } date { match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ] target => "Timestamp" } - if [source] == "/var/log/onap/aai/aai-ml/metrics.log" { - csv { - source => "message" - separator => "|" - quote_char => "`" - columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Log level", "Unknown8", "Unknown9", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18", "message"] - } - } - else if [source] == "/var/log/onap/aai/aai-ml/audit.log" { - csv { - source => "message" - separator => "|" - quote_char => "`" - columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Log level", "Unknown6", "Unknown7", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "message"] - } - } - mutate { - remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8", "Unknown9", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18"] + remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"] } if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") {