+{{/*
+# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
input {
beats {
filter {
+ grok {
+ break_on_match => false
+ match => {
+ "source" => ["/var/log/onap/(?<componentName>[^/]+)/",
+ "/var/log/onap/%{GREEDYDATA:componentLogFile}"
+ ]
+ }
+ }
+
# Filter for log4j xml events
if "</log4j:event>" in [message] {
+
+ #mutate { add_field => { "orgmsg_log4j" => "%{message}" } } # Copy of orginal msg for debug
+
#Filter to parse xml event and retrieve data
xml {
source => "message"
# Filter for logback events
else {
-# mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug
+ #mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug
mutate {
gsub => [
'message', '\t$', '\t'
]
}
+ # The grok below parses the message field for all current logback patterns used by oom components.
+ # Example logback pattern: %d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%msg
+ # Example grok pattern: %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}
+ # Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq
grok {
- break_on_match => false
match => {
- "message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}",
- "(?<MDCs>.*\t)"
- ]
- "source" => ["/var/log/onap/(?<componentName>[^/]+)/",
- "/var/log/onap/%{GREEDYDATA:componentLogFile}"
- ]
+ "message" => [
+ "%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}",
+ "\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}"
+ ]
}
overwrite => ["message"]
}
+ # The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values.
kv {
source => "MDCs"
- field_split => "\t"
+ field_split => ",\t"
trim_key => "\s"
trim_value => "\s"
remove_field => [ "MDCs" ]
}
+ if (![Timestamp] and [EndTimestamp]) {
+ mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } }
+ }
date {
match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "Timestamp"
}
- if [source] == "/var/log/onap/aai/aai-ml/metrics.log" {
- csv {
- source => "message"
- separator => "|"
- quote_char => "`"
- columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Log level", "Unknown8", "Unknown9", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18", "message"]
- }
- }
- else if [source] == "/var/log/onap/aai/aai-ml/audit.log" {
- csv {
- source => "message"
- separator => "|"
- quote_char => "`"
- columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Log level", "Unknown6", "Unknown7", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "message"]
- }
- }
-
mutate {
- remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8", "Unknown9", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18"]
+ remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"]
}
if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") {