From c20dfd0d6b2ce32ff9bbc8eade3b2cdc9dbff581 Mon Sep 17 00:00:00 2001
From: michaere <michaere@amdocs.com>
Date: Mon, 1 Oct 2018 11:45:50 +0100
Subject: [PATCH] Update data-router to use HTTPS DMaaP

Updates to camel route and beans xml to use HTTPS DMaaP, and also add
filebeat sidecar

Issue-ID: AAI-1562

Change-Id: I26e30ba01b0552d7e26c99f9039927accacdcb58
Signed-off-by: michaere <michaere@amdocs.com>
---
 .../resources/config/auth/tomcat_keystore          | Bin 2214 -> 3429 bytes
 .../resources/config/log/logback.xml               | 193 +++++++++++++++++++++
 .../resources/dynamic/conf/entity-event-policy.xml |   8 +-
 .../aai-data-router/templates/configmap.yaml       |  26 +++
 .../aai-data-router/templates/deployment.yaml      |  32 +++-
 kubernetes/aai/charts/aai-data-router/values.yaml  |  12 +-
 6 files changed, 263 insertions(+), 8 deletions(-)
 create mode 100644 kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml

diff --git a/kubernetes/aai/charts/aai-data-router/resources/config/auth/tomcat_keystore b/kubernetes/aai/charts/aai-data-router/resources/config/auth/tomcat_keystore
index 9eec841aa2c1243b5ca3e22b0b116e5bca2afd49..de5325b4aa36f2ca212b8ed149d6a3943e9b1ba2 100644
GIT binary patch
delta 1238
zcmZ1`_*AO?-`jt085kIt;8;2(H!-n5H@8SZBQvigwJ0|=B{Q)kH90ZfP?LdyG4=b&
zJWmGJ2t88+O9lp(Zi6P4c7rD7`~}QRj7&_7Tn4;soLX%jZQpqr8M#>*4BQR54LI4D
zLs{5_nL>jNc@4Nh94;Ofe?P|nL%w<g9uS|Kht)sW(b3IN(m))<XXX(=w%pk<-q1i!
zoY&C8z|6qZ(9{%2Mgh5&2ByY_P_98TUYnE*6d)E!g8i3U<d9pWmy)Xw^s8Qe9*8JP
zH{?Teji3QP#5EkSK$&dHrda>ZWWo&Q-L-c%K8QVeR&IWelwlITRIZepVyM2T>0dMR
zmklo@JmteHS4CKO{JQz)i^+{G4zY7r?mGI<vt~!NR#lo%4pUT`wDqi2hX0fA*r{Gz
z#ZZ`K(rualrYn6}Lbg@gm-(!(YdV@;+wUEb*MG41c&KE^2KC^mcQJLY;$=JQYwq)2
zSrNmKaN&Ay=D%f+w>$2>&UAUo>{D(%Df@-eADpy#<{ephTAWGvnuWO0p{+qdF}W*W
zXtH0>Kcqj&X!`WGKbR9IGykvjbyzYt<7y~h-FpLd^^KJqvs0Hzx2+I%^SD?oYSuLW
z=FRBMQDPyJ9@rReJ*UMVwru&|iZeStg>I?8#mmIZ$iTR`i7~;Ti7^Hozp}zCOa=@F
zf^3`#Z61tmKb#mDSy-5v*bW#NBZPQa)WlT$3sUofgMAd-Q}a@bfGI0QAvm?DEVW3%
zIkl)HGc7Y2C}toF40l;R7BLo)n2lA>a(V7`t@JtZw#@tSq=1k0T8j)qK=R5gz6Rb6
zB2rhviwe9hr>}hd@J^E9ruGZ}-mh9{utd*52O~n!k{@!?Y+__G-~*Y;&&c>6<aE{t
z5Lc+Oh#82macHwKva+%>GqQ2iLzzql$sqNDEHMUAO?<hj#l?x~sk%k^rNCU%Bn3=f
zi3OQ@#hJMUIjJeBx%u#9)+CIo7A(;OPhw5Hu*3$IK+90fO^o%73{!jBJ0rTX4C50R
zI=YG;#>ijORP_qEUAwdF;-AILvvwSd^V)g-?=Q*Oc4wZoE?Ro8wCc6bVXt+Y+{CJE
zW|z!T(+(~^wQQ@Z>CFhnQw#Qdvlp~D!u`TJzWnbY{kJPNPF#`IQ>#%Fu4egsNAeBF
zGtZh!k4-F#RNwxIQDfnQTlGzer~WMXXcXvY8@crJpV^Ok<r-Q-4tpwYovg#OU_#KU
zs1pL(_SQaE9@J?izjD_Xp3Aa0?R%=Hu*}V`2DQ&NdzW30`S>Mr#Y~eO)23#>isoCc
zboAGy=NCdsuPRh-uyySefAsOvM4wA>F4sP-;!)N3KC9y8gLRRLFWr{$IT<wCu^h|!
g8oZIGjYDMXgbq>WX)1xU4&)sw-E`h#)1|{A0QnWWYybcN

delta 43
ycmaDVwM>xb-`jt085kItfS7UfX_ht)k#hZ}6aT+&66d=)e^1+SmaYBL>Y@OCw-KEH

diff --git a/kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml b/kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml
new file mode 100644
index 0000000000..d7ff0143a1
--- /dev/null
+++ b/kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml
@@ -0,0 +1,193 @@
+<!--
+    ============LICENSE_START=======================================================
+    org.onap.aai
+    ================================================================================
+    Copyright © 2018 AT&T Intellectual Property. All rights reserved.
+    Copyright © 2018 Amdocs
+    ================================================================================
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+    ============LICENSE_END=========================================================
+-->
+<configuration scan="true" scanPeriod="3 seconds" debug="false">
+  <!--<jmxConfigurator /> -->
+  <!-- directory path for all other type logs -->
+
+  <property name="logDir"  value="/var/log/onap" />
+
+  <!--  specify the component name -->
+  <property name="componentName" value="AAI-DR" />
+
+  <!--  default eelf log file names -->
+  <property name="generalLogName" value="error" />
+  <property name="metricsLogName" value="metrics" />
+  <property name="auditLogName" value="audit" />
+  <property name="debugLogName" value="debug" />
+
+  <property name="errorLogPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%mdc{RequestId}|%thread|data-router|%mdc{PartnerName}|%logger||%.-5level|%msg%n" />
+  <property name="auditMetricPattern" value="%m%n" />
+
+  <property name="logDirectory" value="${logDir}/${componentName}" />
+
+  <!-- Example evaluator filter applied against console appender -->
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>${errorLogPattern}</pattern>
+    </encoder>
+  </appender>
+
+  <!-- ============================================================================ -->
+  <!-- EELF Appenders -->
+  <!-- ============================================================================ -->
+
+  <!-- The EELFAppender is used to record events to the general application
+       log -->
+
+  <appender name="EELF"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${generalLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${errorLogPattern}</pattern>
+    </encoder>
+  </appender>
+  <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender">
+    <!-- deny all events with a level below INFO, that is TRACE and DEBUG -->
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELF" />
+  </appender>
+
+  <!-- EELF Audit Appender. This appender is used to record audit engine
+       related logging events. The audit logger and appender are specializations
+       of the EELF application root logger and appender. This can be used to segregate
+       Policy engine events from other components, or it can be eliminated to record
+       these events as part of the application root log. -->
+
+  <appender name="EELFAudit"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${auditLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${auditMetricPattern}</pattern>
+    </encoder>
+  </appender>
+  <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELFAudit" />
+  </appender>
+
+  <appender name="EELFMetrics"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${metricsLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${metricsLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <!-- <pattern>"%d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} -
+           %msg%n"</pattern> -->
+      <pattern>${auditMetricPattern}</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELFMetrics"/>
+  </appender>
+
+  <appender name="EELFDebug"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${debugLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${debugLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${errorLogPattern}</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELFDebug" />
+    <includeCallerData>false</includeCallerData>
+  </appender>
+
+  <!-- ============================================================================ -->
+  <!--  EELF loggers -->
+  <!-- ============================================================================ -->
+  <logger name="com.att.eelf" level="info" additivity="false">
+    <appender-ref ref="asyncEELF" />
+    <appender-ref ref="asyncEELFDebug" />
+  </logger>
+
+  <logger name="com.att.eelf.audit" level="info" additivity="false">
+    <appender-ref ref="asyncEELFAudit" />
+  </logger>
+  <logger name="com.att.eelf.metrics" level="info" additivity="false">
+    <appender-ref ref="asyncEELFMetrics" />
+  </logger>
+
+  <!-- Spring related loggers -->
+  <logger name="org.springframework" level="WARN" />
+  <logger name="org.springframework.beans" level="WARN" />
+  <logger name="org.springframework.web" level="WARN" />
+  <logger name="com.blog.spring.jms" level="WARN" />
+
+  <!-- Data Router service loggers -->
+  <logger name="org.onap.aai.data-router" level="INFO" />
+
+  <!-- Other Loggers that may help troubleshoot -->
+  <logger name="net.sf" level="WARN" />
+  <logger name="org.apache" level="WARN" />
+  <logger name="org.apache.commons.httpclient" level="WARN" />
+  <logger name="org.apache.commons" level="WARN" />
+  <logger name="org.apache.coyote" level="WARN" />
+  <logger name="org.apache.jasper" level="WARN" />
+
+  <!-- Camel Related Loggers (including restlet/servlet/jaxrs/cxf logging.
+       May aid in troubleshooting) -->
+  <logger name="org.apache.camel" level="WARN" />
+  <logger name="org.apache.cxf" level="WARN" />
+  <logger name="org.apache.camel.processor.interceptor" level="WARN" />
+  <logger name="org.apache.cxf.jaxrs.interceptor" level="WARN" />
+  <logger name="org.apache.cxf.service" level="WARN" />
+  <logger name="org.restlet" level="WARN" />
+  <logger name="org.apache.camel.component.restlet" level="WARN" />
+
+  <!-- logback internals logging -->
+  <logger name="ch.qos.logback.classic" level="WARN" />
+  <logger name="ch.qos.logback.core" level="WARN" />
+
+  <root>
+    <appender-ref ref="asyncEELF" />
+    <!-- <appender-ref ref="asyncEELFDebug" /> -->
+  </root>
+
+</configuration>
diff --git a/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml b/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml
index acfe0a5933..60e71ecfb0 100644
--- a/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml
+++ b/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml
@@ -34,9 +34,9 @@
     <property name="searchKeystore" value="tomcat_keystore" />
   </bean>
   
-  <bean id="consumerBeanEntityEvent" class="org.onap.aai.event.client.DMaaPEventConsumer " >
-    <constructor-arg name="host" value="message-router.{{.Release.Namespace}}:3904" />
-    <constructor-arg name="topic" value="AAI-EVENT" />
+  <bean id="consumerBeanEntityEvent" class="org.onap.aai.event.client.DMaaPEventConsumer" >
+    <constructor-arg name="host" value="message-router.{{.Release.Namespace}}:{{.Values.event.port.dmaap}}" />
+    <constructor-arg name="topic" value="{{.Values.event.consumer.topic}}" />
     <constructor-arg name="username" value="" />
     <constructor-arg name="password" value="" />
     <constructor-arg name="consumerGroup" value="datarouter" />
@@ -44,6 +44,8 @@
     <constructor-arg name="timeoutMs" value="1000" />
     <constructor-arg name="messageLimit" value="100" />
     <constructor-arg name="transportType" value="HTTPAUTH" />
+    <constructor-arg name="protocol" value="{{.Values.event.protocol}}" />
+    <constructor-arg name="contentType" value="application/json" />
   </bean>
 
   <bean id="entityEventPolicy" class="org.onap.aai.datarouter.policy.EntityEventPolicy" init-method="startup" >
diff --git a/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml b/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml
index 6cfc3bef7a..23d3dbbcb9 100644
--- a/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml
+++ b/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml
@@ -38,3 +38,29 @@ metadata:
 data:
 {{ tpl (.Files.Glob "resources/dynamic/routes/entity-event.route").AsConfig . | indent 2 }}
 {{ tpl (.Files.Glob "resources/dynamic/conf/entity-event-policy.xml").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.fullname" . }}-log-configmap
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/config/log/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.fullname" . }}-filebeat-configmap
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/config/log/filebeat/*").AsConfig . | indent 2 }}
\ No newline at end of file
diff --git a/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml b/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml
index 8ab2d3383a..83542dabe5 100644
--- a/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml
+++ b/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml
@@ -94,7 +94,10 @@ spec:
         - mountPath: /opt/app/data-router/dynamic/conf/entity-event-policy.xml
           subPath: entity-event-policy.xml
           name: {{ include "common.fullname" . }}-dynamic-policy
-        - mountPath: /logs/
+        - mountPath: /opt/app/data-router/bundleconfig/etc/logback.xml
+          name: {{ include "common.fullname" . }}-logback-config
+          subPath: logback.xml
+        - mountPath: /var/log/onap
           name: {{ include "common.fullname" . }}-logs
         ports:
         - containerPort: {{ .Values.service.internalPort }}
@@ -112,10 +115,28 @@ spec:
           periodSeconds: {{ .Values.readiness.periodSeconds }}
         resources:
 {{ include "common.resources" . | indent 12 }}
+
+      # side car containers
+      - name: filebeat-onap
+        image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        volumeMounts:
+        - mountPath: /usr/share/filebeat/filebeat.yml
+          subPath: filebeat.yml
+          name: filebeat-conf
+        - mountPath: /var/log/onap
+          name: {{ include "common.fullname" . }}-logs
+        - mountPath: /usr/share/filebeat/data
+          name: aai-filebeat
       volumes:
       - name: localtime
         hostPath:
           path: /etc/localtime
+      - name: filebeat-conf
+        configMap:
+          name: aai-filebeat
+      - name: aai-filebeat
+        emptyDir: {}
       - name: {{ include "common.fullname" . }}-auth
         secret:
           secretName: {{ include "common.fullname" . }}
@@ -134,8 +155,13 @@ spec:
         configMap:
           name: {{ include "common.fullname" . }}-dynamic
       - name: {{ include "common.fullname" . }}-logs
-        hostPath:
-          path: {{ .Values.persistence.mountPath }}/{{ .Release.Name }}/{{ .Values.persistence.mountSubPath }}
+        emptyDir: {}
+      - name: {{ include "common.fullname" . }}-logback-config
+        configMap:
+          name: {{ include "common.fullname" . }}-log-configmap
+          items:
+          - key: logback.xml
+            path: logback.xml
       restartPolicy: {{ .Values.global.restartPolicy | default .Values.restartPolicy }}
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
diff --git a/kubernetes/aai/charts/aai-data-router/values.yaml b/kubernetes/aai/charts/aai-data-router/values.yaml
index ba4258257a..c1fa49f3f0 100644
--- a/kubernetes/aai/charts/aai-data-router/values.yaml
+++ b/kubernetes/aai/charts/aai-data-router/values.yaml
@@ -17,7 +17,7 @@
 # Declare variables to be passed into your templates.
 global: # global defaults
   nodePortPrefix: 302
-
+  loggingImage: beats/filebeat:5.5.0
 
 # application image
 repository: nexus3.onap.org:10001
@@ -97,4 +97,12 @@ resources:
     requests:
       cpu: 4
       memory: 8Gi
-  unlimited: {}
\ No newline at end of file
+  unlimited: {}
+
+# Entity Event route configuration
+event:
+  port:
+    dmaap: 3905
+  protocol: https
+  consumer:
+    topic: AAI-EVENT
-- 
2.16.6