Update data-router to use HTTPS DMaaP

Updates to camel route and beans xml to use HTTPS DMaaP, and also add
filebeat sidecar

Issue-ID: AAI-1562

Change-Id: I26e30ba01b0552d7e26c99f9039927accacdcb58
Signed-off-by: michaere <michaere@amdocs.com>

diff --git a/kubernetes/aai/charts/aai-data-router/resources/config/auth/tomcat_keystore b/kubernetes/aai/charts/aai-data-router/resources/config/auth/tomcat_keystore
index 9eec841..de5325b 100644 (file)
Binary files a/kubernetes/aai/charts/aai-data-router/resources/config/auth/tomcat_keystore and b/kubernetes/aai/charts/aai-data-router/resources/config/auth/tomcat_keystore differ

diff --git a/kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml b/kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml
new file mode 100644 (file)
index 0000000..d7ff014
--- /dev/null
+++ b/kubernetes/aai/charts/aai-data-router/resources/config/log/logback.xml
@@ -0,0 +1,193 @@
+<!--
+    ============LICENSE_START=======================================================
+    org.onap.aai
+    ================================================================================
+    Copyright © 2018 AT&T Intellectual Property. All rights reserved.
+    Copyright © 2018 Amdocs
+    ================================================================================
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+    ============LICENSE_END=========================================================
+-->
+<configuration scan="true" scanPeriod="3 seconds" debug="false">
+  <!--<jmxConfigurator /> -->
+  <!-- directory path for all other type logs -->
+
+  <property name="logDir"  value="/var/log/onap" />
+
+  <!--  specify the component name -->
+  <property name="componentName" value="AAI-DR" />
+
+  <!--  default eelf log file names -->
+  <property name="generalLogName" value="error" />
+  <property name="metricsLogName" value="metrics" />
+  <property name="auditLogName" value="audit" />
+  <property name="debugLogName" value="debug" />
+
+  <property name="errorLogPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%mdc{RequestId}|%thread|data-router|%mdc{PartnerName}|%logger||%.-5level|%msg%n" />
+  <property name="auditMetricPattern" value="%m%n" />
+
+  <property name="logDirectory" value="${logDir}/${componentName}" />
+
+  <!-- Example evaluator filter applied against console appender -->
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>${errorLogPattern}</pattern>
+    </encoder>
+  </appender>
+
+  <!-- ============================================================================ -->
+  <!-- EELF Appenders -->
+  <!-- ============================================================================ -->
+
+  <!-- The EELFAppender is used to record events to the general application
+       log -->
+
+  <appender name="EELF"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${generalLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${errorLogPattern}</pattern>
+    </encoder>
+  </appender>
+  <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender">
+    <!-- deny all events with a level below INFO, that is TRACE and DEBUG -->
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+      <level>INFO</level>
+    </filter>
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELF" />
+  </appender>
+
+  <!-- EELF Audit Appender. This appender is used to record audit engine
+       related logging events. The audit logger and appender are specializations
+       of the EELF application root logger and appender. This can be used to segregate
+       Policy engine events from other components, or it can be eliminated to record
+       these events as part of the application root log. -->
+
+  <appender name="EELFAudit"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${auditLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${auditMetricPattern}</pattern>
+    </encoder>
+  </appender>
+  <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELFAudit" />
+  </appender>
+
+  <appender name="EELFMetrics"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${metricsLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${metricsLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <!-- <pattern>"%d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} -
+           %msg%n"</pattern> -->
+      <pattern>${auditMetricPattern}</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELFMetrics"/>
+  </appender>
+
+  <appender name="EELFDebug"
+            class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${logDirectory}/${debugLogName}.log</file>
+    <rollingPolicy
+        class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${logDirectory}/${debugLogName}.%d{yyyy-MM-dd}.log.zip
+      </fileNamePattern>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${errorLogPattern}</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>256</queueSize>
+    <appender-ref ref="EELFDebug" />
+    <includeCallerData>false</includeCallerData>
+  </appender>
+
+  <!-- ============================================================================ -->
+  <!--  EELF loggers -->
+  <!-- ============================================================================ -->
+  <logger name="com.att.eelf" level="info" additivity="false">
+    <appender-ref ref="asyncEELF" />
+    <appender-ref ref="asyncEELFDebug" />
+  </logger>
+
+  <logger name="com.att.eelf.audit" level="info" additivity="false">
+    <appender-ref ref="asyncEELFAudit" />
+  </logger>
+  <logger name="com.att.eelf.metrics" level="info" additivity="false">
+    <appender-ref ref="asyncEELFMetrics" />
+  </logger>
+
+  <!-- Spring related loggers -->
+  <logger name="org.springframework" level="WARN" />
+  <logger name="org.springframework.beans" level="WARN" />
+  <logger name="org.springframework.web" level="WARN" />
+  <logger name="com.blog.spring.jms" level="WARN" />
+
+  <!-- Data Router service loggers -->
+  <logger name="org.onap.aai.data-router" level="INFO" />
+
+  <!-- Other Loggers that may help troubleshoot -->
+  <logger name="net.sf" level="WARN" />
+  <logger name="org.apache" level="WARN" />
+  <logger name="org.apache.commons.httpclient" level="WARN" />
+  <logger name="org.apache.commons" level="WARN" />
+  <logger name="org.apache.coyote" level="WARN" />
+  <logger name="org.apache.jasper" level="WARN" />
+
+  <!-- Camel Related Loggers (including restlet/servlet/jaxrs/cxf logging.
+       May aid in troubleshooting) -->
+  <logger name="org.apache.camel" level="WARN" />
+  <logger name="org.apache.cxf" level="WARN" />
+  <logger name="org.apache.camel.processor.interceptor" level="WARN" />
+  <logger name="org.apache.cxf.jaxrs.interceptor" level="WARN" />
+  <logger name="org.apache.cxf.service" level="WARN" />
+  <logger name="org.restlet" level="WARN" />
+  <logger name="org.apache.camel.component.restlet" level="WARN" />
+
+  <!-- logback internals logging -->
+  <logger name="ch.qos.logback.classic" level="WARN" />
+  <logger name="ch.qos.logback.core" level="WARN" />
+
+  <root>
+    <appender-ref ref="asyncEELF" />
+    <!-- <appender-ref ref="asyncEELFDebug" /> -->
+  </root>
+
+</configuration>

diff --git a/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml b/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml
index acfe0a5..60e71ec 100644 (file)
--- a/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml
+++ b/kubernetes/aai/charts/aai-data-router/resources/dynamic/conf/entity-event-policy.xml
@@ -34,9 +34,9 @@
     <property name="searchKeystore" value="tomcat_keystore" />
   </bean>
   
-  <bean id="consumerBeanEntityEvent" class="org.onap.aai.event.client.DMaaPEventConsumer " >
-    <constructor-arg name="host" value="message-router.{{.Release.Namespace}}:3904" />
-    <constructor-arg name="topic" value="AAI-EVENT" />
+  <bean id="consumerBeanEntityEvent" class="org.onap.aai.event.client.DMaaPEventConsumer" >
+    <constructor-arg name="host" value="message-router.{{.Release.Namespace}}:{{.Values.event.port.dmaap}}" />
+    <constructor-arg name="topic" value="{{.Values.event.consumer.topic}}" />
     <constructor-arg name="username" value="" />
     <constructor-arg name="password" value="" />
     <constructor-arg name="consumerGroup" value="datarouter" />
@@ -44,6 +44,8 @@
     <constructor-arg name="timeoutMs" value="1000" />
     <constructor-arg name="messageLimit" value="100" />
     <constructor-arg name="transportType" value="HTTPAUTH" />
+    <constructor-arg name="protocol" value="{{.Values.event.protocol}}" />
+    <constructor-arg name="contentType" value="application/json" />
   </bean>
 
   <bean id="entityEventPolicy" class="org.onap.aai.datarouter.policy.EntityEventPolicy" init-method="startup" >

diff --git a/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml b/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml
index 6cfc3be..23d3dbb 100644 (file)
--- a/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml
+++ b/kubernetes/aai/charts/aai-data-router/templates/configmap.yaml
@@ -38,3 +38,29 @@ metadata:
 data:
 {{ tpl (.Files.Glob "resources/dynamic/routes/entity-event.route").AsConfig . | indent 2 }}
 {{ tpl (.Files.Glob "resources/dynamic/conf/entity-event-policy.xml").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.fullname" . }}-log-configmap
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/config/log/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.fullname" . }}-filebeat-configmap
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/config/log/filebeat/*").AsConfig . | indent 2 }}
\ No newline at end of file

diff --git a/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml b/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml
index 8ab2d33..83542da 100644 (file)
--- a/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml
+++ b/kubernetes/aai/charts/aai-data-router/templates/deployment.yaml
@@ -94,7 +94,10 @@ spec:
         - mountPath: /opt/app/data-router/dynamic/conf/entity-event-policy.xml
           subPath: entity-event-policy.xml
           name: {{ include "common.fullname" . }}-dynamic-policy
-        - mountPath: /logs/
+        - mountPath: /opt/app/data-router/bundleconfig/etc/logback.xml
+          name: {{ include "common.fullname" . }}-logback-config
+          subPath: logback.xml
+        - mountPath: /var/log/onap
           name: {{ include "common.fullname" . }}-logs
         ports:
         - containerPort: {{ .Values.service.internalPort }}
@@ -112,10 +115,28 @@ spec:
           periodSeconds: {{ .Values.readiness.periodSeconds }}
         resources:
 {{ include "common.resources" . | indent 12 }}
+
+      # side car containers
+      - name: filebeat-onap
+        image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        volumeMounts:
+        - mountPath: /usr/share/filebeat/filebeat.yml
+          subPath: filebeat.yml
+          name: filebeat-conf
+        - mountPath: /var/log/onap
+          name: {{ include "common.fullname" . }}-logs
+        - mountPath: /usr/share/filebeat/data
+          name: aai-filebeat
       volumes:
       - name: localtime
         hostPath:
           path: /etc/localtime
+      - name: filebeat-conf
+        configMap:
+          name: aai-filebeat
+      - name: aai-filebeat
+        emptyDir: {}
       - name: {{ include "common.fullname" . }}-auth
         secret:
           secretName: {{ include "common.fullname" . }}
@@ -134,8 +155,13 @@ spec:
         configMap:
           name: {{ include "common.fullname" . }}-dynamic
       - name: {{ include "common.fullname" . }}-logs
-        hostPath:
-          path: {{ .Values.persistence.mountPath }}/{{ .Release.Name }}/{{ .Values.persistence.mountSubPath }}
+        emptyDir: {}
+      - name: {{ include "common.fullname" . }}-logback-config
+        configMap:
+          name: {{ include "common.fullname" . }}-log-configmap
+          items:
+          - key: logback.xml
+            path: logback.xml
       restartPolicy: {{ .Values.global.restartPolicy | default .Values.restartPolicy }}
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"

diff --git a/kubernetes/aai/charts/aai-data-router/values.yaml b/kubernetes/aai/charts/aai-data-router/values.yaml
index ba42582..c1fa49f 100644 (file)
--- a/kubernetes/aai/charts/aai-data-router/values.yaml
+++ b/kubernetes/aai/charts/aai-data-router/values.yaml
@@ -17,7 +17,7 @@
 # Declare variables to be passed into your templates.
 global: # global defaults
   nodePortPrefix: 302
-
+  loggingImage: beats/filebeat:5.5.0
 
 # application image
 repository: nexus3.onap.org:10001
@@ -97,4 +97,12 @@ resources:
     requests:
       cpu: 4
       memory: 8Gi
-  unlimited: {}
\ No newline at end of file
+  unlimited: {}
+
+# Entity Event route configuration
+event:
+  port:
+    dmaap: 3905
+  protocol: https
+  consumer:
+    topic: AAI-EVENT