From: liangke Date: Tue, 30 Jan 2018 10:39:25 +0000 (+0800) Subject: Sidecar filebeat container and ConfigMap X-Git-Tag: 2.0.0-ONAP~392^2 X-Git-Url: https://gerrit.onap.org/r/gitweb?p=oom.git;a=commitdiff_plain;h=2817f440d2b55a0f0e16376a0b73236f21237cdc Sidecar filebeat container and ConfigMap Add filebeat agent to ships logs to logging system. Apply ConfigMap to manager filebeat and log configuration. Change-Id: I02511b698c66d718d010df144f9dc27a6a52c2c6 Issue-ID: MULTICLOUD-151 Issue-ID: LOG-137 Signed-off-by: liangke --- diff --git a/kubernetes/multicloud/resources/config/log/filebeat/filebeat.yml b/kubernetes/multicloud/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000000..706f24d693 --- /dev/null +++ b/kubernetes/multicloud/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,41 @@ +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + ignore_older: 24h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + clean_inactive: 48h + + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["logstash.{{.Values.nsPrefix}}:5044"] + #If enable will do load balancing among availabe Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase diff --git a/kubernetes/multicloud/resources/config/log/framework/log.yml b/kubernetes/multicloud/resources/config/log/framework/log.yml new file mode 100644 index 0000000000..a4943f015a --- /dev/null +++ b/kubernetes/multicloud/resources/config/log/framework/log.yml @@ -0,0 +1,26 @@ +version: 1 +disable_existing_loggers: False + +loggers: + multivimbroker: + handlers: [multivimbroker_handler] + level: "DEBUG" + propagate: False + +handlers: + multivimbroker_handler: + level: "DEBUG" + class: "logging.handlers.RotatingFileHandler" + filename: "/var/log/onap/multicloud/multivimbroker/multivimbroker.log" + formatter: "mdcFormat" + maxBytes: 1024*1024*50 + backupCount: 10 + +formatters: + standard: + format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" + mdcFormat: + format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:[%(mdc)s]: %(message)s" + mdcfmt: "{requestID}" + datefmt: "%Y-%m-%d %H:%M:%S" + (): onaplogging.mdcformatter.MDCFormatter \ No newline at end of file diff --git a/kubernetes/multicloud/resources/config/log/vio/log.yml b/kubernetes/multicloud/resources/config/log/vio/log.yml new file mode 100644 index 0000000000..c9d2bf84f3 --- /dev/null +++ b/kubernetes/multicloud/resources/config/log/vio/log.yml @@ -0,0 +1,26 @@ +version: 1 +disable_existing_loggers: False + +loggers: + vio: + handlers: [vio_handler] + level: "DEBUG" + propagate: False + +handlers: + vio_handler: + level: "DEBUG" + class: "logging.handlers.RotatingFileHandler" + filename: "/var/log/onap/multicloud/vio/vio.log" + formatter: "mdcFormat" + maxBytes: 1024*1024*50 + backupCount: 10 + +formatters: + standard: + format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" + mdcFormat: + format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:[%(mdc)s]: %(message)s" + mdcfmt: "{requestID}" + datefmt: "%Y-%m-%d %H:%M:%S" + (): onaplogging.mdcformatter.MDCFormatter \ No newline at end of file diff --git a/kubernetes/multicloud/templates/framework-deployment.yaml b/kubernetes/multicloud/templates/framework-deployment.yaml index dcdf0a2d75..f0c886506f 100644 --- a/kubernetes/multicloud/templates/framework-deployment.yaml +++ b/kubernetes/multicloud/templates/framework-deployment.yaml @@ -14,7 +14,6 @@ spec: app: framework name: multicloud-framework spec: - hostname: framework containers: - env: - name: MSB_ADDR @@ -31,12 +30,15 @@ spec: value: "AAI" - name: AAI_PASSWORD value: "AAI" + image: {{ .Values.image.framework }} + imagePullPolicy: {{ .Values.pullPolicy }} name: framework volumeMounts: - - mountPath: /service/multivimbroker/logs + - mountPath: /var/log/onap name: framework-log - image: {{ .Values.image.framework }} - imagePullPolicy: {{ .Values.pullPolicy }} + - mountPath: /opt/multivimbroker/multivimbroker/pub/config/log.yml + name: framework-logconfig + subPath: log.yml ports: - containerPort: 9001 livenessProbe: @@ -49,11 +51,29 @@ spec: timeoutSeconds: 10 successThreshold: 1 failureThreshold: 5 - restartPolicy: Always + - image: {{ .Values.image.filebeat }} + imagePullPolicy: {{ .Values.pullPolicy }} + name: filebeat-onap + volumeMounts: + - mountPath: /usr/share/filebeat/filebeat.yml + name: filebeat-conf + subPath: filebeat.yml + - mountPath: /var/log/onap + name: framework-log + - mountPath: /usr/share/filebeat/data + name: framework-data-filebeat volumes: - name: framework-log - hostPath: - path: {{ .Values.dataRootDir }}/{{ .Values.nsPrefix }}/multicloud/framework/logs + emptyDir: {} + - name: framework-data-filebeat + emptyDir: {} + - name: filebeat-conf + configMap: + name: multicloud-filebeat-configmap + - name: framework-logconfig + configMap: + name: multicloud-framework-log-configmap imagePullSecrets: - name: "{{ .Values.nsPrefix }}-docker-registry-key" + restartPolicy: Always #{{ end }} diff --git a/kubernetes/multicloud/templates/multicloud-log-configmap.yaml b/kubernetes/multicloud/templates/multicloud-log-configmap.yaml new file mode 100644 index 0000000000..754178c25f --- /dev/null +++ b/kubernetes/multicloud/templates/multicloud-log-configmap.yaml @@ -0,0 +1,25 @@ +#{{ if not .Values.disableVidVidServer }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: multicloud-filebeat-configmap + namespace: {{ .Values.nsPrefix }} +data: +{{ tpl (.Files.Glob "resources/config/log/filebeat/*").AsConfig . | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: multicloud-framework-log-configmap + namespace: {{ .Values.nsPrefix }} +data: +{{ tpl (.Files.Glob "resources/config/log/framework/*").AsConfig . | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: multicloud-vio-log-configmap + namespace: {{ .Values.nsPrefix }} +data: +{{ tpl (.Files.Glob "resources/config/log/vio/*").AsConfig . | indent 2 }} +#{{ end }} diff --git a/kubernetes/multicloud/templates/multicloud-vio-deployment.yaml b/kubernetes/multicloud/templates/multicloud-vio-deployment.yaml index df99542051..30413af245 100644 --- a/kubernetes/multicloud/templates/multicloud-vio-deployment.yaml +++ b/kubernetes/multicloud/templates/multicloud-vio-deployment.yaml @@ -14,7 +14,6 @@ spec: app: multicloud-vio name: multicloud-vio spec: - hostname: multicloud-vio containers: - env: - name: MSB_ADDR @@ -33,8 +32,11 @@ spec: value: "AAI" name: multicloud-vio volumeMounts: - - mountPath: /service/vio/logs + - mountPath: /var/log/onap name: vio-log + - mountPath: /opt/vio/vio/pub/config/log.yml + name: vio-logconfig + subPath: log.yml image: {{ .Values.image.vio }} imagePullPolicy: {{ .Values.pullPolicy }} ports: @@ -49,11 +51,29 @@ spec: timeoutSeconds: 10 successThreshold: 1 failureThreshold: 5 - restartPolicy: Always + - image: {{ .Values.image.filebeat }} + imagePullPolicy: {{ .Values.pullPolicy }} + name: filebeat-onap + volumeMounts: + - mountPath: /usr/share/filebeat/filebeat.yml + name: filebeat-conf + subPath: filebeat.yml + - mountPath: /var/log/onap + name: vio-log + - mountPath: /usr/share/filebeat/data + name: vio-data-filebeat volumes: - name: vio-log - hostPath: - path: {{ .Values.dataRootDir }}/{{ .Values.nsPrefix }}/multicloud/vio/logs + emptyDir: {} + - name: vio-data-filebeat + emptyDir: {} + - name: filebeat-conf + configMap: + name: multicloud-filebeat-configmap + - name: vio-logconfig + configMap: + name: multicloud-vio-log-configmap imagePullSecrets: - name: "{{ .Values.nsPrefix }}-docker-registry-key" + restartPolicy: Always #{{ end }} diff --git a/kubernetes/multicloud/values.yaml b/kubernetes/multicloud/values.yaml index 2dc24b24fe..704f0250b0 100644 --- a/kubernetes/multicloud/values.yaml +++ b/kubernetes/multicloud/values.yaml @@ -4,11 +4,11 @@ nodePortPrefix: 302 dataRootDir: /dockerdata-nfs image: readiness: oomk8s/readiness-check:1.1.0 - framework: nexus3.onap.org:10001/onap/multicloud/framework:v1.0.0 - vio: nexus3.onap.org:10001/onap/multicloud/vio:v1.0.0 + framework: nexus3.onap.org:10001/onap/multicloud/framework:latest + vio: nexus3.onap.org:10001/onap/multicloud/vio:latest ocata: nexus3.onap.org:10001/onap/multicloud/openstack-ocata:v1.0.0 windriver: nexus3.onap.org:10001/onap/multicloud/openstack-windriver:v1.0.0 - + filebeat: docker.elastic.co/beats/filebeat:5.5.0 # domain name of msb gateway msbgateway: msb-iag.{{ .Values.nsPrefix }} msbPort: 80