From: Jack Lucas Date: Tue, 10 May 2022 15:22:35 +0000 (+0000) Subject: Merge "[HV-VES-STRIMZI] Migrate hv-ves to use strimzi kafka" X-Git-Tag: 10.0.0~11 X-Git-Url: https://gerrit.onap.org/r/gitweb?p=oom.git;a=commitdiff_plain;h=145f52d1f9d06dd4f03fb19e579addeca82cec14;hp=271560c6bc1e5ed027a7cc11683e6688c3bc56bf Merge "[HV-VES-STRIMZI] Migrate hv-ves to use strimzi kafka" --- diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl index fbaaedf0dd..9781e33f1f 100644 --- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl +++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl @@ -58,7 +58,7 @@ the the literal string "An example value". {{- range $envName, $envValue := .Values.applicationEnv }} {{- if kindIs "string" $envValue }} - name: {{ $envName }} - value: {{ $envValue | quote }} + value: {{ tpl $envValue $global | quote }} {{- else }} {{ if or (not $envValue.secretUid) (not $envValue.key) }} {{ fail (printf "Env %s definition is not a string and does not contain secretUid or key fields" $envName) }} diff --git a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-kafka-user.yaml b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-kafka-user.yaml new file mode 100644 index 0000000000..ff977aaa32 --- /dev/null +++ b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-kafka-user.yaml @@ -0,0 +1,47 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: {{ include "common.release" . }}-{{ .Values.hvVesKafkaUser }} + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + authentication: + type: scram-sha-512 + authorization: + type: simple + acls: + - resource: + type: topic + name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-fault-supervision" "kafka_info" "topic_name" }} + operation: Write + - resource: + type: topic + name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-provisioning" "kafka_info" "topic_name" }} + operation: Write + - resource: + type: topic + name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-heartbeat" "kafka_info" "topic_name" }} + operation: Write + - resource: + type: topic + name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-performance-assurance" "kafka_info" "topic_name" }} + operation: Write + - resource: + type: topic + name: {{ .Values.applicationConfig.streams_publishes.perf3gpp.kafka_info.topic_name }} + operation: Write diff --git a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-topics.yaml b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-topics.yaml new file mode 100644 index 0000000000..e40775833a --- /dev/null +++ b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-topics.yaml @@ -0,0 +1,79 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: ves-3gpp-fault-supervision + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-fault-supervision" "kafka_info" "topic_name" }} + partitions: 10 + config: + retention.ms: 7200000 + segment.bytes: 1073741824 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: ves-3gpp-provisioning + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-provisioning" "kafka_info" "topic_name" }} + partitions: 10 + config: + retention.ms: 7200000 + segment.bytes: 1073741824 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: ves-3gpp-heartbeat + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-heartbeat" "kafka_info" "topic_name" }} + partitions: 10 + config: + retention.ms: 7200000 + segment.bytes: 1073741824 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: ves-3gpp-performance-assurance + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-performance-assurance" "kafka_info" "topic_name" }} + partitions: 10 + config: + retention.ms: 7200000 + segment.bytes: 1073741824 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: perf3gpp + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ .Values.applicationConfig.streams_publishes.perf3gpp.kafka_info.topic_name }} + partitions: 10 + config: + retention.ms: 7200000 + segment.bytes: 1073741824 diff --git a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml index 5a6283697c..67add37cea 100644 --- a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml @@ -2,6 +2,7 @@ # ================================================================================ # Copyright (c) 2021-2022 J. F. Lucas. All rights reserved. # Copyright (c) 2021-2022 Nokia. All rights reserved. +# Copyright © 2022 Nordix Foundation # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -61,11 +62,15 @@ certDirectory: /etc/ves-hv/ssl tlsServer: true secrets: - - uid: &aafCredsUID aafcreds - type: basicAuth - login: '{{ .Values.aafCreds.user }}' - password: '{{ .Values.aafCreds.password }}' - passwordPolicy: required + - uid: hv-ves-kafka-secret + externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}' + type: genericKV + envs: + - name: sasl.jaas.config + value: '{{ .Values.config.someConfig }}' + policy: generate +config: + someConfig: blah # CMPv2 certificate # It is used only when: @@ -113,17 +118,8 @@ service: port_protocol: http nodePort: 22 -aafCreds: - user: admin - password: admin_secret - -credentials: -- name: AAF_USER - uid: *aafCredsUID - key: login -- name: AAF_PASSWORD - uid: *aafCredsUID - key: password +#strimzi kafka config +hvVesKafkaUser: dcae-hv-ves-kafka-user # initial application configuration applicationConfig: @@ -139,48 +135,38 @@ applicationConfig: streams_publishes: ves-3gpp-fault-supervision: type: kafka - aaf_credentials: - username: ${AAF_USER} - password: ${AAF_PASSWORD} kafka_info: - bootstrap_servers: message-router-kafka:9092 + bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS} topic_name: SEC_3GPP_FAULTSUPERVISION_OUTPUT ves-3gpp-provisioning: type: kafka - aaf_credentials: - username: ${AAF_USER} - password: ${AAF_PASSWORD} kafka_info: - bootstrap_servers: message-router-kafka:9092 + bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS} topic_name: SEC_3GPP_PROVISIONING_OUTPUT ves-3gpp-heartbeat: type: kafka - aaf_credentials: - username: ${AAF_USER} - password: ${AAF_PASSWORD} kafka_info: - bootstrap_servers: message-router-kafka:9092 + bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS} topic_name: SEC_3GPP_HEARTBEAT_OUTPUT ves-3gpp-performance-assurance: type: kafka - aaf_credentials: - username: ${AAF_USER} - password: ${AAF_PASSWORD} kafka_info: - bootstrap_servers: message-router-kafka:9092 + bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS} topic_name: SEC_3GPP_PERFORMANCEASSURANCE_OUTPUT perf3gpp: type: kafka - aaf_credentials: - username: ${AAF_USER} - password: ${AAF_PASSWORD} kafka_info: - bootstrap_servers: message-router-kafka:9092 + bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS} topic_name: HV_VES_PERF3GPP applicationEnv: JAVA_OPTS: '-Dlogback.configurationFile=/etc/ONAP/dcae-hv-ves-collector/logback.xml' CBS_CLIENT_CONFIG_PATH: '/app-config-input/application_config.yaml' + KAFKA_BOOTSTRAP_SERVERS: '{{ include "common.release" . }}-strimzi-kafka-bootstrap:9092' + USE_SCRAM: 'true' + JAAS_CONFIG: + secretUid: hv-ves-kafka-secret + key: sasl.jaas.config # Resource Limit flavor -By Default using small flavor: small diff --git a/kubernetes/dcaegen2-services/values.yaml b/kubernetes/dcaegen2-services/values.yaml index a7e24d1d9b..1652f04f0f 100644 --- a/kubernetes/dcaegen2-services/values.yaml +++ b/kubernetes/dcaegen2-services/values.yaml @@ -16,6 +16,7 @@ global: centralizedLoggingEnabled: true + hvVesKafkaUser: dcae-hv-ves-kafka-user ################################################################# # Filebeat Configuration Defaults. @@ -46,6 +47,8 @@ dcae-heartbeat: dcae-hv-ves-collector: enabled: true logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' + config: + jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.hvVesKafkaUser }}' dcae-kpi-ms: enabled: false logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' diff --git a/kubernetes/robot/resources/config/eteshare/config/robot_properties.py b/kubernetes/robot/resources/config/eteshare/config/robot_properties.py index 700b17a970..3c5f9ce73a 100644 --- a/kubernetes/robot/resources/config/eteshare/config/robot_properties.py +++ b/kubernetes/robot/resources/config/eteshare/config/robot_properties.py @@ -188,6 +188,9 @@ GLOBAL_DMAAP_KAFKA_SERVER_NAME = '{{include "robot.ingress.svchost" (dict "root" GLOBAL_DMAAP_KAFKA_SERVER_PORT = '{{include "robot.ingress.port" (dict "root" . "hostname" "message-router-kafka" "port" 9092) }}' GLOBAL_DMAAP_KAFKA_JAAS_USERNAME = '{{ .Values.kafkaJaasUsername }}' GLOBAL_DMAAP_KAFKA_JAAS_PASSWORD = '{{ .Values.kafkaJaasPassword }}' +# strimzi kafka +GLOBAL_KAFKA_BOOTSTRAP_SERVICE = '{{ include "common.release" . }}-strimzi-kafka-bootstrap:9092' +GLOBAL_KAFKA_USER = '{{ .Values.strimziKafkaJaasUsername }}' # DROOL server port and credentials GLOBAL_DROOLS_SERVER_PORT = '{{include "robot.ingress.port" (dict "root" . "hostname" "policy-drools-pdp" "port" 9696) }}' GLOBAL_DROOLS_USERNAME = '{{ .Values.droolsUsername }}' diff --git a/kubernetes/robot/values.yaml b/kubernetes/robot/values.yaml index d8beeedb2a..0b1aa0e71d 100644 --- a/kubernetes/robot/values.yaml +++ b/kubernetes/robot/values.yaml @@ -364,10 +364,14 @@ vidHealthPassword: "AppPassword!1" # DMAAP BC bcUsername: "dmaap-bc@dmaap-bc.onap.org" bcPassword: "demo123456!" + # DMAAP KAFKA JAAS kafkaJaasUsername: "admin" kafkaJaasPassword: "admin_secret" +# STRIMZI KAFKA JAAS +strimziKafkaJaasUsername: "strimzi-kafka-admin" + #OOF oofUsername: "oof@oof.onap.org" oofPassword: "demo123456!"