[HV-VES-STRIMZI] Migrate hv-ves to use strimzi kafka 24/128824/11
authorefiacor <fiachra.corcoran@est.tech>
Mon, 25 Apr 2022 12:27:18 +0000 (13:27 +0100)
committerKrzysztof Kuzmicki <krzysztof.kuzmicki@nokia.com>
Mon, 9 May 2022 10:51:59 +0000 (10:51 +0000)
Move hv-ves to use strimzi kafka

Signed-off-by: efiacor <fiachra.corcoran@est.tech>
Change-Id: Ia441d060771186ba41465c8622291ff73b7561aa
Issue-ID: DMAAP-1705

kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl
kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-kafka-user.yaml [new file with mode: 0644]
kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-topics.yaml [new file with mode: 0644]
kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml
kubernetes/dcaegen2-services/values.yaml
kubernetes/robot/resources/config/eteshare/config/robot_properties.py
kubernetes/robot/values.yaml

index fbaaedf..9781e33 100644 (file)
@@ -58,7 +58,7 @@ the the literal string "An example value".
     {{- range $envName, $envValue := .Values.applicationEnv }}
       {{- if kindIs "string" $envValue }}
 - name: {{ $envName }}
-  value: {{ $envValue | quote }}
+  value: {{ tpl $envValue $global | quote }}
       {{- else }}
         {{ if or (not $envValue.secretUid) (not $envValue.key) }}
           {{ fail (printf "Env %s definition is not a string and does not contain secretUid or key fields" $envName) }}
diff --git a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-kafka-user.yaml b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-kafka-user.yaml
new file mode 100644 (file)
index 0000000..ff977aa
--- /dev/null
@@ -0,0 +1,47 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: {{ include "common.release" . }}-{{ .Values.hvVesKafkaUser }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  authentication:
+    type: scram-sha-512
+  authorization:
+    type: simple
+    acls:
+    - resource:
+        type: topic
+        name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-fault-supervision" "kafka_info" "topic_name" }}
+      operation: Write
+    - resource:
+        type: topic
+        name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-provisioning" "kafka_info" "topic_name" }}
+      operation: Write
+    - resource:
+        type: topic
+        name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-heartbeat" "kafka_info" "topic_name" }}
+      operation: Write
+    - resource:
+        type: topic
+        name: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-performance-assurance" "kafka_info" "topic_name" }}
+      operation: Write
+    - resource:
+        type: topic
+        name: {{ .Values.applicationConfig.streams_publishes.perf3gpp.kafka_info.topic_name }}
+      operation: Write
diff --git a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-topics.yaml b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/templates/hv-ves-topics.yaml
new file mode 100644 (file)
index 0000000..e407758
--- /dev/null
@@ -0,0 +1,79 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: ves-3gpp-fault-supervision
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-fault-supervision" "kafka_info" "topic_name" }}
+  partitions: 10
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: ves-3gpp-provisioning
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-provisioning" "kafka_info" "topic_name" }}
+  partitions: 10
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: ves-3gpp-heartbeat
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-heartbeat" "kafka_info" "topic_name" }}
+  partitions: 10
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: ves-3gpp-performance-assurance
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ index .Values "applicationConfig" "streams_publishes" "ves-3gpp-performance-assurance" "kafka_info" "topic_name" }}
+  partitions: 10
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: perf3gpp
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ .Values.applicationConfig.streams_publishes.perf3gpp.kafka_info.topic_name }}
+  partitions: 10
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
index 5a62836..67add37 100644 (file)
@@ -2,6 +2,7 @@
 # ================================================================================
 # Copyright (c) 2021-2022 J. F. Lucas. All rights reserved.
 # Copyright (c) 2021-2022 Nokia.  All rights reserved.
+# Copyright © 2022 Nordix Foundation
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -61,11 +62,15 @@ certDirectory: /etc/ves-hv/ssl
 tlsServer: true
 
 secrets:
-  - uid: &aafCredsUID aafcreds
-    type: basicAuth
-    login: '{{ .Values.aafCreds.user }}'
-    password: '{{ .Values.aafCreds.password }}'
-    passwordPolicy: required
+  - uid: hv-ves-kafka-secret
+    externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+    type: genericKV
+    envs:
+      - name: sasl.jaas.config
+        value: '{{ .Values.config.someConfig }}'
+        policy: generate
+config:
+  someConfig: blah
 
 # CMPv2 certificate
 # It is used only when:
@@ -113,17 +118,8 @@ service:
       port_protocol: http
       nodePort: 22
 
-aafCreds:
-  user: admin
-  password: admin_secret
-
-credentials:
-- name: AAF_USER
-  uid: *aafCredsUID
-  key: login
-- name: AAF_PASSWORD
-  uid: *aafCredsUID
-  key: password
+#strimzi kafka config
+hvVesKafkaUser: dcae-hv-ves-kafka-user
 
 # initial application configuration
 applicationConfig:
@@ -139,48 +135,38 @@ applicationConfig:
   streams_publishes:
     ves-3gpp-fault-supervision:
       type: kafka
-      aaf_credentials:
-        username: ${AAF_USER}
-        password: ${AAF_PASSWORD}
       kafka_info:
-        bootstrap_servers: message-router-kafka:9092
+        bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS}
         topic_name: SEC_3GPP_FAULTSUPERVISION_OUTPUT
     ves-3gpp-provisioning:
       type: kafka
-      aaf_credentials:
-        username: ${AAF_USER}
-        password: ${AAF_PASSWORD}
       kafka_info:
-        bootstrap_servers: message-router-kafka:9092
+        bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS}
         topic_name: SEC_3GPP_PROVISIONING_OUTPUT
     ves-3gpp-heartbeat:
       type: kafka
-      aaf_credentials:
-        username: ${AAF_USER}
-        password: ${AAF_PASSWORD}
       kafka_info:
-        bootstrap_servers: message-router-kafka:9092
+        bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS}
         topic_name: SEC_3GPP_HEARTBEAT_OUTPUT
     ves-3gpp-performance-assurance:
       type: kafka
-      aaf_credentials:
-        username: ${AAF_USER}
-        password: ${AAF_PASSWORD}
       kafka_info:
-        bootstrap_servers: message-router-kafka:9092
+        bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS}
         topic_name: SEC_3GPP_PERFORMANCEASSURANCE_OUTPUT
     perf3gpp:
       type: kafka
-      aaf_credentials:
-        username: ${AAF_USER}
-        password: ${AAF_PASSWORD}
       kafka_info:
-        bootstrap_servers: message-router-kafka:9092
+        bootstrap_servers: ${KAFKA_BOOTSTRAP_SERVERS}
         topic_name: HV_VES_PERF3GPP
 
 applicationEnv:
   JAVA_OPTS: '-Dlogback.configurationFile=/etc/ONAP/dcae-hv-ves-collector/logback.xml'
   CBS_CLIENT_CONFIG_PATH: '/app-config-input/application_config.yaml'
+  KAFKA_BOOTSTRAP_SERVERS: '{{ include "common.release" . }}-strimzi-kafka-bootstrap:9092'
+  USE_SCRAM: 'true'
+  JAAS_CONFIG:
+    secretUid: hv-ves-kafka-secret
+    key: sasl.jaas.config
 
 # Resource Limit flavor -By Default using small
 flavor: small
index a7e24d1..1652f04 100644 (file)
@@ -16,6 +16,7 @@
 
 global:
   centralizedLoggingEnabled: true
+  hvVesKafkaUser: dcae-hv-ves-kafka-user
 
 #################################################################
 # Filebeat Configuration Defaults.
@@ -46,6 +47,8 @@ dcae-heartbeat:
 dcae-hv-ves-collector:
   enabled: true
   logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services'
+  config:
+    jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.hvVesKafkaUser }}'
 dcae-kpi-ms:
   enabled: false
   logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services'
index 700b17a..3c5f9ce 100644 (file)
@@ -188,6 +188,9 @@ GLOBAL_DMAAP_KAFKA_SERVER_NAME = '{{include "robot.ingress.svchost" (dict "root"
 GLOBAL_DMAAP_KAFKA_SERVER_PORT = '{{include "robot.ingress.port" (dict "root" . "hostname" "message-router-kafka" "port" 9092) }}'
 GLOBAL_DMAAP_KAFKA_JAAS_USERNAME = '{{ .Values.kafkaJaasUsername }}'
 GLOBAL_DMAAP_KAFKA_JAAS_PASSWORD = '{{ .Values.kafkaJaasPassword }}'
+# strimzi kafka
+GLOBAL_KAFKA_BOOTSTRAP_SERVICE = '{{ include "common.release" . }}-strimzi-kafka-bootstrap:9092'
+GLOBAL_KAFKA_USER = '{{ .Values.strimziKafkaJaasUsername }}'
 # DROOL server port and credentials
 GLOBAL_DROOLS_SERVER_PORT = '{{include "robot.ingress.port" (dict "root" . "hostname" "policy-drools-pdp" "port" 9696) }}'
 GLOBAL_DROOLS_USERNAME = '{{ .Values.droolsUsername }}'
index d8beeed..0b1aa0e 100644 (file)
@@ -364,10 +364,14 @@ vidHealthPassword: "AppPassword!1"
 # DMAAP BC
 bcUsername: "dmaap-bc@dmaap-bc.onap.org"
 bcPassword: "demo123456!"
+
 # DMAAP KAFKA JAAS
 kafkaJaasUsername: "admin"
 kafkaJaasPassword: "admin_secret"
 
+# STRIMZI KAFKA JAAS
+strimziKafkaJaasUsername: "strimzi-kafka-admin"
+
 #OOF
 oofUsername: "oof@oof.onap.org"
 oofPassword: "demo123456!"