From 3422fe96531c951a3209ac3c2c6a3e8308559812 Mon Sep 17 00:00:00 2001 From: efiacor Date: Tue, 12 Jul 2022 13:19:53 +0100 Subject: [PATCH] [SDC] Add strimzi kafka setup on sdc-be Add SDC KafkaTopics and KafkaUser Signed-off-by: efiacor Change-Id: I78542539567d2e3b67376b154ad069df3d1dad55 Issue-ID: DMAAP-1744 --- .../sdc/components/sdc-be/templates/configmap.yaml | 2 ++ .../components/sdc-be/templates/deployment.yaml | 8 +++++ .../sdc-be/templates/sdc-be-kafka-user.yaml | 39 +++++++++++++++++++++ .../sdc-be/templates/sdc-distro-topics.yaml | 40 ++++++++++++++++++++++ .../sdc/components/sdc-be/templates/secret.yaml | 17 +++++++++ kubernetes/sdc/components/sdc-be/values.yaml | 32 +++++++++++++++-- kubernetes/sdc/components/sdc-cs/values.yaml | 4 +-- kubernetes/sdc/components/sdc-fe/values.yaml | 2 +- .../sdc/components/sdc-onboarding-be/values.yaml | 4 +-- .../sdc/resources/config/environments/AUTO.json | 7 ++++ kubernetes/sdc/values.yaml | 11 ++++++ kubernetes/strimzi/templates/strimzi-kafka.yaml | 1 + kubernetes/strimzi/values.yaml | 1 + 13 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml create mode 100644 kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml create mode 100644 kubernetes/sdc/components/sdc-be/templates/secret.yaml diff --git a/kubernetes/sdc/components/sdc-be/templates/configmap.yaml b/kubernetes/sdc/components/sdc-be/templates/configmap.yaml index aa632f33f4..332cd74661 100644 --- a/kubernetes/sdc/components/sdc-be/templates/configmap.yaml +++ b/kubernetes/sdc/components/sdc-be/templates/configmap.yaml @@ -27,3 +27,5 @@ metadata: heritage: {{ .Release.Service }} data: {{ tpl (.Files.Glob "resources/config/logging/*").AsConfig . | indent 2 }} + + diff --git a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml index 24f169a5b9..16fc57aa92 100644 --- a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml @@ -31,8 +31,10 @@ spec: args: - --container-name - "sdc-onboarding-be" + {{- if not .Values.global.kafka.useKafka }} - --container-name - "message-router" + {{- end }} env: - name: NAMESPACE valueFrom: @@ -159,6 +161,12 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + {{- if .Values.global.kafka.useKafka }} + - name: SASL_JAAS_CONFIG + {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "sdc-be-kafka-secret" "key" "sasl.jaas.config") | indent 12 }} + - name: USE_KAFKA + value: {{ .Values.global.kafka.useKafka | quote }} + {{- end }} volumeMounts: - name: sdc-environments mountPath: /app/jetty/chef-solo/environments/ diff --git a/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml b/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml new file mode 100644 index 0000000000..6cd7f93c5a --- /dev/null +++ b/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml @@ -0,0 +1,39 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{- if .Values.global.kafka.useKafka }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: {{ include "common.release" . }}-{{ .Values.global.kafka.sdcBeKafkaUser }} + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + authentication: + type: {{ .Values.config.kafka.saslMech }} + authorization: + type: {{ .Values.config.kafka.authType }} + acls: + - resource: + type: group + name: {{ .Values.config.kafka.topicConsumer.groupId }}-{{ .Values.env.name }} + operation: Read + - resource: + type: topic + patternType: prefix + name: {{ .Values.config.kafka.topicConsumer.pattern }} + operation: All +{{- end }} diff --git a/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml b/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml new file mode 100644 index 0000000000..9a6f7579e8 --- /dev/null +++ b/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml @@ -0,0 +1,40 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- if .Values.global.kafka.useKafka }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: sdc-distro-notif-topic + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ .Values.global.kafka.topics.sdcDistNotifTopic }}-{{ .Values.env.name }} + config: + retention.ms: {{ .Values.config.kafka.topicRetentionMs }} + segment.bytes: {{ .Values.config.kafka.topicSegmentBytes }} +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: sdc-distro-status-topic + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + topicName: {{ .Values.global.kafka.topics.sdcDistStatusTopic }}-{{ .Values.env.name }} + config: + retention.ms: {{ .Values.config.kafka.topicRetentionMs }} + segment.bytes: {{ .Values.config.kafka.topicSegmentBytes }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/sdc/components/sdc-be/templates/secret.yaml b/kubernetes/sdc/components/sdc-be/templates/secret.yaml new file mode 100644 index 0000000000..bb5091f01a --- /dev/null +++ b/kubernetes/sdc/components/sdc-be/templates/secret.yaml @@ -0,0 +1,17 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{ include "common.secretFast" . }} diff --git a/kubernetes/sdc/components/sdc-be/values.yaml b/kubernetes/sdc/components/sdc-be/values.yaml index 9f216352c0..faf46e5549 100644 --- a/kubernetes/sdc/components/sdc-be/values.yaml +++ b/kubernetes/sdc/components/sdc-be/values.yaml @@ -1,5 +1,6 @@ # Copyright © 2017 Amdocs, Bell Canada # Modifications Copyright © 2018 AT&T, ZTE +# Modifications Copyright © 2022 Nordix Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,13 +31,20 @@ global: replicaCount: 3 clusterName: cassandra dataCenter: Pod + # Strimzi kafka config + kafka: + useKafka: overridden-from-parent-values-yaml + sdcBeKafkaUser: overridden-from-parent-values-yaml + topics: + sdcDistNotifTopic: overridden-from-parent-values-yaml + sdcDistStatusTopic: overridden-from-parent-values-yaml ################################################################# # Application configuration defaults. ################################################################# # application image -image: onap/sdc-backend-all-plugins:1.11.9 -backendInitImage: onap/sdc-backend-init:1.11.9 +image: onap/sdc-backend-all-plugins:1.12.0 +backendInitImage: onap/sdc-backend-init:1.12.0 pullPolicy: Always @@ -70,9 +78,29 @@ certInitializer: ################################################################# # SDC Config part ################################################################# + +secrets: + - uid: sdc-be-kafka-secret + externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}' + type: genericKV + envs: + - name: sasl.jaas.config + value: '{{ .Values.config.someConfig }}' + policy: generate + config: javaOptions: "-Xmx1536m -Xms1536m" cassandraSslEnabled: "false" + # Strimzi kafka config + kafka: + saslMech: scram-sha-512 + securityProtocol: SASL_PLAINTEXT + authType: simple + topicRetentionMs: 7200000 + topicSegmentBytes: 1073741824 + topicConsumer: + pattern: SDC-DIST + groupId: sdc # default number of instances replicaCount: 1 diff --git a/kubernetes/sdc/components/sdc-cs/values.yaml b/kubernetes/sdc/components/sdc-cs/values.yaml index 46208a21c6..b3dfbfa0ba 100644 --- a/kubernetes/sdc/components/sdc-cs/values.yaml +++ b/kubernetes/sdc/components/sdc-cs/values.yaml @@ -51,8 +51,8 @@ cassandra: # application image repository: nexus3.onap.org:10001 -image: onap/sdc-cassandra:1.11.9 -cassandraInitImage: onap/sdc-cassandra-init:1.11.9 +image: onap/sdc-cassandra:1.12.0 +cassandraInitImage: onap/sdc-cassandra-init:1.12.0 pullPolicy: Always config: diff --git a/kubernetes/sdc/components/sdc-fe/values.yaml b/kubernetes/sdc/components/sdc-fe/values.yaml index 76c9e13767..abac356d40 100644 --- a/kubernetes/sdc/components/sdc-fe/values.yaml +++ b/kubernetes/sdc/components/sdc-fe/values.yaml @@ -47,7 +47,7 @@ certInitializer: # Application configuration defaults. ################################################################# # application image -image: onap/sdc-frontend:1.11.9 +image: onap/sdc-frontend:1.12.0 pullPolicy: Always config: diff --git a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml index 45cbcd617a..1db5af98d0 100644 --- a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml +++ b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml @@ -59,8 +59,8 @@ certInitializer: # Application configuration defaults. ################################################################# # application image -image: onap/sdc-onboard-backend:1.11.9 -onboardingInitImage: onap/sdc-onboard-cassandra-init:1.11.9 +image: onap/sdc-onboard-backend:1.12.0 +onboardingInitImage: onap/sdc-onboard-cassandra-init:1.12.0 pullPolicy: Always # flag to enable debugging - application support required diff --git a/kubernetes/sdc/resources/config/environments/AUTO.json b/kubernetes/sdc/resources/config/environments/AUTO.json index c9535592d8..aee666f4af 100755 --- a/kubernetes/sdc/resources/config/environments/AUTO.json +++ b/kubernetes/sdc/resources/config/environments/AUTO.json @@ -35,6 +35,13 @@ "message-router.{{include "common.namespace" .}}" ] }, + "Kafka": { + "bootstrap": "{{ include "common.release" . }}-{{ .Values.global.kafka.kafkaBootstrap }}" + }, + "DistributionTopics": { + "notificationTopicName": "{{ .Values.global.kafka.topics.sdcDistNotifTopic }}", + "statusTopicName": "{{ .Values.global.kafka.topics.sdcDistStatusTopic }}" + }, "Nodes": { "CS": [ "{{.Values.global.sdc_cassandra.serviceName}}.{{include "common.namespace" .}}" diff --git a/kubernetes/sdc/values.yaml b/kubernetes/sdc/values.yaml index e7643069bf..60a361eae6 100644 --- a/kubernetes/sdc/values.yaml +++ b/kubernetes/sdc/values.yaml @@ -1,5 +1,6 @@ # Copyright © 2017 Amdocs, Bell Canada # Modifications Copyright © 2018 AT&T, ZTE +# Modifications Copyright © 2021 Nordix Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,9 +42,19 @@ global: clusterName: cassandra dataCenter: Pod centralizedLoggingEnabled: true + # Kafka config + kafka: + useKafka: true + sdcBeKafkaUser: sdc-be-kafka-user + kafkaBootstrap: strimzi-kafka-bootstrap:9092 + topics: + sdcDistNotifTopic: SDC-DISTR-NOTIF-TOPIC + sdcDistStatusTopic: SDC-DISTR-STATUS-TOPIC sdc-be: logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc' + config: + jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.kafka.sdcBeKafkaUser }}' sdc-fe: logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc' sdc-onboarding-be: diff --git a/kubernetes/strimzi/templates/strimzi-kafka.yaml b/kubernetes/strimzi/templates/strimzi-kafka.yaml index c7dc2aff08..03ee56a7a4 100644 --- a/kubernetes/strimzi/templates/strimzi-kafka.yaml +++ b/kubernetes/strimzi/templates/strimzi-kafka.yaml @@ -70,6 +70,7 @@ spec: min.insync.replicas: {{ .Values.replicaCount }} offsets.topic.replication.factor: {{ .Values.replicaCount }} transaction.state.log.replication.factor: {{ .Values.replicaCount }} + num.partitions: {{ .Values.numPartitions }} transaction.state.log.min.isr: {{ .Values.replicaCount }} log.message.format.version: {{ .Values.version }} inter.broker.protocol.version: {{ .Values.version }} diff --git a/kubernetes/strimzi/values.yaml b/kubernetes/strimzi/values.yaml index dcfa913604..99ccde5040 100644 --- a/kubernetes/strimzi/values.yaml +++ b/kubernetes/strimzi/values.yaml @@ -24,6 +24,7 @@ global: # Application configuration defaults. ################################################################# replicaCount: 3 +numPartitions: 10 kafkaInternalPort: 9092 saslMechanism: scram-sha-512 version: 3.2.3 -- 2.16.6