From: Andreas Geissler Date: Mon, 9 Jan 2023 14:45:16 +0000 (+0000) Subject: Merge "[SO-SDC-DIST] Updgrade sdc-controller to kafka native" X-Git-Tag: 12.0.0~141 X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=3aec3a5badde764720105cf83be8d3a9e2b42e01;hp=8d3ff807180b3721ce2551842ec963eff2554404;p=oom.git Merge "[SO-SDC-DIST] Updgrade sdc-controller to kafka native" --- diff --git a/kubernetes/aai/components/aai-modelloader/resources/config/model-loader.properties b/kubernetes/aai/components/aai-modelloader/resources/config/model-loader.properties index bc53b4b764..a3e27f5517 100644 --- a/kubernetes/aai/components/aai-modelloader/resources/config/model-loader.properties +++ b/kubernetes/aai/components/aai-modelloader/resources/config/model-loader.properties @@ -30,12 +30,14 @@ ml.distribution.KEYSTORE_PASSWORD= ml.distribution.KEYSTORE_FILE= ml.distribution.PASSWORD=OBF:1ks51l8d1o3i1pcc1r2r1e211r391kls1pyj1z7u1njf1lx51go21hnj1y0k1mli1sop1k8o1j651vu91mxw1vun1mze1vv11j8x1k5i1sp11mjc1y161hlr1gm41m111nkj1z781pw31kku1r4p1e391r571pbm1o741l4x1ksp {{ end }} -ml.distribution.CONSUMER_GROUP={{ .Values.config.kafka.sdcTopic.consumerGroup }} -ml.distribution.CONSUMER_ID={{ .Values.config.kafka.sdcTopic.clientId }} +{{- with (first .Values.kafkaUser.acls) }} +ml.distribution.CONSUMER_GROUP={{ .name }} +ml.distribution.CONSUMER_ID={{ .name }}-model-loader ml.distribution.ENVIRONMENT_NAME=AUTO ml.distribution.POLLING_INTERVAL=30 ml.distribution.POLLING_TIMEOUT=20 -ml.distribution.USER=aai +ml.distribution.USER={{ .name }} +{{- end }} ml.distribution.ARTIFACT_TYPES=MODEL_QUERY_SPEC,TOSCA_CSAR # Model Loader AAI REST Client Configuration diff --git a/kubernetes/aai/components/aai-modelloader/templates/aai-sdc-list-kafka-user.yaml b/kubernetes/aai/components/aai-modelloader/templates/aai-sdc-list-kafka-user.yaml deleted file mode 100644 index d7e37e215a..0000000000 --- a/kubernetes/aai/components/aai-modelloader/templates/aai-sdc-list-kafka-user.yaml +++ /dev/null @@ -1,37 +0,0 @@ -{{/* - # Copyright © 2022 Nordix Foundation - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - */}} - -apiVersion: kafka.strimzi.io/v1beta2 -kind: KafkaUser -metadata: - name: {{ include "common.release" . }}-{{ .Values.global.aaiSdcListenerKafkaUser }} - labels: - strimzi.io/cluster: {{ include "common.release" . }}-strimzi -spec: - authentication: - type: {{ .Values.config.kafka.saslMechanism | lower }} - authorization: - type: {{ .Values.config.kafka.authType }} - acls: - - resource: - type: group - name: {{ .Values.config.kafka.sdcTopic.consumerGroup }} - operation: All - - resource: - type: topic - patternType: prefix - name: {{ .Values.config.kafka.sdcTopic.pattern }} - operation: All diff --git a/kubernetes/aai/components/aai-modelloader/templates/deployment.yaml b/kubernetes/aai/components/aai-modelloader/templates/deployment.yaml index d3136d8dda..bab0e0cc8f 100644 --- a/kubernetes/aai/components/aai-modelloader/templates/deployment.yaml +++ b/kubernetes/aai/components/aai-modelloader/templates/deployment.yaml @@ -102,12 +102,11 @@ spec: env: - name: CONFIG_HOME value: /opt/app/model-loader/config/ - - name: SECURITY_PROTOCOL - value: {{ .Values.config.kafka.securityProtocol }} - - name: SASL_MECHANISM - value: {{ .Values.config.kafka.saslMechanism }} - name: SASL_JAAS_CONFIG - value: {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "aai-sdc-kafka-secret" "key" "sasl.jaas.config") | indent 10 }} + valueFrom: + secretKeyRef: + name: {{ include "common.name" . }}-ku + key: sasl.jaas.config volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }} - mountPath: /etc/localtime name: localtime diff --git a/kubernetes/aai/components/aai-modelloader/templates/kafkauser.yaml b/kubernetes/aai/components/aai-modelloader/templates/kafkauser.yaml new file mode 100644 index 0000000000..324a068cf0 --- /dev/null +++ b/kubernetes/aai/components/aai-modelloader/templates/kafkauser.yaml @@ -0,0 +1,16 @@ +{{/* +# Copyright © 2022-23 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{ include "common.kafkauser" . }} diff --git a/kubernetes/aai/components/aai-modelloader/templates/secret.yaml b/kubernetes/aai/components/aai-modelloader/templates/secret.yaml index 70b0857938..d6013c832e 100644 --- a/kubernetes/aai/components/aai-modelloader/templates/secret.yaml +++ b/kubernetes/aai/components/aai-modelloader/templates/secret.yaml @@ -27,5 +27,3 @@ metadata: type: Opaque data: {{ tpl (.Files.Glob "resources/config/auth/*").AsSecrets . | indent 2 }} ---- -{{ include "common.secretFast" . }} \ No newline at end of file diff --git a/kubernetes/aai/components/aai-modelloader/values.yaml b/kubernetes/aai/components/aai-modelloader/values.yaml index 09bb32dd43..825ef06cf1 100644 --- a/kubernetes/aai/components/aai-modelloader/values.yaml +++ b/kubernetes/aai/components/aai-modelloader/values.yaml @@ -18,18 +18,6 @@ # Declare variables to be passed into your templates. global: # global defaults nodePortPrefix: 302 - aaiSdcListenerKafkaUser: aai-sdc-list-user -################################################################# -# Secrets metaconfig -################################################################# -secrets: - - uid: aai-sdc-kafka-secret - externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}' - type: genericKV - envs: - - name: sasl.jaas.config - value: '{{ .Values.config.someConfig }}' - policy: generate ################################################################# # Certificate configuration @@ -74,17 +62,16 @@ restartPolicy: Always flavor: small flavorOverride: small -# application configuration -config: - someConfig: blah - kafka: - securityProtocol: SASL_PLAINTEXT - saslMechanism: SCRAM-SHA-512 - authType: simple - sdcTopic: - pattern: SDC-DIST - consumerGroup: aai - clientId: aai-model-loader +# Strimzi KafkaUser config +kafkaUser: + acls: + - name: aai + type: group + operations: [Read] + - name: SDC-DISTR + type: topic + patternType: prefix + operations: [Read, Write] # default number of instances replicaCount: 1 diff --git a/kubernetes/aai/values.yaml b/kubernetes/aai/values.yaml index 7f4ef863f2..f30222a2cf 100644 --- a/kubernetes/aai/values.yaml +++ b/kubernetes/aai/values.yaml @@ -277,16 +277,12 @@ global: # global defaults # since when this is enabled, it prints a lot of information to console enabled: false - aaiSdcListenerKafkaUser: aai-sdc-list-user - aai-babel: logConfigMapNamePrefix: '{{ include "common.release" . }}-aai' aai-graphadmin: logConfigMapNamePrefix: '{{ include "common.release" . }}-aai' aai-modelloader: logConfigMapNamePrefix: '{{ include "common.release" . }}-aai' - config: - jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.aaiSdcListenerKafkaUser }}' aai-resources: logConfigMapNamePrefix: '{{ include "common.release" . }}-aai' aai-schema-service: diff --git a/kubernetes/policy/components/policy-distribution/resources/config/config.json b/kubernetes/policy/components/policy-distribution/resources/config/config.json index 1aa9044eab..94fc37b4ad 100755 --- a/kubernetes/policy/components/policy-distribution/resources/config/config.json +++ b/kubernetes/policy/components/policy-distribution/resources/config/config.json @@ -56,27 +56,23 @@ "sdcConfiguration":{ "parameterClassName":"org.onap.policy.distribution.reception.handling.sdc.SdcReceptionHandlerConfigurationParameterGroup", "parameters":{ - "asdcAddress": "sdc-be.{{ include "common.namespace" . }}:{{ (eq "true" (include "common.needTLS" .)) | ternary 8443 8080 }}", - "messageBusAddress": [ - "message-router.{{ include "common.namespace" . }}" - ], + "environmentName": "AUTO", + "isUseHttpsWithSDC": {{ (eq "true" (include "common.needTLS" .)) | ternary true false }}, + "keyStorePath": "null", + "keyStorePassword": "null", + "activeserverTlsAuth": false, + "sdcAddress": "sdc-be.{{ include "common.namespace" . }}:{{ (eq "true" (include "common.needTLS" .)) | ternary 8443 8080 }}", "user": "${SDCBE_USER}", "password": "${SDCBE_PASSWORD}", + "consumerGroup": "{{ .Values.config.kafka.sdcTopic.consumerGroup }}", + "consumerId": "{{ .Values.config.kafka.sdcTopic.clientId }}", "pollingInterval":20, "pollingTimeout":30, - "consumerId": "policy-id", "artifactTypes": [ "TOSCA_CSAR", "HEAT" ], - "consumerGroup": "policy-group", - "environmentName": "AUTO", - "keyStorePath": "null", - "keyStorePassword": "null", - "activeserverTlsAuth": false, - "isFilterinEmptyResources": true, - "isUseHttpsWithDmaap": {{ (eq "true" (include "common.needTLS" .)) | ternary true false }}, - "isUseHttpsWithSDC": {{ (eq "true" (include "common.needTLS" .)) | ternary true false }} + "isFilterinEmptyResources": true } } }, diff --git a/kubernetes/policy/components/policy-distribution/templates/deployment.yaml b/kubernetes/policy/components/policy-distribution/templates/deployment.yaml index 4745aac23b..9c71ac1920 100755 --- a/kubernetes/policy/components/policy-distribution/templates/deployment.yaml +++ b/kubernetes/policy/components/policy-distribution/templates/deployment.yaml @@ -75,6 +75,13 @@ spec: - name: {{ include "common.name" . }} image: {{ include "repositoryGenerator.repository" . }}/{{ .Values.image }} imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + env: + - name: SECURITY_PROTOCOL + value: {{ .Values.config.kafka.securityProtocol }} + - name: SASL_MECHANISM + value: {{ .Values.config.kafka.saslMechanism }} + - name: SASL_JAAS_CONFIG + {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "policy-dist-kafka-user" "key" "sasl.jaas.config") | indent 12 }} {{- if .Values.global.aafEnabled }} command: ["sh","-c"] args: ["source {{ .Values.certInitializer.credsPath }}/.ci;\ diff --git a/kubernetes/policy/components/policy-distribution/templates/policy-dist-kafka-user.yaml b/kubernetes/policy/components/policy-distribution/templates/policy-dist-kafka-user.yaml new file mode 100644 index 0000000000..eb721b07bb --- /dev/null +++ b/kubernetes/policy/components/policy-distribution/templates/policy-dist-kafka-user.yaml @@ -0,0 +1,36 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: {{ include "common.release" . }}-{{ .Values.global.policyDistKafkaUser }} + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + authentication: + type: {{ .Values.config.kafka.saslMechanism | lower }} + authorization: + type: {{ .Values.config.kafka.authType }} + acls: + - resource: + type: group + name: {{ .Values.config.kafka.sdcTopic.consumerGroup }} + operation: Read + - resource: + type: topic + patternType: prefix + name: {{ .Values.config.kafka.sdcTopic.pattern }} + operation: All diff --git a/kubernetes/policy/components/policy-distribution/values.yaml b/kubernetes/policy/components/policy-distribution/values.yaml index d9df88ce79..5cdda2f0b8 100755 --- a/kubernetes/policy/components/policy-distribution/values.yaml +++ b/kubernetes/policy/components/policy-distribution/values.yaml @@ -55,7 +55,13 @@ secrets: externalSecret: '{{ tpl (default "" .Values.certStores.trustStorePasswordExternalSecret) . }}' password: '{{ .Values.certStores.trustStorePassword }}' passwordPolicy: required - + - uid: policy-dist-kafka-user + externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}' + type: genericKV + envs: + - name: sasl.jaas.config + value: '{{ .Values.config.someConfig }}' + policy: generate ################################################################# # Global configuration defaults. ################################################################# @@ -67,13 +73,24 @@ global: # Application configuration defaults. ################################################################# # application image -image: onap/policy-distribution:2.8.2 +image: onap/policy-distribution:2.9.0 pullPolicy: Always # flag to enable debugging - application support required debugEnabled: false # application configuration +config: + someConfig: blah + kafka: + bootstrapServer: strimzi-kafka-bootstrap:9092 + securityProtocol: SASL_PLAINTEXT + saslMechanism: SCRAM-SHA-512 + authType: simple + sdcTopic: + pattern: SDC-DIST + consumerGroup: policy-group + clientId: policy-distribution restServer: user: healthcheck diff --git a/kubernetes/policy/values.yaml b/kubernetes/policy/values.yaml index 45ade02a49..c9e236a65c 100755 --- a/kubernetes/policy/values.yaml +++ b/kubernetes/policy/values.yaml @@ -38,6 +38,7 @@ global: name: postgres kafkaBootstrap: strimzi-kafka-bootstrap policyKafkaUser: policy-kafka-user + policyDistKafkaUser: policy-dist-kafka-user ################################################################# # Secrets metaconfig @@ -128,7 +129,7 @@ policy-distribution: enabled: true db: *dbSecretsHook config: - jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.policyKafkaUser }}' + jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.policyDistKafkaUser }}' policy-clamp-ac-k8s-ppnt: enabled: true config: