Single K8S Node Of Deploying SDNC Cluster POC 67/25467/23
authorHao Kuang <Hao.Kuang@amdocs.com>
Fri, 15 Dec 2017 20:38:03 +0000 (20:38 +0000)
committerHao Kuang <Hao.Kuang@amdocs.com>
Wed, 20 Dec 2017 16:43:29 +0000 (16:43 +0000)
It's a reference of how to deploy sdnc cluster under
single node k8s environment.

Currently the config of nfs for mysql db only
supports one k8s node. For mutiple nodes, because
of dockernfs mount, nfs provisioner has conflict with
it, which fails mount operation of SDNC pod.

Also, if SDNC needs to fully utilize mysql cluster under
k8s, in core project, DbLibService.java should be enhanced
to support that writing to master only and reading from them.

Issue-ID: SDNC-163
Change-Id: I85ad327f2fe04fe8e9a135289d7c25a917234438
Signed-off-by: Hao Kuang <Hao.Kuang@amdocs.com>
kubernetes/sdnc/templates/all-services.yaml
kubernetes/sdnc/templates/db-deployment.yaml [deleted file]
kubernetes/sdnc/templates/db-statefulset.yaml [new file with mode: 0644]
kubernetes/sdnc/templates/mysql-configmap.yaml [new file with mode: 0644]
kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml [new file with mode: 0644]
kubernetes/sdnc/templates/sdnc-data-storageclass.yaml [new file with mode: 0644]
kubernetes/sdnc/templates/sdnc-pv-pvc.yaml [deleted file]
kubernetes/sdnc/templates/sdnc-statefulset.yaml [moved from kubernetes/sdnc/templates/sdnc-deployment.yaml with 79% similarity]
kubernetes/sdnc/values.yaml

index cfb24bf..9425ef4 100644 (file)
@@ -13,6 +13,22 @@ spec:
     app: sdnc-dbhost
   clusterIP: None
 ---
+# Client service for connecting to any MySQL instance for reads.
+# Only master: sdnc-dbhost-0 accepts the write request.
+apiVersion: v1
+kind: Service
+metadata:
+  name: dbhost-read
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+  labels:
+    app: sdnc-dbhost
+spec:
+  ports:
+  - name: sdnc-dbhost
+    port: 3306
+  selector:
+    app: sdnc-dbhost
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -26,7 +42,6 @@ spec:
   selector:
     app: sdnc-dbhost
   clusterIP: None
-#{{ end }}
 ---
 apiVersion: v1
 kind: Service
@@ -41,7 +56,8 @@ spec:
   selector:
     app: sdnc-dbhost
   clusterIP: None
-#{{ if not .Values.disableSdncSdncDgbuilder }}
+#{{ end }}
+#{{ if not .Values.disableSdncSdnc }}
 ---
 apiVersion: v1
 kind: Service
@@ -59,8 +75,6 @@ spec:
   type: NodePort
   selector:
     app: sdnc-dgbuilder
-#{{ end }}
-#{{ if not .Values.disableSdncSdnc }}
 ---
 apiVersion: v1
 kind: Service
@@ -91,9 +105,34 @@ spec:
     port: 8201
     targetPort: 8101
     nodePort: {{ .Values.nodePortPrefix }}08
+  - name: "sdnc-jolokia-port-8080"
+    port: 8280
+    targetPort: 8080
+    nodePort: {{ .Values.nodePortPrefix }}46
   type: NodePort
   selector:
     app: sdnc
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: nfs-provisioner
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+  labels:
+    app: nfs-provisioner
+spec:
+  ports:
+    - name: nfs
+      port: 2049
+    - name: mountd
+      port: 20048
+    - name: rpcbind
+      port: 111
+    - name: rpcbind-udp
+      port: 111
+      protocol: UDP
+  selector:
+    app: nfs-provisioner
 #{{ end }}
 #{{ if not .Values.disableSdncSdncPortal }}
 ---
@@ -124,3 +163,24 @@ spec:
   selector:
     app: sdnc-portal
 #{{ end }}
+#{{ if .Values.enableODLCluster }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sdnhost-cluster
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+  labels:
+    app: sdnc
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+spec:
+  ports:
+   - name: "sdnc-cluster-port"
+     port: 2550
+  clusterIP: None
+  selector:
+    app: sdnc
+  sessionAffinity: None
+  type: ClusterIP
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/db-deployment.yaml b/kubernetes/sdnc/templates/db-deployment.yaml
deleted file mode 100644 (file)
index e2723b5..0000000
+++ /dev/null
@@ -1,48 +0,0 @@
-#{{ if not .Values.disableSdncSdncDbhost }}
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: sdnc-dbhost
-  namespace: "{{ .Values.nsPrefix }}-sdnc"
-spec:
-  selector:
-    matchLabels:
-      app: sdnc-dbhost
-  template:
-    metadata:
-      labels:
-        app: sdnc-dbhost
-      name: sdnc-dbhost
-    spec:
-      containers:
-      - env:
-        - name: MYSQL_ROOT_PASSWORD
-          value: openECOMP1.0
-        - name: MYSQL_ROOT_HOST
-          value: '%'
-        image: {{ .Values.image.mysqlServer }}
-        imagePullPolicy: {{ .Values.pullPolicy }}
-        name: sdnc-db-container
-        volumeMounts:
-        - mountPath: /etc/localtime
-          name: localtime
-          readOnly: true
-        - mountPath: /var/lib/mysql
-          name: sdnc-data
-        ports:
-        - containerPort: 3306
-        readinessProbe:
-          tcpSocket:
-            port: 3306
-          initialDelaySeconds: 5
-          periodSeconds: 10
-      volumes:
-      - name: localtime
-        hostPath:
-          path: /etc/localtime
-      - name: sdnc-data
-        persistentVolumeClaim:
-          claimName: sdnc-db
-      imagePullSecrets:
-      - name: "{{ .Values.nsPrefix }}-docker-registry-key"
-#{{ end }}
\ No newline at end of file
diff --git a/kubernetes/sdnc/templates/db-statefulset.yaml b/kubernetes/sdnc/templates/db-statefulset.yaml
new file mode 100644 (file)
index 0000000..f2e079d
--- /dev/null
@@ -0,0 +1,193 @@
+#{{ if not .Values.disableSdncSdncDbhost }}
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: sdnc-dbhost
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+spec:
+  serviceName: "dbhost"
+  replicas: {{ .Values.numberOfDbReplicas }}
+  selector:
+    matchLabels:
+      app: sdnc-dbhost
+  template:
+    metadata:
+      labels:
+        app: sdnc-dbhost
+      name: sdnc-dbhost
+    spec:
+      initContainers:
+      - name: init-mysql
+        image: {{ .Values.image.mysql }}
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        command:
+        - bash
+        - "-c"
+        - |
+          set -ex
+          # Generate mysql server-id from pod ordinal index.
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          echo BASH_REMATCH=${BASH_REMATCH}
+          echo [mysqld] > /mnt/conf.d/server-id.cnf
+          # Add an offset to avoid reserved server-id=0 value.
+          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
+          # Copy appropriate conf.d files from config-map to emptyDir.
+          if [[ $ordinal -eq 0 ]]; then
+            cp /mnt/config-map/master.cnf /mnt/conf.d/
+          else
+            cp /mnt/config-map/slave.cnf /mnt/conf.d/
+          fi
+        volumeMounts:
+        - name: conf
+          mountPath: /mnt/conf.d
+        - name: config-map
+          mountPath: /mnt/config-map
+      - name: clone-mysql
+        image: {{ .Values.image.xtrabackup }}
+        env:
+        - name: MYSQL_ROOT_PASSWORD
+          value: openECOMP1.0
+        command:
+        - bash
+        - "-c"
+        - |
+          set -ex
+          # Skip the clone if data already exists.
+          [[ -d /var/lib/mysql/mysql ]] && exit 0
+          # Skip the clone on master (ordinal index 0).
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          echo ${BASH_REMATCH}
+          [[ $ordinal -eq 0 ]] && exit 0
+          # Clone data from previous peer.
+          ncat --recv-only sdnc-dbhost-$(($ordinal-1)).dbhost.{{ .Values.nsPrefix }}-sdnc 3307 | xbstream -x -C /var/lib/mysql
+          # Prepare the backup.
+          xtrabackup --user=root --password=$MYSQL_ROOT_PASSWORD --prepare --target-dir=/var/lib/mysql
+          ls -l /var/lib/mysql
+        volumeMounts:
+        - name: sdnc-data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      containers:
+      - env:
+        - name: MYSQL_ROOT_PASSWORD
+          value: openECOMP1.0
+        - name: MYSQL_ROOT_HOST
+          value: '%'
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "0"
+        image: {{ .Values.image.mysql }}
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        name: sdnc-db-container
+        volumeMounts:
+        - mountPath: /var/lib/mysql
+          name: sdnc-data
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        ports:
+        - containerPort: 3306
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        livenessProbe:
+          exec:
+            command: ["mysqladmin", "ping"]
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          tcpSocket:
+            port: 3306
+          initialDelaySeconds: 5
+          periodSeconds: 10
+      - name: xtrabackup
+        image: {{ .Values.image.xtrabackup }}
+        env:
+        - name: MYSQL_ROOT_PASSWORD
+          value: openECOMP1.0
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        command:
+        - bash
+        - "-c"
+        - |
+          set -ex
+          cd /var/lib/mysql
+          ls -l
+          # Determine binlog position of cloned data, if any.
+          if [[ -f xtrabackup_slave_info ]]; then
+            echo "Inside xtrabackup_slave_info"
+            # XtraBackup already generated a partial "CHANGE MASTER TO" query
+            # because we're cloning from an existing slave.
+            mv xtrabackup_slave_info change_master_to.sql.in
+            # Ignore xtrabackup_binlog_info in this case (it's useless).
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            echo "Inside xtrabackup_binlog_info"
+            # We're cloning directly from master. Parse binlog position.
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+
+          # Check if we need to complete a clone by starting replication.
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+            ordinal=${BASH_REMATCH[1]}
+            echo $ordinal
+            until mysql --user=root --password=$MYSQL_ROOT_PASSWORD -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+
+            echo "Initializing replication from clone position"
+            # In case of container restart, attempt this at-most-once.
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql --user=root --password=$MYSQL_ROOT_PASSWORD -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST="sdnc-dbhost-0.dbhost.{{ .Values.nsPrefix }}-sdnc",
+            MASTER_USER="root",
+            MASTER_PASSWORD="$MYSQL_ROOT_PASSWORD",
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+
+          # Start a server to send backups when requested by peers.
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --user=root --password=$MYSQL_ROOT_PASSWORD --backup --slave-info --stream=xbstream --host=127.0.0.1"
+        volumeMounts:
+        - name: sdnc-data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+  volumeClaimTemplates:
+  - metadata:
+      name: sdnc-data
+      annotations:
+        volume.beta.kubernetes.io/storage-class: "{{ .Values.nsPrefix }}-sdnc-data"
+    spec:
+      accessModes: ["ReadWriteMany"]
+      resources:
+        requests:
+          storage: 1Gi
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/mysql-configmap.yaml b/kubernetes/sdnc/templates/mysql-configmap.yaml
new file mode 100644 (file)
index 0000000..deaa997
--- /dev/null
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+  labels:
+    app: mysql
+data:
+  master.cnf: |
+    # Apply this config only on the master.
+    [mysqld]
+    log-bin
+    [localpathprefix]
+    master
+  slave.cnf: |
+    # Apply this config only on slaves.
+    [mysqld]
+    super-read-only
+    [localpathprefix]
+    slave
diff --git a/kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml b/kubernetes/sdnc/templates/nfs-provisoner-deployment.yaml
new file mode 100644 (file)
index 0000000..9d3feef
--- /dev/null
@@ -0,0 +1,55 @@
+#{{ if not .Values.disableSdncSdncDbhost }}
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: nfs-provisioner
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: nfs-provisioner
+    spec:
+      containers:
+        - name: nfs-provisioner
+          image: quay.io/kubernetes_incubator/nfs-provisioner:v1.0.8
+          ports:
+            - name: nfs
+              containerPort: 2049
+            - name: mountd
+              containerPort: 20048
+            - name: rpcbind
+              containerPort: 111
+            - name: rpcbind-udp
+              containerPort: 111
+              protocol: UDP
+          securityContext:
+            capabilities:
+              add:
+                - DAC_READ_SEARCH
+                - SYS_RESOURCE
+          args:
+            - "-provisioner=sdnc/nfs"
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: SERVICE_NAME
+              value: nfs-provisioner
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: export-volume
+              mountPath: /export
+      volumes:
+        - name: export-volume
+          hostPath:
+            path: /dockerdata-nfs/{{ .Values.nsPrefix }}/sdnc/data
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/sdnc-data-storageclass.yaml b/kubernetes/sdnc/templates/sdnc-data-storageclass.yaml
new file mode 100644 (file)
index 0000000..7c46cf4
--- /dev/null
@@ -0,0 +1,8 @@
+#{{ if not .Values.disableSdncSdncDbhost }}
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: "{{ .Values.nsPrefix }}-sdnc-data"
+  namespace: "{{ .Values.nsPrefix }}-sdnc"
+provisioner: sdnc/nfs
+#{{ end }}
diff --git a/kubernetes/sdnc/templates/sdnc-pv-pvc.yaml b/kubernetes/sdnc/templates/sdnc-pv-pvc.yaml
deleted file mode 100644 (file)
index f45d8f8..0000000
+++ /dev/null
@@ -1,32 +0,0 @@
-#{{ if not .Values.disableSdncSdncDbhost }}
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: "{{ .Values.nsPrefix }}-sdnc-db"
-  namespace: "{{ .Values.nsPrefix }}-sdnc"
-  labels:
-    name: "{{ .Values.nsPrefix }}-sdnc-db"
-spec:
-  capacity:
-    storage: 2Gi
-  accessModes:
-    - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
-  hostPath:
-    path: /dockerdata-nfs/{{ .Values.nsPrefix }}/sdnc/data
----
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: sdnc-db
-  namespace: "{{ .Values.nsPrefix }}-sdnc"
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 2Gi
-  selector:
-    matchLabels:
-      name: "{{ .Values.nsPrefix }}-sdnc-db"
-#{{ end }}
\ No newline at end of file
@@ -1,10 +1,13 @@
 #{{ if not .Values.disableSdncSdnc }}
-apiVersion: extensions/v1beta1
-kind: Deployment
+apiVersion: apps/v1beta1
+kind: StatefulSet
 metadata:
   name: sdnc
   namespace: "{{ .Values.nsPrefix }}-sdnc"
 spec:
+  serviceName: "sdnhost-cluster"
+  replicas: {{ .Values.numberOfODLReplicas }}
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app: sdnc
@@ -31,12 +34,21 @@ spec:
         name: sdnc-readiness
       containers:
       - command:
-        - /opt/onap/sdnc/bin/startODL.sh
+        - bash
+        - "-c"
+        - |
+          sed -i 's/dbhost/sdnc-dbhost-0.dbhost.onap-sdnc.svc.cluster.local/g' /opt/onap/sdnc/data/properties/svclogic.properties
+          sed -i 's/^\(org.onap.ccsdk.sli.jdbc.hosts=\).*/\1sdnc-dbhost-0.dbhost.onap-sdnc.svc.cluster.local/' /opt/onap/sdnc/data/properties/dblib.properties
+          /opt/onap/sdnc/bin/startODL.sh
         env:
         - name: MYSQL_ROOT_PASSWORD
           value: openECOMP1.0
         - name: SDNC_CONFIG_DIR
           value: /opt/onap/sdnc/data/properties
+        - name: ENABLE_ODL_CLUSTER
+          value: "{{ .Values.enableODLCluster }}"
+        - name: SDNC_REPLICAS
+          value: "{{ .Values.numberOfODLReplicas }}"
         image: {{ .Values.image.sdnc }}
         imagePullPolicy: {{ .Values.pullPolicy }}
         name: sdnc-controller-container
@@ -55,6 +67,8 @@ spec:
         ports:
         - containerPort: 8181
         - containerPort: 8101
+        - containerPort: 2550
+        - containerPort: 8080
         readinessProbe:
           tcpSocket:
             port: 8181
index 407d047..d04ff84 100644 (file)
@@ -3,13 +3,19 @@ pullPolicy: Always
 nodePortPrefix: 302
 image:
   readiness: oomk8s/readiness-check:1.0.0
-  mysqlServer: mysql/mysql-server:5.6
+  mysql: mysql:5.7
+  xtrabackup: gcr.io/google-samples/xtrabackup:1.0
   dgbuilderSdnc: nexus3.onap.org:10001/onap/ccsdk-dgbuilder-image:v0.1.0
   sdnc: nexus3.onap.org:10001/onap/sdnc-image:v1.2.1
   admportalSdnc: nexus3.onap.org:10001/onap/admportal-sdnc-image:v1.2.1
   filebeat: docker.elastic.co/beats/filebeat:5.5.0
+enableODLCluster: false
+numberOfODLReplicas: 1
+numberOfDbReplicas: 1
+disableSdncSdncDgbuilder: false
+disableSdncSdncPortal: false
 
 # FIXME
 # Missing the following deps
 # nexus3.onap.org:10001/onap/sdnc-dmaap-listener-image:v1.2.1
-# nexus3.onap.org:10001/onap/sdnc-ueb-listener-image:v1.2.1
\ No newline at end of file
+# nexus3.onap.org:10001/onap/sdnc-ueb-listener-image:v1.2.1