Add support for parsing profile yaml files 40/76240/15
authorKiran Kamineni <kiran.k.kamineni@intel.com>
Wed, 23 Jan 2019 20:54:50 +0000 (12:54 -0800)
committerKiran Kamineni <kiran.k.kamineni@intel.com>
Tue, 12 Mar 2019 05:41:33 +0000 (22:41 -0700)
Add code to parse profile configuration yaml
The parsing function is global and returns a client
which can then be used to get or apply specific parts
of the configuration on top of an extracted helm chart.
P14: Add unit test that covers both ProcessProfileYaml
     and CopyConfigurationOverrides
P15: Adding mock_charts and mock_profiles
     We expect to reuse these files for other unit tests

Issue-ID: ONAPARC-348
Change-Id: I4504d0b158fdfef476b8c2a461d33306926545d7
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>
20 files changed:
src/k8splugin/go.mod
src/k8splugin/internal/rb/profile.go
src/k8splugin/internal/rb/profile_yaml.go [new file with mode: 0644]
src/k8splugin/internal/rb/profile_yaml_test.go [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/Chart.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/Chart.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/templates/service.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/values.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/Chart.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/templates/service.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/values.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/templates/NOTES.txt [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/templates/service.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_charts/testchart1/values.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_profiles/profile1/faulty-dest-manifest.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_profiles/profile1/faulty-manifest.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_profiles/profile1/faulty-src-manifest.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_profiles/profile1/manifest.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_profiles/profile1/override_values.yaml [new file with mode: 0644]
src/k8splugin/mock_files/mock_profiles/profile1/subdir/p1.yaml [new file with mode: 0644]

index c6d2ef2..1e7115d 100644 (file)
@@ -2,7 +2,7 @@ module k8splugin
 
 require (
        github.com/davecgh/go-spew v1.1.1 // indirect
-       github.com/ghodss/yaml v1.0.0 // indirect
+       github.com/ghodss/yaml v1.0.0
        github.com/go-stack/stack v1.8.0 // indirect
        github.com/gogo/protobuf v1.0.0 // indirect
        github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
index 2456ad2..d78e32e 100644 (file)
@@ -51,6 +51,7 @@ type ProfileManager interface {
 type ProfileClient struct {
        storeName           string
        tagMeta, tagContent string
+       manifestName        string
 }
 
 // NewProfileClient returns an instance of the ProfileClient
@@ -58,9 +59,10 @@ type ProfileClient struct {
 // Uses rb/def prefix
 func NewProfileClient() *ProfileClient {
        return &ProfileClient{
-               storeName:  "rbprofile",
-               tagMeta:    "metadata",
-               tagContent: "content",
+               storeName:    "rbprofile",
+               tagMeta:      "metadata",
+               tagContent:   "content",
+               manifestName: "manifest.yaml",
        }
 }
 
diff --git a/src/k8splugin/internal/rb/profile_yaml.go b/src/k8splugin/internal/rb/profile_yaml.go
new file mode 100644 (file)
index 0000000..ba55fba
--- /dev/null
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rb
+
+import (
+       "io/ioutil"
+       "log"
+       "path/filepath"
+
+       "github.com/ghodss/yaml"
+       pkgerrors "github.com/pkg/errors"
+)
+
+/*
+#Sample Yaml format for profile manifest.yaml
+---
+version: v1
+type:
+  values: "values_override.yaml"
+  configresource:
+    - filepath: config.yaml
+      chartpath: chart/config/resources/config.yaml
+    - filepath: config2.yaml
+      chartpath: chart/config/resources/config2.yaml
+*/
+
+type overrideFiles struct {
+       FilePath  string `yaml:"filepath"`
+       ChartPath string `yaml:"chartpath"`
+}
+
+type supportedOverrides struct {
+       ConfigResource []overrideFiles `yaml:"configresource"`
+       Values         string          `yaml:"values"`
+}
+
+type profileOverride struct {
+       Version string             `yaml:"version"`
+       Type    supportedOverrides `yaml:"type"`
+}
+
+type ProfileYamlClient struct {
+       path     string
+       override profileOverride
+}
+
+func (p ProfileYamlClient) Print() {
+       log.Println(p.override)
+}
+
+//GetValues returns a path to the override values.yam
+//that was part of the profile
+func (p ProfileYamlClient) GetValues() string {
+       return filepath.Join(p.path, p.override.Type.Values)
+}
+
+//CopyConfigurationOverrides copies the various files that are
+//provided as overrides to their corresponding locations within
+//the destination chart.
+func (p ProfileYamlClient) CopyConfigurationOverrides(chartPath string) error {
+
+       //Iterate over each configresource and copy that file into
+       //the respective path in the chart.
+       for _, v := range p.override.Type.ConfigResource {
+               data, err := ioutil.ReadFile(filepath.Join(p.path, v.FilePath))
+               if err != nil {
+                       return pkgerrors.Wrap(err, "Reading configuration file")
+               }
+               err = ioutil.WriteFile(filepath.Join(chartPath, v.ChartPath), data, 0644)
+               if err != nil {
+                       return pkgerrors.Wrap(err, "Writing configuration file into chartpath")
+               }
+       }
+
+       return nil
+}
+
+//ProcessProfileYaml parses the manifest.yaml file that is part of the profile
+//package and creates the appropriate structures out of it.
+func ProcessProfileYaml(fpath string, manifestFileName string) (ProfileYamlClient, error) {
+
+       p := filepath.Join(fpath, manifestFileName)
+       data, err := ioutil.ReadFile(p)
+       if err != nil {
+               return ProfileYamlClient{}, pkgerrors.Wrap(err, "Reading manifest file")
+       }
+
+       out := profileOverride{}
+       err = yaml.Unmarshal(data, &out)
+       if err != nil {
+               return ProfileYamlClient{}, pkgerrors.Wrap(err, "Marshaling manifest yaml file")
+       }
+
+       return ProfileYamlClient{path: fpath, override: out}, nil
+}
diff --git a/src/k8splugin/internal/rb/profile_yaml_test.go b/src/k8splugin/internal/rb/profile_yaml_test.go
new file mode 100644 (file)
index 0000000..c29bf9e
--- /dev/null
@@ -0,0 +1,139 @@
+// +build unit
+
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rb
+
+import (
+       "os"
+       "path/filepath"
+       "strings"
+       "testing"
+)
+
+func TestProcessProfileYaml(t *testing.T) {
+
+       profileDir := "../../mock_files/mock_profiles/profile1"
+       manifestFile := "manifest.yaml"
+       faultymanifestfile := "faulty-manifest.yaml"
+
+       testCases := []struct {
+               label           string
+               prDir, manifest string
+               expectedError   string
+       }{
+               {
+                       label:         "Process Profile Yaml",
+                       prDir:         profileDir,
+                       manifest:      manifestFile,
+                       expectedError: "",
+               },
+               {
+                       label:         "Non existent manifest file",
+                       prDir:         profileDir,
+                       manifest:      "non-existant-file.yaml",
+                       expectedError: "Reading manifest file",
+               },
+               {
+                       label:         "Faulty manifest file",
+                       prDir:         profileDir,
+                       manifest:      faultymanifestfile,
+                       expectedError: "Marshaling manifest yaml file",
+               },
+       }
+
+       for _, testCase := range testCases {
+               t.Run(testCase.label, func(t *testing.T) {
+                       _, err := ProcessProfileYaml(testCase.prDir, testCase.manifest)
+                       if err != nil {
+                               if testCase.expectedError == "" {
+                                       t.Fatalf("Got an error %s", err)
+                               }
+                               if strings.Contains(err.Error(), testCase.expectedError) == false {
+                                       t.Fatalf("Got unexpected error message %s", err)
+                               }
+                       }
+               })
+       }
+}
+
+func TestCopyConfigurationOverrides(t *testing.T) {
+
+       profileDir := "../../mock_files/mock_profiles/profile1"
+       profileFileName := "p1.yaml"
+       manifestFile := "manifest.yaml"
+       faultySrcManifestFile := "faulty-src-manifest.yaml"
+       faultyDestManifestFile := "faulty-dest-manifest.yaml"
+       chartBaseDir := "../../mock_files/mock_charts"
+
+       //Remove the testchart1/templates/p1.yaml file that gets copied over
+       defer os.Remove(filepath.Join(chartBaseDir, "testchart1", "templates", profileFileName))
+
+       testCases := []struct {
+               label                  string
+               prDir, chDir, manifest string
+               expectedError          string
+       }{
+               {
+                       label:         "Copy Configuration Overrides",
+                       prDir:         profileDir,
+                       manifest:      manifestFile,
+                       chDir:         chartBaseDir,
+                       expectedError: "",
+               },
+               {
+                       label:         "Copy Configuration Overrides Faulty Source",
+                       prDir:         profileDir,
+                       manifest:      faultySrcManifestFile,
+                       chDir:         chartBaseDir,
+                       expectedError: "Reading configuration file",
+               },
+               {
+                       label:         "Copy Configuration Overrides Faulty Destination",
+                       prDir:         profileDir,
+                       manifest:      faultyDestManifestFile,
+                       chDir:         chartBaseDir,
+                       expectedError: "Writing configuration file",
+               },
+       }
+
+       for _, testCase := range testCases {
+               t.Run(testCase.label, func(t *testing.T) {
+                       p, err := ProcessProfileYaml(testCase.prDir, testCase.manifest)
+                       if err != nil {
+                               t.Fatalf("Got unexpected error processing yaml %s", err)
+                       }
+
+                       err = p.CopyConfigurationOverrides(testCase.chDir)
+                       if err != nil {
+                               if testCase.expectedError == "" {
+                                       t.Fatalf("Got error %s", err)
+                               }
+                               if strings.Contains(err.Error(), testCase.expectedError) == false {
+                                       t.Fatalf("Got unexpected error message %s", err)
+                               }
+
+                       } else {
+                               //Check if the file got copied over
+                               if _, err = os.Stat(filepath.Join(testCase.chDir, "testchart1",
+                                       "templates", profileFileName)); os.IsNotExist(err) {
+                                       t.Fatalf("Failed to copy override file: %s", profileFileName)
+                               }
+                       }
+               })
+       }
+}
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/Chart.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/Chart.yaml
new file mode 100644 (file)
index 0000000..91a641b
--- /dev/null
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A Helm chart for Kubernetes
+name: testchart1
+version: 0.1.0
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/Chart.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/Chart.yaml
new file mode 100644 (file)
index 0000000..be3edce
--- /dev/null
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A Helm chart for Kubernetes
+name: subcharta
+version: 0.1.0
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/templates/service.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/templates/service.yaml
new file mode 100644 (file)
index 0000000..fdf75aa
--- /dev/null
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+  - port: {{ .Values.service.externalPort }}
+    targetPort: {{ .Values.service.internalPort }}
+    protocol: TCP
+    name: {{ .Values.service.name }}
+  selector:
+    app: {{ .Chart.Name }}
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/values.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/charts/subcharta/values.yaml
new file mode 100644 (file)
index 0000000..f0381ae
--- /dev/null
@@ -0,0 +1,17 @@
+# Default values for subchart.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+# subchartA
+service:
+  name: apache
+  type: ClusterIP
+  externalPort: 80
+  internalPort: 80
+SCAdata:
+  SCAbool: false
+  SCAfloat: 3.1
+  SCAint: 55
+  SCAstring: "jabba"
+  SCAnested1:
+    SCAnested2: true
+
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/Chart.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/Chart.yaml
new file mode 100644 (file)
index 0000000..c3c6bba
--- /dev/null
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A Helm chart for Kubernetes
+name: subchartb
+version: 0.1.0
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/templates/service.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/templates/service.yaml
new file mode 100644 (file)
index 0000000..fdf75aa
--- /dev/null
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+  - port: {{ .Values.service.externalPort }}
+    targetPort: {{ .Values.service.internalPort }}
+    protocol: TCP
+    name: {{ .Values.service.name }}
+  selector:
+    app: {{ .Chart.Name }}
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/values.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/charts/subchartb/values.yaml
new file mode 100644 (file)
index 0000000..774fdd7
--- /dev/null
@@ -0,0 +1,35 @@
+# Default values for subchart.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+service:
+  name: nginx
+  type: ClusterIP
+  externalPort: 80
+  internalPort: 80
+
+SCBdata:
+  SCBbool: true
+  SCBfloat: 7.77
+  SCBint: 33
+  SCBstring: "boba"
+
+exports:
+  SCBexported1:
+    SCBexported1A:
+      SCBexported1B: 1965
+
+  SCBexported2:
+    SCBexported2A: "blaster"
+
+global:
+  kolla:
+    nova:
+      api:
+        all:
+          port: 8774
+      metadata:
+        all:
+          port: 8775
+
+
+
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/templates/NOTES.txt b/src/k8splugin/mock_files/mock_charts/testchart1/templates/NOTES.txt
new file mode 100644 (file)
index 0000000..4bdf443
--- /dev/null
@@ -0,0 +1 @@
+Sample notes for {{ .Chart.Name }}
\ No newline at end of file
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/templates/service.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/templates/service.yaml
new file mode 100644 (file)
index 0000000..e06d19b
--- /dev/null
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    namespace: "{{ .Release.Namespace }}"
+    release-name: "{{ .Release.Name }}"
+    release-is-upgrade: "{{ .Release.IsUpgrade }}"
+    release-is-install: "{{ .Release.IsInstall }}"
+    kube-version/major: "{{ .Capabilities.KubeVersion.Major }}"
+    kube-version/minor: "{{ .Capabilities.KubeVersion.Minor }}"
+    kube-version/gitversion: "v{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}.0"
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+  - port: {{ .Values.service.externalPort }}
+    targetPort: {{ .Values.service.internalPort }}
+    protocol: TCP
+    name: {{ .Values.service.name }}
+  selector:
+    app: {{ .Chart.Name }}
diff --git a/src/k8splugin/mock_files/mock_charts/testchart1/values.yaml b/src/k8splugin/mock_files/mock_charts/testchart1/values.yaml
new file mode 100644 (file)
index 0000000..72d3fa5
--- /dev/null
@@ -0,0 +1,55 @@
+# Default values for subchart.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+# subchart1
+service:
+  name: nginx
+  type: ClusterIP
+  externalPort: 80
+  internalPort: 80
+
+
+SC1data:
+  SC1bool: true
+  SC1float: 3.14
+  SC1int: 100
+  SC1string: "dollywood"
+  SC1extra1: 11
+
+imported-chartA:
+  SC1extra2: 1.337
+
+overridden-chartA:
+  SCAbool: true
+  SCAfloat: 3.14
+  SCAint: 100
+  SCAstring: "jabathehut"
+  SC1extra3: true
+
+imported-chartA-B:
+  SC1extra5: "tiller"
+
+overridden-chartA-B:
+  SCAbool: true
+  SCAfloat: 3.33
+  SCAint: 555
+  SCAstring: "wormwood"
+  SCAextra1: 23
+
+  SCBbool: true
+  SCBfloat: 0.25
+  SCBint: 98
+  SCBstring: "murkwood"
+  SCBextra1: 13
+
+  SC1extra6: 77
+
+SCBexported1A:
+  SC1extra7: true
+
+exports:
+  SC1exported1:
+    global:
+      SC1exported2:
+        all:
+          SC1exported3: "SC1expstr"
\ No newline at end of file
diff --git a/src/k8splugin/mock_files/mock_profiles/profile1/faulty-dest-manifest.yaml b/src/k8splugin/mock_files/mock_profiles/profile1/faulty-dest-manifest.yaml
new file mode 100644 (file)
index 0000000..8c61a4e
--- /dev/null
@@ -0,0 +1,7 @@
+---
+version: v1
+type:
+  values: "override_values.yaml"
+  configresource:
+    - filepath: subdir/p1.yaml
+      chartpath: testchart1/folderdoesNOTexist/p1.yaml
diff --git a/src/k8splugin/mock_files/mock_profiles/profile1/faulty-manifest.yaml b/src/k8splugin/mock_files/mock_profiles/profile1/faulty-manifest.yaml
new file mode 100644 (file)
index 0000000..a123111
--- /dev/null
@@ -0,0 +1,8 @@
+---
+version: v1
+type:
+  values: 
+    - override_values.yaml
+  configresource:
+  - filepath: subdir/p1.yaml
+    chartpath: testchart1/templates/p1.yaml
diff --git a/src/k8splugin/mock_files/mock_profiles/profile1/faulty-src-manifest.yaml b/src/k8splugin/mock_files/mock_profiles/profile1/faulty-src-manifest.yaml
new file mode 100644 (file)
index 0000000..eff534b
--- /dev/null
@@ -0,0 +1,7 @@
+---
+version: v1
+type:
+  values: "override_values.yaml"
+  configresource:
+    - filepath: subdir/filedoesNOTexist.yaml
+      chartpath: testchart1/templates/p1.yaml
diff --git a/src/k8splugin/mock_files/mock_profiles/profile1/manifest.yaml b/src/k8splugin/mock_files/mock_profiles/profile1/manifest.yaml
new file mode 100644 (file)
index 0000000..e4beada
--- /dev/null
@@ -0,0 +1,7 @@
+---
+version: v1
+type:
+  values: "override_values.yaml"
+  configresource:
+    - filepath: subdir/p1.yaml
+      chartpath: testchart1/templates/p1.yaml
diff --git a/src/k8splugin/mock_files/mock_profiles/profile1/override_values.yaml b/src/k8splugin/mock_files/mock_profiles/profile1/override_values.yaml
new file mode 100644 (file)
index 0000000..0186c66
--- /dev/null
@@ -0,0 +1,9 @@
+# Default values for subchart.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+# subchart1
+service:
+  name: nginx
+  type: ClusterIP
+  externalPort: 8080
+  internalPort: 8080
\ No newline at end of file
diff --git a/src/k8splugin/mock_files/mock_profiles/profile1/subdir/p1.yaml b/src/k8splugin/mock_files/mock_profiles/profile1/subdir/p1.yaml
new file mode 100644 (file)
index 0000000..2dad677
--- /dev/null
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-override
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    namespace: "{{ .Release.Namespace }}"
+    release-name: "{{ .Release.Name }}"
+    release-is-upgrade: "{{ .Release.IsUpgrade }}"
+    release-is-install: "{{ .Release.IsInstall }}"
+    kube-version/major: "{{ .Capabilities.KubeVersion.Major }}"
+    kube-version/minor: "{{ .Capabilities.KubeVersion.Minor }}"
+    kube-version/gitversion: "v{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}.0"
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+  - port: {{ .Values.service.externalPort }}
+    targetPort: {{ .Values.service.internalPort }}
+    protocol: TCP
+    name: {{ .Values.service.name }}
+  selector:
+    app: {{ .Chart.Name }}