Add support for downloading content

Add support for downloading content for creating
the merged helm charts. This api will be used mainly
by the profile api to create a converged chart which
will then be created by the instantiation code
P2: Add unit tests for archive.go
    Add download method for profile.go
    Update comments for download method
P3: Add unit test for empty files
P4: Add unit tests for Download functions
P5: Rebase against new folder structure

Issue-ID: MULTICLOUD-291
Change-Id: I9779eaf95366f527f0360eaddea663722c13b196
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>

diff --git a/src/k8splugin/internal/rb/archive.go b/src/k8splugin/internal/rb/archive.go
index 8eb0fbe..624adfb 100644 (file)
--- a/src/k8splugin/internal/rb/archive.go
+++ b/src/k8splugin/internal/rb/archive.go
@@ -21,13 +21,16 @@ import (
        "compress/gzip"
        pkgerrors "github.com/pkg/errors"
        "io"
+       "io/ioutil"
+       "os"
+       "path/filepath"
 )
 
 func isTarGz(r io.Reader) error {
        //Check if it is a valid gz
        gzf, err := gzip.NewReader(r)
        if err != nil {
-               return pkgerrors.Errorf("Invalid gz format %s", err.Error())
+               return pkgerrors.Wrap(err, "Invalid gzip format")
        }
 
        //Check if it is a valid tar file
@@ -63,3 +66,70 @@ func isTarGz(r io.Reader) error {
 
        return nil
 }
+
+//ExtractTarBall provides functionality to extract a tar.gz file
+//into a temporary location for later use.
+//It returns the path to the new location
+func ExtractTarBall(r io.Reader) (string, error) {
+       //Check if it is a valid gz
+       gzf, err := gzip.NewReader(r)
+       if err != nil {
+               return "", pkgerrors.Wrap(err, "Invalid gzip format")
+       }
+
+       //Check if it is a valid tar file
+       //Unfortunately this can only be done by inspecting all the tar contents
+       tarR := tar.NewReader(gzf)
+       first := true
+
+       outDir, _ := ioutil.TempDir("", "k8s-ext-")
+
+       for true {
+               header, err := tarR.Next()
+
+               if err == io.EOF {
+                       //Check if we have just a gzip file without a tar archive inside
+                       if first {
+                               return "", pkgerrors.New("Empty or non-existant Tar file found")
+                       }
+                       //End of archive
+                       break
+               }
+
+               if err != nil {
+                       return "", pkgerrors.Wrap(err, "Error reading tar file")
+               }
+
+               target := filepath.Join(outDir, header.Name)
+
+               switch header.Typeflag {
+               case tar.TypeDir:
+                       if _, err := os.Stat(target); err != nil {
+                               // Using 755 read, write, execute for owner
+                               // groups and others get read and execute permissions
+                               // on the folder.
+                               if err := os.MkdirAll(target, 0755); err != nil {
+                                       return "", pkgerrors.Wrap(err, "Creating directory")
+                               }
+                       }
+               case tar.TypeReg:
+                       f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+                       if err != nil {
+                               return "", pkgerrors.Wrap(err, "Creating file")
+                       }
+
+                       // copy over contents
+                       if _, err := io.Copy(f, tarR); err != nil {
+                               return "", pkgerrors.Wrap(err, "Copying file content")
+                       }
+
+                       // close for each file instead of a defer for all
+                       // at the end of the function
+                       f.Close()
+               }
+
+               first = false
+       }
+
+       return outDir, nil
+}

diff --git a/src/k8splugin/internal/rb/archive_test.go b/src/k8splugin/internal/rb/archive_test.go
index a327dfd..5fa66e7 100644 (file)
--- a/src/k8splugin/internal/rb/archive_test.go
+++ b/src/k8splugin/internal/rb/archive_test.go
@@ -18,6 +18,9 @@ package rb
 
 import (
        "bytes"
+       "io/ioutil"
+       "path/filepath"
+       "strings"
        "testing"
 )
 
@@ -48,7 +51,7 @@ func TestIsTarGz(t *testing.T) {
 
                err := isTarGz(bytes.NewBuffer(content))
                if err != nil {
-                       t.Errorf("Error reading valid Zip file %s", err.Error())
+                       t.Errorf("Error reading valid tar.gz file %s", err.Error())
                }
        })
 
@@ -63,4 +66,71 @@ func TestIsTarGz(t *testing.T) {
                        t.Errorf("Error should NOT be nil")
                }
        })
+
+       t.Run("Empty tar.gz", func(t *testing.T) {
+               content := []byte{}
+               err := isTarGz(bytes.NewBuffer(content))
+               if err == nil {
+                       t.Errorf("Error should NOT be nil")
+               }
+       })
+}
+
+func TestExtractTarBall(t *testing.T) {
+
+       t.Run("Valid tar.gz", func(t *testing.T) {
+               content := []byte{
+                       0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                       0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                       0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                       0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                       0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                       0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                       0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                       0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                       0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                       0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                       0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                       0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                       0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                       0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                       0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                       0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                       0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                       0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+               }
+
+               path, err := ExtractTarBall(bytes.NewBuffer(content))
+               if err != nil {
+                       t.Errorf("Error reading valid tar.gz file %s", err.Error())
+               }
+               fcontent, err := ioutil.ReadFile(filepath.Join(path, "test.txt"))
+               if err != nil {
+                       t.Errorf("Error reading content of valid tar.gz file %s", err.Error())
+               }
+               if strings.Contains(string(fcontent), "helloworld") == false {
+                       t.Errorf("Incorrect content read from path: %s", path)
+               }
+       })
+
+       t.Run("Invalid tar.gz", func(t *testing.T) {
+               content := []byte{
+                       0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+                       0x00, 0xff, 0xf2, 0x48, 0xcd,
+               }
+
+               _, err := ExtractTarBall(bytes.NewBuffer(content))
+               if err == nil {
+                       t.Errorf("Error should NOT be nil")
+               }
+       })
+
+       t.Run("Empty tar.gz", func(t *testing.T) {
+               content := []byte{}
+               err := isTarGz(bytes.NewBuffer(content))
+               if err == nil {
+                       t.Errorf("Error should NOT be nil")
+               }
+       })
 }

diff --git a/src/k8splugin/internal/rb/definition.go b/src/k8splugin/internal/rb/definition.go
index 8a26332..1984499 100644 (file)
--- a/src/k8splugin/internal/rb/definition.go
+++ b/src/k8splugin/internal/rb/definition.go
@@ -162,3 +162,34 @@ func (v *DefinitionClient) Upload(id string, inp []byte) error {
 
        return nil
 }
+
+// Download the contents of the resource bundle definition from DB
+// Returns a byte array of the contents which is used by the
+// ExtractTarBall code to create the folder structure on disk
+func (v *DefinitionClient) Download(id string) ([]byte, error) {
+
+       //ignore the returned data here
+       //Check if id is valid
+       _, err := v.Get(id)
+       if err != nil {
+               return nil, pkgerrors.Errorf("Invalid Definition ID provided: %s", err.Error())
+       }
+
+       value, err := db.DBconn.Read(v.storeName, id, v.tagContent)
+       if err != nil {
+               return nil, pkgerrors.Wrap(err, "Get Resource Bundle definition content")
+       }
+
+       if value != nil {
+               //Decode the string from base64
+               out, err := base64.StdEncoding.DecodeString(string(value))
+               if err != nil {
+                       return nil, pkgerrors.Wrap(err, "Decode base64 string")
+               }
+
+               if out != nil && len(out) != 0 {
+                       return out, nil
+               }
+       }
+       return nil, pkgerrors.New("Error downloading Definition content")
+}

diff --git a/src/k8splugin/internal/rb/definition_test.go b/src/k8splugin/internal/rb/definition_test.go
index 46ab3c0..f720b6a 100644 (file)
--- a/src/k8splugin/internal/rb/definition_test.go
+++ b/src/k8splugin/internal/rb/definition_test.go
@@ -19,6 +19,7 @@
 package rb
 
 import (
+       "bytes"
        "k8splugin/internal/db"
        "reflect"
        "sort"
@@ -418,3 +419,99 @@ func TestUploadDefinition(t *testing.T) {
                })
        }
 }
+
+func TestDownloadDefinition(t *testing.T) {
+       testCases := []struct {
+               label         string
+               inp           string
+               expected      []byte
+               expectedError string
+               mockdb        *db.MockDB
+       }{
+               {
+                       label: "Download Resource Bundle Definition",
+                       inp:   "123e4567-e89b-12d3-a456-426655440000",
+                       expected: []byte{
+                               0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                               0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                               0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                               0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                               0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                               0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                               0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                               0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                               0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                               0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                               0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                               0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                               0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                               0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                               0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                               0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                               0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+                       },
+                       mockdb: &db.MockDB{
+                               Items: map[string]map[string][]byte{
+                                       "123e4567-e89b-12d3-a456-426655440000": {
+                                               "metadata": []byte(
+                                                       "{\"name\":\"testresourcebundle\"," +
+                                                               "\"description\":\"testresourcebundle\"," +
+                                                               "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+                                                               "\"service-type\":\"firewall\"}"),
+                                               "content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" +
+                                                       "QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" +
+                                                       "b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" +
+                                                       "R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Download with an Invalid Resource Bundle Definition",
+                       inp:           "123e4567-e89b-12d3-a456-426655440000",
+                       expectedError: "Invalid Definition ID provided",
+                       mockdb: &db.MockDB{
+                               Items: map[string]map[string][]byte{
+                                       "123e4567-e89b-12d3-a456-426655441111": {
+                                               "metadata": []byte(
+                                                       "{\"name\":\"testresourcebundle\"," +
+                                                               "\"description\":\"testresourcebundle\"," +
+                                                               "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
+                                                               "\"service-type\":\"firewall\"}"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Download Error",
+                       expectedError: "DB Error",
+                       inp:           "123e4567-e89b-12d3-a456-426655440000",
+                       mockdb: &db.MockDB{
+                               Err: pkgerrors.New("DB Error"),
+                       },
+               },
+       }
+
+       for _, testCase := range testCases {
+               t.Run(testCase.label, func(t *testing.T) {
+                       db.DBconn = testCase.mockdb
+                       impl := NewDefinitionClient()
+                       data, err := impl.Download(testCase.inp)
+                       if err != nil {
+                               if testCase.expectedError == "" {
+                                       t.Errorf("Download returned an unexpected error %s", err)
+                               }
+                               if strings.Contains(err.Error(), testCase.expectedError) == false {
+                                       t.Errorf("Download returned an unexpected error %s", err)
+                               }
+                       } else {
+                               if bytes.Equal(testCase.expected, data) == false {
+                                       t.Errorf("Download returned unexpected data: got %v - expected %v",
+                                               data, testCase.expected)
+                               }
+                       }
+               })
+       }
+}

diff --git a/src/k8splugin/internal/rb/profile.go b/src/k8splugin/internal/rb/profile.go
index a0245af..2456ad2 100644 (file)
--- a/src/k8splugin/internal/rb/profile.go
+++ b/src/k8splugin/internal/rb/profile.go
@@ -183,3 +183,34 @@ func (v *ProfileClient) Upload(id string, inp []byte) error {
 
        return nil
 }
+
+// Download the contents of the resource bundle profile from DB
+// Returns a byte array of the contents which is used by the
+// ExtractTarBall code to create the folder structure on disk
+func (v *ProfileClient) Download(id string) ([]byte, error) {
+
+       //ignore the returned data here
+       //Check if id is valid
+       _, err := v.Get(id)
+       if err != nil {
+               return nil, pkgerrors.Errorf("Invalid Profile ID provided: %s", err.Error())
+       }
+
+       value, err := db.DBconn.Read(v.storeName, id, v.tagContent)
+       if err != nil {
+               return nil, pkgerrors.Wrap(err, "Get Resource Bundle Profile content")
+       }
+
+       if value != nil {
+               //Decode the string from base64
+               out, err := base64.StdEncoding.DecodeString(string(value))
+               if err != nil {
+                       return nil, pkgerrors.Wrap(err, "Decode base64 string")
+               }
+
+               if out != nil && len(out) != 0 {
+                       return out, nil
+               }
+       }
+       return nil, pkgerrors.New("Error downloading Profile content")
+}

diff --git a/src/k8splugin/internal/rb/profile_test.go b/src/k8splugin/internal/rb/profile_test.go
index 15ff895..d97969d 100644 (file)
--- a/src/k8splugin/internal/rb/profile_test.go
+++ b/src/k8splugin/internal/rb/profile_test.go
@@ -19,6 +19,7 @@
 package rb
 
 import (
+       "bytes"
        "k8splugin/internal/db"
        "reflect"
        "sort"
@@ -424,3 +425,101 @@ func TestUploadProfile(t *testing.T) {
                })
        }
 }
+
+func TestDownloadProfile(t *testing.T) {
+       testCases := []struct {
+               label         string
+               inp           string
+               expected      []byte
+               expectedError string
+               mockdb        *db.MockDB
+       }{
+               {
+                       label: "Download Resource Bundle Profile",
+                       inp:   "123e4567-e89b-12d3-a456-426655440000",
+                       expected: []byte{
+                               0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                               0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                               0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                               0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                               0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                               0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                               0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                               0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                               0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                               0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                               0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                               0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                               0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                               0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                               0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                               0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                               0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+                       },
+                       mockdb: &db.MockDB{
+                               Items: map[string]map[string][]byte{
+                                       "123e4567-e89b-12d3-a456-426655440000": {
+                                               "metadata": []byte(
+                                                       "{\"name\":\"testresourcebundle\"," +
+                                                               "\"namespace\":\"default\"," +
+                                                               "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+                                                               "\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," +
+                                                               "\"kubernetesversion\":\"1.12.3\"}"),
+                                               "content": []byte("H4sICLBr9FsAA3Rlc3QudGFyAO3OQQrCMBCF4aw9RU5" +
+                                                       "QEtLE40igAUtSC+2IHt9IEVwIpYtShP/bvGFmFk/SLI08Re3IVCG077Rn" +
+                                                       "b75zYZ2yztVV8N7XP9vWSWmzZ6mP+yxx0lrF7pJzjkN/Sz//1u5/6ppKG" +
+                                                       "R/jVLrT0VUAAAAAAAAAAAAAAAAAABu8ALXoSvkAKAAA"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Download with an Invalid Resource Bundle Profile",
+                       inp:           "123e4567-e89b-12d3-a456-426655440000",
+                       expectedError: "Invalid Profile ID provided",
+                       mockdb: &db.MockDB{
+                               Items: map[string]map[string][]byte{
+                                       "123e4567-e89b-12d3-a456-426655441111": {
+                                               "metadata": []byte(
+                                                       "{\"name\":\"testresourcebundle\"," +
+                                                               "\"uuid\":\"123e4567-e89b-12d3-a456-426655441111\"," +
+                                                               "\"namespace\":\"default\"," +
+                                                               "\"rbdid\":\"abcde123-e89b-8888-a456-986655447236\"," +
+                                                               "\"kubernetesversion\":\"1.12.3\"}"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Download Error",
+                       expectedError: "DB Error",
+                       inp:           "123e4567-e89b-12d3-a456-426655440000",
+                       mockdb: &db.MockDB{
+                               Err: pkgerrors.New("DB Error"),
+                       },
+               },
+       }
+
+       for _, testCase := range testCases {
+               t.Run(testCase.label, func(t *testing.T) {
+                       db.DBconn = testCase.mockdb
+                       impl := NewProfileClient()
+                       data, err := impl.Download(testCase.inp)
+                       if err != nil {
+                               if testCase.expectedError == "" {
+                                       t.Errorf("Download returned an unexpected error %s", err)
+                               }
+                               if strings.Contains(err.Error(), testCase.expectedError) == false {
+                                       t.Errorf("Download returned an unexpected error %s", err)
+                               }
+                       } else {
+                               if bytes.Equal(testCase.expected, data) == false {
+                                       t.Errorf("Download returned unexpected data: got %v - expected %v",
+                                               data, testCase.expected)
+                               }
+                       }
+               })
+       }
+}