Add upload backend implementation

Upload is a seperate API where it takes
a binary stream and stores it. The api supports
tar.gz file format only.
P2: Check if ID is valid before trying upload
    Add test with an invalid ID

Issue-ID: MULTICLOUD-393
Change-Id: Id636a95823a046e1795d3be72d0214e953a8c5fc
Signed-off-by: Kiran Kamineni <kiran.k.kamineni@intel.com>

diff --git a/src/k8splugin/api/api.go b/src/k8splugin/api/api.go
index 571a957..530537d 100644 (file)
--- a/src/k8splugin/api/api.go
+++ b/src/k8splugin/api/api.go
@@ -109,7 +109,7 @@ func NewRouter(kubeconfig string) *mux.Router {
        resRouter := router.PathPrefix("/v1/rb").Subrouter()
        rbdef := rbDefinitionHandler{client: rb.NewDefinitionClient()}
        resRouter.HandleFunc("/definition", rbdef.createHandler).Methods("POST")
-       resRouter.HandleFunc("/definition/{rbdID}/upload", rbdef.uploadHandler).Methods("POST")
+       resRouter.HandleFunc("/definition/{rbdID}/content", rbdef.uploadHandler).Methods("POST")
        resRouter.HandleFunc("/definition", rbdef.listHandler).Methods("GET")
        resRouter.HandleFunc("/definition/{rbdID}", rbdef.getHandler).Methods("GET")
        resRouter.HandleFunc("/definition/{rbdID}", rbdef.deleteHandler).Methods("DELETE")

diff --git a/src/k8splugin/api/defhandler.go b/src/k8splugin/api/defhandler.go
index c8c0349..222baae 100644 (file)
--- a/src/k8splugin/api/defhandler.go
+++ b/src/k8splugin/api/defhandler.go
@@ -18,9 +18,9 @@ package api
 
 import (
        "encoding/json"
-       "net/http"
-
+       "io/ioutil"
        "k8splugin/rb"
+       "net/http"
 
        "github.com/gorilla/mux"
 )
@@ -72,6 +72,27 @@ func (h rbDefinitionHandler) createHandler(w http.ResponseWriter, r *http.Reques
 // uploadHandler handles upload of the bundle tar file into the database
 // Note: This will be implemented in a different patch
 func (h rbDefinitionHandler) uploadHandler(w http.ResponseWriter, r *http.Request) {
+       vars := mux.Vars(r)
+       uuid := vars["rbdID"]
+
+       if r.Body == nil {
+               http.Error(w, "Empty Body", http.StatusBadRequest)
+               return
+       }
+
+       inpBytes, err := ioutil.ReadAll(r.Body)
+       if err != nil {
+               http.Error(w, "Unable to read body", http.StatusBadRequest)
+               return
+       }
+
+       err = h.client.Upload(uuid, inpBytes)
+       if err != nil {
+               http.Error(w, err.Error(), http.StatusInternalServerError)
+               return
+       }
+
+       w.WriteHeader(http.StatusOK)
 }
 
 // listHandler handles GET (list) operations on the endpoint

diff --git a/src/k8splugin/api/defhandler_test.go b/src/k8splugin/api/defhandler_test.go
index b83f0b7..9739ab1 100644 (file)
--- a/src/k8splugin/api/defhandler_test.go
+++ b/src/k8splugin/api/defhandler_test.go
@@ -68,6 +68,10 @@ func (m *mockRBDefinition) Delete(id string) error {
        return m.Err
 }
 
+func (m *mockRBDefinition) Upload(id string, inp []byte) error {
+       return m.Err
+}
+
 func TestRBDefCreateHandler(t *testing.T) {
        testCases := []struct {
                label        string
@@ -334,3 +338,64 @@ func TestRBDefDeleteHandler(t *testing.T) {
                })
        }
 }
+
+func TestRBDefUploadHandler(t *testing.T) {
+
+       testCases := []struct {
+               label        string
+               inpUUID      string
+               body         io.Reader
+               expectedCode int
+               rbDefClient  *mockRBDefinition
+       }{
+               {
+                       label:        "Upload Bundle Definition Content",
+                       expectedCode: http.StatusOK,
+                       inpUUID:      "123e4567-e89b-12d3-a456-426655441111",
+                       body: bytes.NewBuffer([]byte{
+                               0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0xff, 0xf2, 0x48, 0xcd,
+                       }),
+                       rbDefClient: &mockRBDefinition{},
+               },
+               {
+                       label:        "Upload Invalid Bundle Definition Content",
+                       expectedCode: http.StatusInternalServerError,
+                       inpUUID:      "123e4567-e89b-12d3-a456-426655440000",
+                       body: bytes.NewBuffer([]byte{
+                               0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0xff, 0xf2, 0x48, 0xcd,
+                       }),
+                       rbDefClient: &mockRBDefinition{
+                               Err: pkgerrors.New("Internal Error"),
+                       },
+               },
+               {
+                       label:        "Upload Empty Body Content",
+                       expectedCode: http.StatusBadRequest,
+                       inpUUID:      "123e4567-e89b-12d3-a456-426655440000",
+                       rbDefClient:  &mockRBDefinition{},
+               },
+       }
+
+       for _, testCase := range testCases {
+               t.Run(testCase.label, func(t *testing.T) {
+                       vh := rbDefinitionHandler{client: testCase.rbDefClient}
+                       req, err := http.NewRequest("POST",
+                               "/v1/resource/definition/"+testCase.inpUUID+"/content", testCase.body)
+
+                       if err != nil {
+                               t.Fatal(err)
+                       }
+
+                       rr := httptest.NewRecorder()
+                       hr := http.HandlerFunc(vh.uploadHandler)
+
+                       hr.ServeHTTP(rr, req)
+                       //Check returned code
+                       if rr.Code != testCase.expectedCode {
+                               t.Fatalf("Expected %d; Got: %d", testCase.expectedCode, rr.Code)
+                       }
+               })
+       }
+}

diff --git a/src/k8splugin/rb/archive.go b/src/k8splugin/rb/archive.go
new file mode 100644 (file)
index 0000000..8eb0fbe
--- /dev/null
+++ b/src/k8splugin/rb/archive.go
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rb
+
+import (
+       "archive/tar"
+       "compress/gzip"
+       pkgerrors "github.com/pkg/errors"
+       "io"
+)
+
+func isTarGz(r io.Reader) error {
+       //Check if it is a valid gz
+       gzf, err := gzip.NewReader(r)
+       if err != nil {
+               return pkgerrors.Errorf("Invalid gz format %s", err.Error())
+       }
+
+       //Check if it is a valid tar file
+       //Unfortunately this can only be done by inspecting all the tar contents
+       tarR := tar.NewReader(gzf)
+       first := true
+
+       for true {
+               header, err := tarR.Next()
+
+               if err == io.EOF {
+                       //Check if we have just a gzip file without a tar archive inside
+                       if first {
+                               return pkgerrors.New("Empty or non-existant Tar file found")
+                       }
+                       //End of archive
+                       break
+               }
+
+               if err != nil {
+                       return pkgerrors.Errorf("Error reading tar file %s", err.Error())
+               }
+
+               //Check if files are of type directory and regular file
+               if header.Typeflag != tar.TypeDir &&
+                       header.Typeflag != tar.TypeReg {
+                       return pkgerrors.Errorf("Unknown header in tar %s, %s",
+                               header.Name, string(header.Typeflag))
+               }
+
+               first = false
+       }
+
+       return nil
+}

diff --git a/src/k8splugin/rb/archive_test.go b/src/k8splugin/rb/archive_test.go
new file mode 100644 (file)
index 0000000..a327dfd
--- /dev/null
+++ b/src/k8splugin/rb/archive_test.go
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rb
+
+import (
+       "bytes"
+       "testing"
+)
+
+func TestIsTarGz(t *testing.T) {
+
+       t.Run("Valid tar.gz", func(t *testing.T) {
+               content := []byte{
+                       0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                       0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                       0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                       0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                       0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                       0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                       0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                       0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                       0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                       0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                       0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                       0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                       0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                       0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                       0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                       0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                       0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                       0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+               }
+
+               err := isTarGz(bytes.NewBuffer(content))
+               if err != nil {
+                       t.Errorf("Error reading valid Zip file %s", err.Error())
+               }
+       })
+
+       t.Run("Invalid tar.gz", func(t *testing.T) {
+               content := []byte{
+                       0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+                       0x00, 0xff, 0xf2, 0x48, 0xcd,
+               }
+
+               err := isTarGz(bytes.NewBuffer(content))
+               if err == nil {
+                       t.Errorf("Error should NOT be nil")
+               }
+       })
+}

diff --git a/src/k8splugin/rb/definition.go b/src/k8splugin/rb/definition.go
index 03fffdd..2d1c2cd 100644 (file)
--- a/src/k8splugin/rb/definition.go
+++ b/src/k8splugin/rb/definition.go
@@ -17,6 +17,8 @@
 package rb
 
 import (
+       "bytes"
+       "encoding/base64"
        "k8splugin/db"
        "log"
 
@@ -39,6 +41,7 @@ type DefinitionManager interface {
        List() ([]Definition, error)
        Get(resID string) (Definition, error)
        Delete(resID string) error
+       Upload(resID string, inp []byte) error
 }
 
 // DefinitionClient implements the DefinitionManager
@@ -63,7 +66,7 @@ func (v *DefinitionClient) Create(def Definition) (Definition, error) {
        }
        key := v.keyPrefix + def.UUID
 
-       serData, err := db.Serialize(v)
+       serData, err := db.Serialize(def)
        if err != nil {
                return Definition{}, pkgerrors.Wrap(err, "Serialize Resource Bundle Definition")
        }
@@ -133,3 +136,27 @@ func (v *DefinitionClient) Delete(id string) error {
 
        return nil
 }
+
+// Upload the contents of resource bundle into database
+func (v *DefinitionClient) Upload(id string, inp []byte) error {
+
+       //ignore the returned data here.
+       _, err := v.Get(id)
+       if err != nil {
+               return pkgerrors.Errorf("Invalid ID provided %s", err.Error())
+       }
+
+       err = isTarGz(bytes.NewBuffer(inp))
+       if err != nil {
+               return pkgerrors.Errorf("Error in file format %s", err.Error())
+       }
+
+       encodedStr := base64.StdEncoding.EncodeToString(inp)
+       key := v.keyPrefix + id + "/content"
+       err = db.DBconn.Create(key, encodedStr)
+       if err != nil {
+               return pkgerrors.Errorf("Error uploading data to db %s", err.Error())
+       }
+
+       return nil
+}

diff --git a/src/k8splugin/rb/definition_test.go b/src/k8splugin/rb/definition_test.go
index a3993c8..58eb718 100644 (file)
--- a/src/k8splugin/rb/definition_test.go
+++ b/src/k8splugin/rb/definition_test.go
@@ -228,7 +228,6 @@ func TestDelete(t *testing.T) {
                inp           string
                expectedError string
                mockdb        *db.MockDB
-               expected      []Definition
        }{
                {
                        label:  "Delete Resource Bundle Definition",
@@ -260,3 +259,151 @@ func TestDelete(t *testing.T) {
                })
        }
 }
+
+func TestUpload(t *testing.T) {
+       testCases := []struct {
+               label         string
+               inp           string
+               content       []byte
+               expectedError string
+               mockdb        *db.MockDB
+       }{
+               {
+                       label: "Upload Resource Bundle Definition",
+                       inp:   "123e4567-e89b-12d3-a456-426655440000",
+                       content: []byte{
+                               0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                               0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                               0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                               0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                               0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                               0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                               0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                               0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                               0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                               0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                               0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                               0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                               0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                               0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                               0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                               0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                               0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+                       },
+                       mockdb: &db.MockDB{
+                               Items: api.KVPairs{
+                                       &api.KVPair{
+                                               Key: "rb/def/123e4567-e89b-12d3-a456-426655440000",
+                                               Value: []byte("{\"name\":\"testresourcebundle\"," +
+                                                       "\"description\":\"testresourcebundle\"," +
+                                                       "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+                                                       "\"service-type\":\"firewall\"}"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Upload with an Invalid Resource Bundle Definition",
+                       inp:           "123e4567-e89b-12d3-a456-426655440000",
+                       expectedError: "Invalid ID provided",
+                       content: []byte{
+                               0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                               0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                               0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                               0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                               0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                               0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                               0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                               0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                               0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                               0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                               0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                               0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                               0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                               0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                               0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                               0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                               0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+                       },
+                       mockdb: &db.MockDB{
+                               Items: api.KVPairs{
+                                       &api.KVPair{
+                                               Key: "rb/def/123e4567-e89b-12d3-a456-426655441111",
+                                               Value: []byte("{\"name\":\"testresourcebundle\"," +
+                                                       "\"description\":\"testresourcebundle\"," +
+                                                       "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+                                                       "\"service-type\":\"firewall\"}"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Invalid File Format Error",
+                       inp:           "123e4567-e89b-12d3-a456-426655440000",
+                       expectedError: "Error in file format",
+                       content: []byte{
+                               0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0xff, 0xf2, 0x48, 0xcd,
+                       },
+                       mockdb: &db.MockDB{
+                               Items: api.KVPairs{
+                                       &api.KVPair{
+                                               Key: "rb/def/123e4567-e89b-12d3-a456-426655440000",
+                                               Value: []byte("{\"name\":\"testresourcebundle\"," +
+                                                       "\"description\":\"testresourcebundle\"," +
+                                                       "\"uuid\":\"123e4567-e89b-12d3-a456-426655440000\"," +
+                                                       "\"service-type\":\"firewall\"}"),
+                                       },
+                               },
+                       },
+               },
+               {
+                       label:         "Upload Error",
+                       expectedError: "DB Error",
+                       content: []byte{
+                               0x1f, 0x8b, 0x08, 0x08, 0xb0, 0x6b, 0xf4, 0x5b,
+                               0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74,
+                               0x61, 0x72, 0x00, 0xed, 0xce, 0x41, 0x0a, 0xc2,
+                               0x30, 0x10, 0x85, 0xe1, 0xac, 0x3d, 0x45, 0x4e,
+                               0x50, 0x12, 0xd2, 0xc4, 0xe3, 0x48, 0xa0, 0x01,
+                               0x4b, 0x52, 0x0b, 0xed, 0x88, 0x1e, 0xdf, 0x48,
+                               0x11, 0x5c, 0x08, 0xa5, 0x8b, 0x52, 0x84, 0xff,
+                               0xdb, 0xbc, 0x61, 0x66, 0x16, 0x4f, 0xd2, 0x2c,
+                               0x8d, 0x3c, 0x45, 0xed, 0xc8, 0x54, 0x21, 0xb4,
+                               0xef, 0xb4, 0x67, 0x6f, 0xbe, 0x73, 0x61, 0x9d,
+                               0xb2, 0xce, 0xd5, 0x55, 0xf0, 0xde, 0xd7, 0x3f,
+                               0xdb, 0xd6, 0x49, 0x69, 0xb3, 0x67, 0xa9, 0x8f,
+                               0xfb, 0x2c, 0x71, 0xd2, 0x5a, 0xc5, 0xee, 0x92,
+                               0x73, 0x8e, 0x43, 0x7f, 0x4b, 0x3f, 0xff, 0xd6,
+                               0xee, 0x7f, 0xea, 0x9a, 0x4a, 0x19, 0x1f, 0xe3,
+                               0x54, 0xba, 0xd3, 0xd1, 0x55, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                               0x00, 0x00, 0x00, 0x1b, 0xbc, 0x00, 0xb5, 0xe8,
+                               0x4a, 0xf9, 0x00, 0x28, 0x00, 0x00,
+                       },
+                       mockdb: &db.MockDB{
+                               Err: pkgerrors.New("DB Error"),
+                       },
+               },
+       }
+
+       for _, testCase := range testCases {
+               t.Run(testCase.label, func(t *testing.T) {
+                       db.DBconn = testCase.mockdb
+                       impl := NewDefinitionClient()
+                       err := impl.Upload(testCase.inp, testCase.content)
+                       if err != nil {
+                               if testCase.expectedError == "" {
+                                       t.Errorf("Upload returned an unexpected error %s", err)
+                               }
+                               if strings.Contains(err.Error(), testCase.expectedError) == false {
+                                       t.Errorf("Upload returned an unexpected error %s", err)
+                               }
+                       }
+               })
+       }
+}