k8s_path="$(git rev-parse --show-toplevel)"
-VERSION="0.9.0-SNAPSHOT"
+VERSION="0.9.1-SNAPSHOT"
export IMAGE_NAME="nexus3.onap.org:10003/onap/multicloud/k8s"
function _compile_src {
when: "'emco' in emco_roles"
- name: Install emco helm charts
- command: /usr/local/bin/helm install --wait --namespace emco -f helm_value_overrides.yaml --set emco-db.etcd.clusterDomain={{ cluster_name.stdout }} --set emco-tools.fluentd.clusterDomain={{ cluster_name.stdout }} emco dist/packages/emco-0.1.0.tgz
+ command: /usr/local/bin/helm install --wait --timeout 10m --namespace emco -f helm_value_overrides.yaml --set emco-db.etcd.clusterDomain={{ cluster_name.stdout }} --set emco-tools.fluentd.clusterDomain={{ cluster_name.stdout }} emco dist/packages/emco-0.1.0.tgz
args:
chdir: "{{ emco_dir }}/deployments/helm/emcoOpenNESS"
when: "'emco' in emco_roles"
emco_git_url: "https://github.com/open-ness/EMCO.git"
emco_repository: "integratedcloudnative/"
-emco_version: "openness-21.03"
+emco_version: "openness-21.03.06"
emco_dir: "/opt/emco"
emco_values:
global:
imageTag: "{{ emco_version }}"
dtc:
imageTag: "{{ emco_version }}"
+ nps:
+ imageTag: "{{ emco_version }}"
+ sds:
+ imageTag: "{{ emco_version }}"
gac:
imageTag: "{{ emco_version }}"
+ sfc:
+ imageTag: "{{ emco_version }}"
+ sfcclient:
+ imageTag: "{{ emco_version }}"
emcoconfig_localhost: true
emcoctl_localhost: true
emco_roles:
This deploys the addons listed in the \`Addons\` and
\`AddonResources\` values in values.yaml.
+NOTE: On a single node cluster, the SRIOV addon resource will trigger
+a drain of the worker node. KubeVirt will prevent the drain from
+completing due to its PodDisruptionBudget. The workaround is to scale
+down the KubeVirt operator before applying 04-addon-resources-app.yaml
+then scaling it back up after the node is ready again.
+
\`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 03-addons-app.yaml -v values.yaml\`
\`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f 04-addon-resources-app.yaml -v values.yaml\`
AddonsDeploymentIntent: addons-deployment-intent
AddonsPlacementIntent: addons-placement-intent
Addons:
+- kubevirt-operator
+- cdi-operator
- multus-cni
- ovn4nfv
- node-feature-discovery
AddonResources:
- ovn4nfv-network
- sriov-network
+- kubevirt
+- cdi
}
function terminateOrchData {
- call_api -d "{ }" "${base_url_dcm}/projects/${projectname}/logical-clouds/${admin_logical_cloud_name}/terminate"
call_api -d "{ }" "${base_url_orchestrator}/projects/${projectname}/composite-apps/${collection_compositeapp_name}/${compositeapp_version}/deployment-intent-groups/${deployment_intent_group_name}/terminate"
call_api -d "{ }" "${base_url_dcm}/projects/${projectname}/logical-clouds/${admin_logical_cloud_name}/terminate"
}
--- /dev/null
+distribution_type: 'container'
+container_release_tag: '0.9.0'
+project: 'multicloud-k8s'
+log_dir: 'multicloud-k8s-master-docker-golang-shell-daily/1041/'
+ref: 0a1b40c1042ebdf212c46f67dd62bf13c2eee5aa
+containers:
+ - name: 'multicloud/k8s'
+ version: '0.9.0-SNAPSHOT'
/*
Copyright 2018 Intel Corporation.
Copyright © 2021 Samsung Electronics
+Copyright © 2021 Orange
+
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
func NewRouter(defClient rb.DefinitionManager,
profileClient rb.ProfileManager,
instClient app.InstanceManager,
+ queryClient app.QueryManager,
configClient app.ConfigManager,
connectionClient connection.ConnectionManager,
templateClient rb.ConfigTemplateManager,
Queries("rb-name", "{rb-name}",
"rb-version", "{rb-version}",
"profile-name", "{profile-name}").Methods("GET")
+ //Want to get full Data -> add query param: /install/{instID}?full=true
instRouter.HandleFunc("/instance/{instID}", instHandler.getHandler).Methods("GET")
instRouter.HandleFunc("/instance/{instID}/status", instHandler.statusHandler).Methods("GET")
instRouter.HandleFunc("/instance/{instID}/query", instHandler.queryHandler).Methods("GET")
"Name", "{Name}",
"Labels", "{Labels}").Methods("GET")
instRouter.HandleFunc("/instance/{instID}", instHandler.deleteHandler).Methods("DELETE")
- // (TODO): Fix update method
- // instRouter.HandleFunc("/{vnfInstanceId}", UpdateHandler).Methods("PUT")
+
+ // Query handler routes
+ if queryClient == nil {
+ queryClient = app.NewQueryClient()
+ }
+ queryHandler := queryHandler{client: queryClient}
+ queryRouter := router.PathPrefix("/v1").Subrouter()
+ queryRouter.HandleFunc("/query", queryHandler.queryHandler).Methods("GET")
+ queryRouter.HandleFunc("/query", queryHandler.queryHandler).
+ Queries("Namespace", "{Namespace}",
+ "CloudRegion", "{CloudRegion}",
+ "ApiVersion", "{ApiVersion}",
+ "Kind", "{Kind}",
+ "Name", "{Name}",
+ "Labels", "{Labels}").Methods("GET")
//Setup the broker handler here
//Use the base router without any path prefixes
}
configHandler := rbConfigHandler{client: configClient}
instRouter.HandleFunc("/instance/{instID}/config", configHandler.createHandler).Methods("POST")
+ instRouter.HandleFunc("/instance/{instID}/config", configHandler.listHandler).Methods("GET")
instRouter.HandleFunc("/instance/{instID}/config/{cfgname}", configHandler.getHandler).Methods("GET")
instRouter.HandleFunc("/instance/{instID}/config/{cfgname}", configHandler.updateHandler).Methods("PUT")
instRouter.HandleFunc("/instance/{instID}/config/{cfgname}", configHandler.deleteHandler).Methods("DELETE")
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("POST", "/cloudowner/cloudregion/infra_workload", testCase.input)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
defer resp.Body.Close()
if testCase.expectedCode != resp.StatusCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/cloudowner/cloudregion/infra_workload/"+testCase.input, nil)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
t.Fatalf("Request method returned: %v and it was expected: %v",
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/cloudowner/cloudregion/infra_workload?name="+testCase.input, nil)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
t.Fatalf("Request method returned: %v and it was expected: %v",
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("DELETE", "/cloudowner/cloudregion/infra_workload/"+testCase.input, nil)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
t.Fatalf("Request method returned: %v and it was expected: %v", resp.StatusCode, testCase.expectedCode)
/*
* Copyright 2018 Intel Corporation, Inc
* Copyright © 2021 Samsung Electronics
+ * Copyright © 2021 Orange
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
}
}
+// listHandler handles GET operations for all configs of instance
+// Returns a app.Definition
+func (h rbConfigHandler) listHandler(w http.ResponseWriter, r *http.Request) {
+ vars := mux.Vars(r)
+ instanceID := vars["instID"]
+
+ ret, err := h.client.List(instanceID)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ err = json.NewEncoder(w).Encode(ret)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+}
+
// deleteHandler handles DELETE operations on a config
func (h rbConfigHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("POST", "/v1/rb/definition", testCase.reader)
- resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/v1/rb/definition/testresourcebundle", nil)
- resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/v1/rb/definition", nil)
- resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/v1/rb/definition/"+testCase.name+"/"+testCase.version, nil)
- resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("DELETE", "/v1/rb/definition/"+testCase.name+"/"+testCase.version, nil)
- resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("POST",
"/v1/rb/definition/"+testCase.name+"/"+testCase.version+"/content", testCase.body)
- resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(testCase.rbDefClient, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
Err: nil,
}
request := httptest.NewRequest("GET", "/v1/healthcheck", nil)
- resp := executeRequest(request, NewRouter(nil, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != http.StatusOK {
Err: pkgerrors.New("Runtime Error in DB"),
}
request := httptest.NewRequest("GET", "/v1/healthcheck", nil)
- resp := executeRequest(request, NewRouter(nil, nil, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != http.StatusInternalServerError {
/*
Copyright 2018 Intel Corporation.
Copyright © 2021 Samsung Electronics
+Copyright © 2021 Orange
+
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
func (i instanceHandler) getHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["instID"]
+ var resp interface{}
+ var err error
+
+ if r.URL.Query().Get("full") == "true" {
+ resp, err = i.client.GetFull(id)
+ } else {
+ resp, err = i.client.Get(id)
+ }
- resp, err := i.client.Get(id)
if err != nil {
log.Error("Error getting Instance", log.Fields{
"error": err,
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
-
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
err = json.NewEncoder(w).Encode(resp)
name := r.FormValue("Name")
labels := r.FormValue("Labels")
if apiVersion == "" {
- http.Error(w, "Missing apiVersion mandatory parameter", http.StatusBadRequest)
+ http.Error(w, "Missing ApiVersion mandatory parameter", http.StatusBadRequest)
return
}
if kind == "" {
- http.Error(w, "Missing kind mandatory parameter", http.StatusBadRequest)
- return
- }
- if name == "" && labels == "" {
- http.Error(w, "Name or Labels parameter must be provided", http.StatusBadRequest)
+ http.Error(w, "Missing Kind mandatory parameter", http.StatusBadRequest)
return
}
resp, err := i.client.Query(id, apiVersion, kind, name, labels)
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("POST", "/v1/instance", testCase.input)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
body, _ := ioutil.ReadAll(resp.Body)
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/v1/instance/"+testCase.input, nil)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
t.Fatalf("Request method returned: %v and it was expected: %v",
}
request.URL.RawQuery = q.Encode()
}
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
t.Fatalf("Request method returned: %v and it was expected: %v",
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("DELETE", "/v1/instance/"+testCase.input, nil)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
t.Fatalf("Request method returned: %v and it was expected: %v", resp.StatusCode, testCase.expectedCode)
err: pkgerrors.New("Missing kind mandatory parameter"),
},
},
- {
- label: "Missing name or label mandatory parameters",
- id: "HaKpys8e",
- input: map[string]string{
- "ApiVersion": "v1",
- "Kind": "Pod",
- },
- expectedCode: http.StatusBadRequest,
- instClient: &mockInstanceClient{
- err: pkgerrors.New("Name or Labels parameter must be provided"),
- },
- },
{
label: "Query instance by name",
id: "HaKpys8e",
}
url := "/v1/instance/" + testCase.id + "/query?" + params.Encode()
request := httptest.NewRequest("GET", url, nil)
- resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, nil, testCase.instClient, nil, nil, nil, nil, nil))
if testCase.expectedCode != resp.StatusCode {
body, _ := ioutil.ReadAll(resp.Body)
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("POST", "/v1/rb/definition/test-rbdef/v1/profile",
testCase.reader)
- resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/v1/rb/definition/test-rbdef/v1/profile/"+testCase.prname, nil)
- resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("GET", "/v1/rb/definition/"+testCase.def+"/"+testCase.version+"/profile", nil)
- resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
for _, testCase := range testCases {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("DELETE", "/v1/rb/definition/test-rbdef/v1/profile/"+testCase.prname, nil)
- resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
t.Run(testCase.label, func(t *testing.T) {
request := httptest.NewRequest("POST",
"/v1/rb/definition/test-rbdef/v1/profile/"+testCase.prname+"/content", testCase.body)
- resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil))
+ resp := executeRequest(request, NewRouter(nil, testCase.rbProClient, nil, nil, nil, nil, nil, nil))
//Check returned code
if resp.StatusCode != testCase.expectedCode {
--- /dev/null
+/*
+Copyright 2018 Intel Corporation.
+Copyright © 2021 Samsung Electronics
+Copyright © 2021 Orange
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/app"
+ log "github.com/onap/multicloud-k8s/src/k8splugin/internal/logutils"
+)
+
+// Used to store the backend implementation objects
+// Also simplifies the mocking needed for unit testing
+type queryHandler struct {
+ // Interface that implements the Instance operations
+ client app.QueryManager
+}
+
+// queryHandler retrieves information about specified resources for instance
+func (i queryHandler) queryHandler(w http.ResponseWriter, r *http.Request) {
+ namespace := r.FormValue("Namespace")
+ cloudRegion := r.FormValue("CloudRegion")
+ apiVersion := r.FormValue("ApiVersion")
+ kind := r.FormValue("Kind")
+ name := r.FormValue("Name")
+ labels := r.FormValue("Labels")
+ if cloudRegion == "" {
+ http.Error(w, "Missing CloudRegion mandatory parameter", http.StatusBadRequest)
+ return
+ }
+ if apiVersion == "" {
+ http.Error(w, "Missing ApiVersion mandatory parameter", http.StatusBadRequest)
+ return
+ }
+ if kind == "" {
+ http.Error(w, "Missing Kind mandatory parameter", http.StatusBadRequest)
+ return
+ }
+ // instance id is irrelevant here
+ resp, err := i.client.Query(namespace, cloudRegion, apiVersion, kind, name, labels, "query")
+ if err != nil {
+ log.Error("Error getting Query results", log.Fields{
+ "error": err,
+ "cloudRegion": cloudRegion,
+ "namespace": namespace,
+ "apiVersion": apiVersion,
+ "kind": kind,
+ "name": name,
+ "labels": labels,
+ })
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ err = json.NewEncoder(w).Encode(resp)
+ if err != nil {
+ log.Error("Error Marshaling Response", log.Fields{
+ "error": err,
+ "response": resp,
+ })
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+}
import (
"context"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
"log"
"math/rand"
"net/http"
"time"
"github.com/onap/multicloud-k8s/src/k8splugin/api"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/auth"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/config"
rand.Seed(time.Now().UnixNano())
- httpRouter := api.NewRouter(nil, nil, nil, nil, nil, nil, nil)
+ httpRouter := api.NewRouter(nil, nil, nil, nil, nil, nil, nil, nil)
loggedRouter := handlers.LoggingHandler(os.Stdout, httpRouter)
log.Println("Starting Kubernetes Multicloud API")
gopkg.in/yaml.v2 v2.3.0
helm.sh/helm/v3 v3.5.0
k8s.io/api v0.20.1
+ k8s.io/apiextensions-apiserver v0.20.1
k8s.io/apimachinery v0.20.1
+ k8s.io/cli-runtime v0.20.1
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
rsc.io/letsencrypt v0.0.3 // indirect
)
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
+github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc=
github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/sprig/v3 v3.2.0 h1:P1ekkbuU73Ui/wS0nK1HOM37hh4xdfZo485UPf8rc+Y=
github.com/Masterminds/sprig/v3 v3.2.0/go.mod h1:tWhwTbUTndesPNeF0C900vKoq283u6zp4APT9vaF3SI=
+github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
+github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
github.com/Masterminds/squirrel v1.5.0 h1:JukIZisrUXadA9pl3rMkjhiamxiB0cXiu+HGp/Y8cY8=
github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=
github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990 h1:1xpVY4dSUSbW3PcSGxZJhI8Z+CJiqbd933kM7HIinTc=
github.com/Microsoft/hcsshim v0.8.10-0.20200715222032-5eafd1556990/go.mod h1:ay/0dTb7NsG8QMDfsRfLHgZo/6xAJShLe1+ePPflihk=
+github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 h1:kIFnQBO7rQ0XkMe6xEwbybYHBEaWmh/f++laI6Emt7M=
github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
+github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 h1:6ejg6Lkk8dskcM7wQ28gONkukbQkM4qpj4RnYbpFzrI=
+github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
github.com/deislabs/oras v0.8.1 h1:If674KraJVpujYR00rzdi0QAmW4BxzMJPVAZJKuhQ0c=
github.com/deislabs/oras v0.8.1/go.mod h1:Mx0rMSbBNaNfY9hjpccEnxkOqJL6KGjtxNHPLC4G4As=
+github.com/deislabs/oras v0.11.1 h1:oo2J/3vXdcti8cjFi8ghMOkx0OacONxHC8dhJ17NdJ0=
+github.com/deislabs/oras v0.11.1/go.mod h1:39lCtf8Q6WDC7ul9cnyWXONNzKvabEKk+AX+L0ImnQk=
github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 h1:FwssHbCDJD025h+BchanCwE1Q8fyMgqDr2mOQAWOLGw=
github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.5+incompatible h1:bjflayQbWg+xOkF2WPEAOi4Y7zWhR7ptoPhV/VqLVDE=
+github.com/docker/cli v20.10.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v0.0.0-20191216044856-a8371794149d/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v1.4.2-0.20200309214505-aa6a9891b09c h1:zviRyz1SWO8+WVJbi9/jlJCkrsZ54r/lTRbgtcaQhLs=
github.com/docker/docker v1.4.2-0.20200309214505-aa6a9891b09c/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible h1:iWPIG7pWIsCwT6ZtHnTUpoVMnete7O/pzd9HFE3+tn8=
+github.com/docker/docker v17.12.0-ce-rc1.0.20200618181300-9dc6525e6118+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ=
github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
github.com/docker/engine v0.0.0-20190620014054-c513a4c6c298 h1:dDGt5n84DvY05kaJT26cw1TDxNW1NymRZ13j0KeEQaw=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM=
github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ini/ini v1.9.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
+github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU=
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8=
github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w=
+github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg=
github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs=
github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE=
+github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes/kubernetes v1.19.4/go.mod h1:yhT1/ltQajQsha3tnYc9QPFYSumGM45nlZdjf7WqE1A=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8=
github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
+github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
+github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do=
github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
+github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4=
+github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI=
github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk=
+github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ=
+github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
+github.com/onap/multicloud-k8s v0.0.0-20210806192838-b64758e33cb2 h1:J7xclwJ0im3VtKDoix6FiVYazqNOP4bZGTwaI9HtNXU=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
+github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 h1:p7OofyZ509h8DmPLh8Hn+EIIZm/xYhdZHJ9GnXHdr6U=
github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
go.etcd.io/etcd v3.3.12+incompatible h1:V6PRYRGpU4k5EajJaaj/GL3hqIdzyPnBU8aPUp+35yw=
go.etcd.io/etcd v3.3.12+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8=
golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.1-0.20200106000736-b8fc810ca6b5/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.1/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
+google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
helm.sh/helm/v3 v3.5.0 h1:uqIT3Bh4hVEyZRThyTPik8FkiABj3VJIY+POvDFT3a4=
helm.sh/helm/v3 v3.5.0/go.mod h1:bjwXfmGAF+SEuJZ2AtN1xmTuz4FqaNYOJrXP+vtj6Tw=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.19.4 h1:I+1I4cgJYuCDgiLNjKx7SLmIbwgj9w7N7Zr5vSIdwpo=
k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk=
k8s.io/api v0.20.1 h1:ud1c3W3YNzGd6ABJlbFfKXBKXO+1KdGfcgGGNgFR03E=
k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
+k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
+k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
k8s.io/apiextensions-apiserver v0.19.4 h1:D9ak9T012tb3vcGFWYmbQuj9SCC8YM4zhA4XZqsAQC4=
k8s.io/apiextensions-apiserver v0.19.4/go.mod h1:B9rpH/nu4JBCtuUp3zTTk8DEjZUupZTBEec7/2zNRYw=
k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ=
k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=
+k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE=
+k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA=
k8s.io/apimachinery v0.19.4 h1:+ZoddM7nbzrDCp0T3SWnyxqf8cbWPT2fkZImoyvHUG0=
k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ=
k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
+k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
+k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
k8s.io/apiserver v0.19.4 h1:X40UuyVt6DcYWIh2olcePkyKO0LRJFvxWC0kLxYvkZU=
k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw=
k8s.io/apiserver v0.20.1 h1:yEqdkxlnQbxi/3e74cp0X16h140fpvPrNnNRAJBDuBk=
k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
+k8s.io/apiserver v0.21.2 h1:vfGLD8biFXHzbcIEXyW3652lDwkV8tZEFJAaS2iuJlw=
+k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw=
k8s.io/cli-runtime v0.19.4 h1:FPpoqFbWsFzRbZNRI+o/+iiLFmWMYTmBueIj3OaNVTI=
k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw=
k8s.io/cli-runtime v0.20.1 h1:fJhRQ9EfTpJpCqSFOAqnYLuu5aAM7yyORWZ26qW1jJc=
k8s.io/cli-runtime v0.20.1/go.mod h1:6wkMM16ZXTi7Ow3JLYPe10bS+XBnIkL6V9dmEz0mbuY=
+k8s.io/cli-runtime v0.21.2 h1:x40XY8UqrlWYY/lYH0PwqPk0i/Jo3C/PJM2V5zYkksk=
+k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
k8s.io/client-go v0.19.4 h1:85D3mDNoLF+xqpyE9Dh/OtrJDyJrSRKkHmDXIbEzer8=
k8s.io/client-go v0.19.4/go.mod h1:ZrEy7+wj9PjH5VMBCuu/BDlvtUAku0oVFk4MmnW9mWA=
k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
+k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA=
+k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0=
+k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
+k8s.io/client-go v1.5.2 h1:JOxmv4FxrCIOS54kAABbN8/hA9jqGpns+Zc6soNgd8U=
+k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
+k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible h1:U5Bt+dab9K8qaUmXINrkXO135kA11/i5Kg1RUydgaMQ=
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
k8s.io/cloud-provider v0.19.4 h1:f9VA9OHVS82/7ht/djlrc1ZyQRxKh9YjgP6WYknvVoY=
k8s.io/cloud-provider v0.19.4/go.mod h1:NZGs+2FmZ9PUmhXxXJmAwmiDjmPSc+uqGHUAvlZRXxo=
k8s.io/cluster-bootstrap v0.19.4/go.mod h1:6+8sq9PmyEmFI61YdU7/5ANx+net3usE7Hq0tJvR3D8=
k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
+k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
k8s.io/component-base v0.19.4 h1:HobPRToQ8KJ9ubRju6PUAk9I5V1GNMJZ4PyWbiWA0uI=
k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70=
+k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4=
+k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
+k8s.io/component-helpers v0.21.2/go.mod h1:DbyFt/A0p6Cv+R5+QOGSJ5f5t4xDfI8Yb89a57DgJlQ=
k8s.io/cri-api v0.19.4/go.mod h1:UN/iU9Ua0iYdDREBXNE9vqCJ7MIh/FW3VIL0d8pw7Fw=
k8s.io/csi-translation-lib v0.19.4/go.mod h1:Zqw8qVIHp6OU65hvIYeOlkMwHLpKy72h3P15AkY90W0=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
+k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU=
k8s.io/kube-controller-manager v0.19.4/go.mod h1:39Dh9ygYryrvBn6PCeGJRK7qHB0SjSOPmv32h2wT7Kc=
k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ=
k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
+k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/kube-proxy v0.19.4/go.mod h1:HqXWMgFOJQB0geLBXeMn2e5k9864NM/t3wrHxJjqqrk=
k8s.io/kube-scheduler v0.19.4/go.mod h1:r0SdH7fwIKyA/MURcRS4jNjTjfBEju3oIBll0Dxqjj4=
k8s.io/kubectl v0.19.4 h1:XFrHibf5fS4Ot8h3EnzdVsKrYj+pndlzKbwPkfra5hI=
k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0=
k8s.io/kubectl v0.20.1 h1:7h1vSrL/B3hLrhlCJhbTADElPKDbx+oVUt3+QDSXxBo=
k8s.io/kubectl v0.20.1/go.mod h1:2bE0JLYTRDVKDiTREFsjLAx4R2GvUtL/mGYFXfFFMzY=
+k8s.io/kubectl v0.21.2 h1:9XPCetvOMDqrIZZXb1Ei+g8t6KrIp9ENJaysQjUuLiE=
+k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU=
k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
k8s.io/legacy-cloud-providers v0.19.4/go.mod h1:/zI3R0GKwt29cn4Xaf2u5aM3tiWvQk7YqJk/PNwUp6Y=
k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU=
+k8s.io/metrics v0.21.2/go.mod h1:wzlOINZMCtWq8dR9gHlyaOemmYlOpAoldEIXE82gAhI=
k8s.io/sample-apiserver v0.19.4/go.mod h1:W6CbyItC0Hz0q1lLsHIWeRwSuijevDkk/P+txorhqy4=
k8s.io/system-validators v1.1.2/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg=
k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
+k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
rsc.io/letsencrypt v0.0.3 h1:H7xDfhkaFFSYEJlKeq38RwX2jYcnTeHuDQyT+mMNMwM=
rsc.io/letsencrypt v0.0.3/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9 h1:rusRLrDhjBp6aYtl9sGEvQJr6faoHoDLd0YcUBTZguI=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
+sigs.k8s.io/kustomize/api v0.8.8 h1:G2z6JPSSjtWWgMeWSoHdXqyftJNmMmyxXpwENGoOtGE=
+sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY=
+sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIMgYrEDfwOeFdMK0=
+sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
+sigs.k8s.io/kustomize/kyaml v0.10.17 h1:4zrV0ym5AYa0e512q7K3Wp1u7mzoWW0xR3UHJcGWGIg=
+sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA=
sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
/*
Copyright 2018 Intel Corporation.
Copyright © 2021 Samsung Electronics
+Copyright © 2021 Orange
+Copyright © 2021 Nokia Bell Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
import (
"context"
"io/ioutil"
+ appsv1 "k8s.io/api/apps/v1"
+ //appsv1beta1 "k8s.io/api/apps/v1beta1"
+ //appsv1beta2 "k8s.io/api/apps/v1beta2"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ //extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+ //apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+ //apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"os"
"strings"
"time"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
log "github.com/onap/multicloud-k8s/src/k8splugin/internal/logutils"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
+ logger "log"
pkgerrors "github.com/pkg/errors"
"k8s.io/apimachinery/pkg/api/meta"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
Status unstructured.Unstructured `json:"status"`
}
+func (k *KubernetesClient) getObjTypeForHook(kind string) (runtime.Object, error) {
+ switch kind {
+ case "Job":
+ return &batchv1.Job{}, nil
+ case "Pod":
+ return &corev1.Pod{}, nil
+ case "Deployment":
+ return &appsv1.Deployment{}, nil
+ case "DaemonSet":
+ return &appsv1.DaemonSet{}, nil
+ case "StatefulSet":
+ return &appsv1.StatefulSet{}, nil
+ default:
+ return nil, pkgerrors.New("kind " + kind + " unknown")
+ }
+}
+
+func (k *KubernetesClient) getRestApi(apiVersion string) (rest.Interface, error) {
+ //based on kubectl api-versions
+ switch apiVersion {
+ case "admissionregistration.k8s.io/v1":
+ return k.clientSet.AdmissionregistrationV1().RESTClient(), nil
+ case "admissionregistration.k8s.io/v1beta1":
+ return k.clientSet.AdmissionregistrationV1beta1().RESTClient(), nil
+ case "apps/v1":
+ return k.clientSet.AppsV1().RESTClient(), nil
+ case "apps/v1beta1":
+ return k.clientSet.AppsV1beta1().RESTClient(), nil
+ case "apps/v1beta2":
+ return k.clientSet.AppsV1beta2().RESTClient(), nil
+ case "authentication.k8s.io/v1":
+ return k.clientSet.AuthenticationV1().RESTClient(), nil
+ case "authentication.k8s.io/v1beta1":
+ return k.clientSet.AuthenticationV1beta1().RESTClient(), nil
+ case "authorization.k8s.io/v1":
+ return k.clientSet.AuthorizationV1().RESTClient(), nil
+ case "authorization.k8s.io/v1beta1":
+ return k.clientSet.AuthorizationV1beta1().RESTClient(), nil
+ case "autoscaling/v1":
+ return k.clientSet.AutoscalingV1().RESTClient(), nil
+ case "autoscaling/v2beta1":
+ return k.clientSet.AutoscalingV2beta1().RESTClient(), nil
+ case "autoscaling/v2beta2":
+ return k.clientSet.AutoscalingV2beta2().RESTClient(), nil
+ case "batch/v1":
+ return k.clientSet.BatchV1().RESTClient(), nil
+ case "batch/v1beta1":
+ return k.clientSet.BatchV1beta1().RESTClient(), nil
+ case "certificates.k8s.io/v1":
+ return k.clientSet.CertificatesV1().RESTClient(), nil
+ case "certificates.k8s.io/v1beta1":
+ return k.clientSet.CertificatesV1beta1().RESTClient(), nil
+ case "coordination.k8s.io/v1":
+ return k.clientSet.CoordinationV1().RESTClient(), nil
+ case "coordination.k8s.io/v1beta1":
+ return k.clientSet.CoordinationV1beta1().RESTClient(), nil
+ case "v1":
+ return k.clientSet.CoreV1().RESTClient(), nil
+ case "discovery.k8s.io/v1beta1":
+ return k.clientSet.DiscoveryV1beta1().RESTClient(), nil
+ case "events.k8s.io/v1":
+ return k.clientSet.EventsV1().RESTClient(), nil
+ case "events.k8s.io/v1beta1":
+ return k.clientSet.EventsV1beta1().RESTClient(), nil
+ case "extensions/v1beta1":
+ return k.clientSet.ExtensionsV1beta1().RESTClient(), nil
+ case "flowcontrol.apiserver.k8s.io/v1alpha1":
+ return k.clientSet.FlowcontrolV1alpha1().RESTClient(), nil
+ case "networking.k8s.io/v1":
+ return k.clientSet.NetworkingV1().RESTClient(), nil
+ case "networking.k8s.io/v1beta1":
+ return k.clientSet.NetworkingV1beta1().RESTClient(), nil
+ case "node.k8s.io/v1alpha1":
+ return k.clientSet.NodeV1alpha1().RESTClient(), nil
+ case "node.k8s.io/v1beta1":
+ return k.clientSet.NodeV1beta1().RESTClient(), nil
+ case "policy/v1beta1":
+ return k.clientSet.PolicyV1beta1().RESTClient(), nil
+ case "rbac.authorization.k8s.io/v1":
+ return k.clientSet.RbacV1().RESTClient(), nil
+ case "rbac.authorization.k8s.io/v1alpha1":
+ return k.clientSet.RbacV1alpha1().RESTClient(), nil
+ case "rbac.authorization.k8s.io/v1beta1":
+ return k.clientSet.RbacV1beta1().RESTClient(), nil
+ case "scheduling.k8s.io/v1":
+ return k.clientSet.SchedulingV1().RESTClient(), nil
+ case "scheduling.k8s.io/v1alpha1":
+ return k.clientSet.SchedulingV1alpha1().RESTClient(), nil
+ case "scheduling.k8s.io/v1beta1":
+ return k.clientSet.SchedulingV1beta1().RESTClient(), nil
+ case "storage.k8s.io/v1":
+ return k.clientSet.StorageV1().RESTClient(), nil
+ case "storage.k8s.io/v1alpha1":
+ return k.clientSet.StorageV1alpha1().RESTClient(), nil
+ case "storage.k8s.io/v1beta1":
+ return k.clientSet.StorageV1beta1().RESTClient(), nil
+ default:
+ return nil, pkgerrors.New("Api version " + apiVersion + " unknown")
+ }
+}
+
+func (k *KubernetesClient) WatchHookUntilReady(timeout time.Duration, ns string, res helm.KubernetesResource) error {
+ //for now, only generic plugin has dedicated WatchUntilReady implemented. Later, we can implement this function
+ //for each plugin separately.
+ pluginImpl, err := plugin.GetPluginByKind("generic")
+ if err != nil {
+ return pkgerrors.Wrap(err, "Error loading plugin")
+ }
+
+ mapper := k.GetMapper()
+ apiVersion, kind := res.GVK.ToAPIVersionAndKind()
+ if apiVersion == "" {
+ //apiVersion is empty -> we can suppose that the rss is ready
+ logger.Printf("apiVersion is empty, consider that the rss is ready")
+ return nil
+ }
+ objType, err := k.getObjTypeForHook(kind)
+ if err != nil {
+ //have error from getObjTypeForHook -> this kind is not considered in hook -> consider ready
+ return nil
+ }
+
+ logger.Printf("apiVersion: %s, Kind: %s", apiVersion, kind)
+ restClient, err := k.getRestApi(apiVersion)
+ if err != nil {
+ return pkgerrors.Wrap(err, "Get rest client")
+ }
+
+ return pluginImpl.WatchUntilReady(timeout, ns, res, mapper, restClient, objType, k.clientSet)
+}
+
// getPodsByLabel yields status of all pods under given instance ID
func (k *KubernetesClient) getPodsByLabel(namespace string) ([]ResourceStatus, error) {
client := k.GetStandardClient().CoreV1().Pods(namespace)
return nil, pkgerrors.Wrap(err, "Querying for resources")
}
- resp := make([]ResourceStatus, len(unstrList.Items))
+ resp := make([]ResourceStatus, 0)
for _, unstr := range unstrList.Items {
- resp = append(resp, ResourceStatus{unstr.GetName(), gvk, unstr})
+ if unstr.GetName() != "" {
+ resp = append(resp, ResourceStatus{unstr.GetName(), gvk, unstr})
+ }
}
return resp, nil
}
return nil
}
-func (k *KubernetesClient) CreateKind(resTempl helm.KubernetesResourceTemplate,
- namespace string) (helm.KubernetesResource, error) {
+func (k *KubernetesClient) CreateKind(resTempl helm.KubernetesResourceTemplate, namespace string) (helm.KubernetesResource, error) {
if _, err := os.Stat(resTempl.FilePath); os.IsNotExist(err) {
return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + "does not exists")
namespace string) (helm.KubernetesResource, error) {
if _, err := os.Stat(resTempl.FilePath); os.IsNotExist(err) {
- return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + "does not exists")
+ return helm.KubernetesResource{}, pkgerrors.New("File " + resTempl.FilePath + " does not exists")
}
log.Info("Processing Kubernetes Resource", log.Fields{
func (k *KubernetesClient) createResources(sortedTemplates []helm.KubernetesResourceTemplate,
namespace string) ([]helm.KubernetesResource, error) {
+ var createdResources []helm.KubernetesResource
+
err := k.ensureNamespace(namespace)
if err != nil {
- return nil, pkgerrors.Wrap(err, "Creating Namespace")
+ return createdResources, pkgerrors.Wrap(err, "Creating Namespace")
}
- var createdResources []helm.KubernetesResource
for _, resTempl := range sortedTemplates {
resCreated, err := k.CreateKind(resTempl, namespace)
if err != nil {
- return nil, pkgerrors.Wrapf(err, "Error creating kind: %+v", resTempl.GVK)
+ return createdResources, pkgerrors.Wrapf(err, "Error creating kind: %+v", resTempl.GVK)
}
createdResources = append(createdResources, resCreated)
}
import (
"encoding/base64"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
"io/ioutil"
"os"
"plugin"
"reflect"
"testing"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
ProfileName string `json:"profile-name"`
ConfigName string `json:"config-name"`
TemplateName string `json:"template-name"`
- ConfigVersion uint `json:"config-verion"`
+ ConfigVersion uint `json:"config-version"`
}
//ConfigRollback input
type ConfigManager interface {
Create(instanceID string, p Config) (ConfigResult, error)
Get(instanceID, configName string) (Config, error)
+ List(instanceID string) ([]Config, error)
Help() map[string]string
Update(instanceID, configName string, p Config) (ConfigResult, error)
Delete(instanceID, configName string) (ConfigResult, error)
return cfg, nil
}
+// List config entry in the database
+func (v *ConfigClient) List(instanceID string) ([]Config, error) {
+
+ // Acquire per profile Mutex
+ lock, _ := getProfileData(instanceID)
+ lock.Lock()
+ defer lock.Unlock()
+ // Read Config DB
+ cs := ConfigStore{
+ instanceID: instanceID,
+ }
+ cfg, err := cs.getConfigList()
+ if err != nil {
+ return []Config{}, pkgerrors.Wrap(err, "Get Config DB Entry")
+ }
+ return cfg, nil
+}
+
// Delete the Config from database
func (v *ConfigClient) Delete(instanceID, configName string) (ConfigResult, error) {
return Config{}, pkgerrors.Wrap(err, "Get Config DB Entry")
}
+// Read the config entry in the database
+func (c ConfigStore) getConfigList() ([]Config, error) {
+ rbName, rbVersion, profileName, _, err := resolveModelFromInstance(c.instanceID)
+ if err != nil {
+ return []Config{}, pkgerrors.Wrap(err, "Retrieving model info")
+ }
+ cfgKey := constructKey(rbName, rbVersion, profileName, c.instanceID, tagConfig)
+ values, err := db.Etcd.GetAll(cfgKey)
+ if err != nil {
+ return []Config{}, pkgerrors.Wrap(err, "Get Config DB List")
+ }
+ //value is a byte array
+ if values != nil {
+ result := make([]Config, 0)
+ for _, value := range values {
+ cfg := Config{}
+ err = db.DeSerialize(string(value), &cfg)
+ if err != nil {
+ return []Config{}, pkgerrors.Wrap(err, "Unmarshaling Config Value")
+ }
+ result = append(result, cfg)
+ }
+ return result, nil
+ }
+ return []Config{}, pkgerrors.Wrap(err, "Get Config DB List")
+}
+
// Delete the config entry in the database
func (c ConfigStore) deleteConfig() (Config, error) {
// Apply Config
func applyConfig(instanceID string, p Config, pChannel chan configResourceList, action string) error {
- rbName, rbVersion, profileName, _, err := resolveModelFromInstance(instanceID)
+ rbName, rbVersion, profileName, releaseName, err := resolveModelFromInstance(instanceID)
if err != nil {
return pkgerrors.Wrap(err, "Retrieving model info")
}
// Get Template and Resolve the template with values
- crl, err := resolve(rbName, rbVersion, profileName, p)
+ crl, err := resolve(rbName, rbVersion, profileName, p, releaseName)
if err != nil {
return pkgerrors.Wrap(err, "Resolve Config")
}
//Resolve returns the path where the helm chart merged with
//configuration overrides resides.
-var resolve = func(rbName, rbVersion, profileName string, p Config) (configResourceList, error) {
+var resolve = func(rbName, rbVersion, profileName string, p Config, releaseName string) (configResourceList, error) {
var resTemplates []helm.KubernetesResourceTemplate
return configResourceList{}, pkgerrors.Wrap(err, "Extracting Template")
}
+ var finalReleaseName string
+
+ if releaseName == "" {
+ finalReleaseName = profile.ReleaseName
+ } else {
+ finalReleaseName = releaseName
+ }
+
helmClient := helm.NewTemplateClient(profile.KubernetesVersion,
profile.Namespace,
- profile.ReleaseName)
+ finalReleaseName)
chartPath := filepath.Join(chartBasePath, t.ChartName)
resTemplates, _, err = helmClient.GenerateKubernetesArtifacts(chartPath,
import (
"fmt"
- "github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
"reflect"
"strings"
"testing"
+
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
// pkgerrors "github.com/pkg/errors"
)
db.Etcd = testCase.mockdb
db.DBconn = provideMockModelData(testCase.instanceID, testCase.rbName,
testCase.rbVersion, testCase.profileName)
- resolve = func(rbName, rbVersion, profileName string, p Config) (configResourceList, error) {
+ resolve = func(rbName, rbVersion, profileName string, p Config, releaseName string) (configResourceList, error) {
return configResourceList{}, nil
}
impl := NewConfigClient()
}
} else {
if reflect.DeepEqual(testCase.expected, got) == false {
- t.Errorf("Create Resource Bundle returned unexpected body: got %v;"+
+ t.Errorf("Create returned unexpected body: got %v;"+
" expected %v", got, testCase.expected)
}
}
db.Etcd = testCase.mockdb
db.DBconn = provideMockModelData(testCase.instanceID, testCase.rbName,
testCase.rbVersion, testCase.profileName)
- resolve = func(rbName, rbVersion, profileName string, p Config) (configResourceList, error) {
+ resolve = func(rbName, rbVersion, profileName string, p Config, releaseName string) (configResourceList, error) {
return configResourceList{}, nil
}
impl := NewConfigClient()
}
} else {
if reflect.DeepEqual(testCase.expected1, got) == false {
- t.Errorf("Create Resource Bundle returned unexpected body: got %v;"+
+ t.Errorf("Create returned unexpected body: got %v;"+
" expected %v", got, testCase.expected1)
}
}
+ get, err := impl.Get(testCase.instanceID, testCase.inp.ConfigName)
+ if err != nil {
+ if testCase.expectedError == "" {
+ t.Fatalf("Get returned an unexpected error %s", err)
+ }
+ if strings.Contains(err.Error(), testCase.expectedError) == false {
+ t.Fatalf("Get returned an unexpected error %s", err)
+ }
+ } else {
+ if reflect.DeepEqual(testCase.inp, get) == false {
+ t.Errorf("Get returned unexpected body: got %v;"+
+ " expected %v", get, testCase.inp)
+ }
+ }
+ getList, err := impl.List(testCase.instanceID)
+ if err != nil {
+ if testCase.expectedError == "" {
+ t.Fatalf("List returned an unexpected error %s", err)
+ }
+ if strings.Contains(err.Error(), testCase.expectedError) == false {
+ t.Fatalf("List returned an unexpected error %s", err)
+ }
+ } else {
+ if reflect.DeepEqual([]Config{testCase.inp}, getList) == false {
+ t.Errorf("List returned unexpected body: got %v;"+
+ " expected %v", getList, []Config{testCase.inp})
+ }
+ }
got, err = impl.Update(testCase.instanceID, testCase.inp.ConfigName, testCase.inpUpdate1)
if err != nil {
if testCase.expectedError == "" {
}
} else {
if reflect.DeepEqual(testCase.expected2, got) == false {
- t.Errorf("Create Resource Bundle returned unexpected body: got %v;"+
+ t.Errorf("Create returned unexpected body: got %v;"+
" expected %v", got, testCase.expected2)
}
}
}
} else {
if reflect.DeepEqual(testCase.expected3, got) == false {
- t.Errorf("Create Resource Bundle returned unexpected body: got %v;"+
+ t.Errorf("Create returned unexpected body: got %v;"+
" expected %v", got, testCase.expected3)
}
}
}
} else {
if reflect.DeepEqual(testCase.expected4, got) == false {
- t.Errorf("Create Resource Bundle returned unexpected body: got %v;"+
+ t.Errorf("Create returned unexpected body: got %v;"+
" expected %v", got, testCase.expected4)
}
}
--- /dev/null
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+ "context"
+ "sort"
+
+ apps "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ apiequality "k8s.io/apimachinery/pkg/api/equality"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ intstrutil "k8s.io/apimachinery/pkg/util/intstr"
+ appsclient "k8s.io/client-go/kubernetes/typed/apps/v1"
+)
+
+// deploymentutil contains a copy of a few functions from Kubernetes controller code to avoid a dependency on k8s.io/kubernetes.
+// This code is copied from https://github.com/kubernetes/kubernetes/blob/e856613dd5bb00bcfaca6974431151b5c06cbed5/pkg/controller/deployment/util/deployment_util.go
+// No changes to the code were made other than removing some unused functions
+
+// RsListFunc returns the ReplicaSet from the ReplicaSet namespace and the List metav1.ListOptions.
+type RsListFunc func(string, metav1.ListOptions) ([]*apps.ReplicaSet, error)
+
+// ListReplicaSets returns a slice of RSes the given deployment targets.
+// Note that this does NOT attempt to reconcile ControllerRef (adopt/orphan),
+// because only the controller itself should do that.
+// However, it does filter out anything whose ControllerRef doesn't match.
+func ListReplicaSets(deployment *apps.Deployment, getRSList RsListFunc) ([]*apps.ReplicaSet, error) {
+ // TODO: Right now we list replica sets by their labels. We should list them by selector, i.e. the replica set's selector
+ // should be a superset of the deployment's selector, see https://github.com/kubernetes/kubernetes/issues/19830.
+ namespace := deployment.Namespace
+ selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
+ if err != nil {
+ return nil, err
+ }
+ options := metav1.ListOptions{LabelSelector: selector.String()}
+ all, err := getRSList(namespace, options)
+ if err != nil {
+ return nil, err
+ }
+ // Only include those whose ControllerRef matches the Deployment.
+ owned := make([]*apps.ReplicaSet, 0, len(all))
+ for _, rs := range all {
+ if metav1.IsControlledBy(rs, deployment) {
+ owned = append(owned, rs)
+ }
+ }
+ return owned, nil
+}
+
+// ReplicaSetsByCreationTimestamp sorts a list of ReplicaSet by creation timestamp, using their names as a tie breaker.
+type ReplicaSetsByCreationTimestamp []*apps.ReplicaSet
+
+func (o ReplicaSetsByCreationTimestamp) Len() int { return len(o) }
+func (o ReplicaSetsByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
+func (o ReplicaSetsByCreationTimestamp) Less(i, j int) bool {
+ if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) {
+ return o[i].Name < o[j].Name
+ }
+ return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp)
+}
+
+// FindNewReplicaSet returns the new RS this given deployment targets (the one with the same pod template).
+func FindNewReplicaSet(deployment *apps.Deployment, rsList []*apps.ReplicaSet) *apps.ReplicaSet {
+ sort.Sort(ReplicaSetsByCreationTimestamp(rsList))
+ for i := range rsList {
+ if EqualIgnoreHash(&rsList[i].Spec.Template, &deployment.Spec.Template) {
+ // In rare cases, such as after cluster upgrades, Deployment may end up with
+ // having more than one new ReplicaSets that have the same template as its template,
+ // see https://github.com/kubernetes/kubernetes/issues/40415
+ // We deterministically choose the oldest new ReplicaSet.
+ return rsList[i]
+ }
+ }
+ // new ReplicaSet does not exist.
+ return nil
+}
+
+// EqualIgnoreHash returns true if two given podTemplateSpec are equal, ignoring the diff in value of Labels[pod-template-hash]
+// We ignore pod-template-hash because:
+// 1. The hash result would be different upon podTemplateSpec API changes
+// (e.g. the addition of a new field will cause the hash code to change)
+// 2. The deployment template won't have hash labels
+func EqualIgnoreHash(template1, template2 *v1.PodTemplateSpec) bool {
+ t1Copy := template1.DeepCopy()
+ t2Copy := template2.DeepCopy()
+ // Remove hash labels from template.Labels before comparing
+ delete(t1Copy.Labels, apps.DefaultDeploymentUniqueLabelKey)
+ delete(t2Copy.Labels, apps.DefaultDeploymentUniqueLabelKey)
+ return apiequality.Semantic.DeepEqual(t1Copy, t2Copy)
+}
+
+// GetNewReplicaSet returns a replica set that matches the intent of the given deployment; get ReplicaSetList from client interface.
+// Returns nil if the new replica set doesn't exist yet.
+func GetNewReplicaSet(deployment *apps.Deployment, c appsclient.AppsV1Interface) (*apps.ReplicaSet, error) {
+ rsList, err := ListReplicaSets(deployment, RsListFromClient(c))
+ if err != nil {
+ return nil, err
+ }
+ return FindNewReplicaSet(deployment, rsList), nil
+}
+
+// RsListFromClient returns an rsListFunc that wraps the given client.
+func RsListFromClient(c appsclient.AppsV1Interface) RsListFunc {
+ return func(namespace string, options metav1.ListOptions) ([]*apps.ReplicaSet, error) {
+ rsList, err := c.ReplicaSets(namespace).List(context.Background(), options)
+ if err != nil {
+ return nil, err
+ }
+ var ret []*apps.ReplicaSet
+ for i := range rsList.Items {
+ ret = append(ret, &rsList.Items[i])
+ }
+ return ret, err
+ }
+}
+
+// IsRollingUpdate returns true if the strategy type is a rolling update.
+func IsRollingUpdate(deployment *apps.Deployment) bool {
+ return deployment.Spec.Strategy.Type == apps.RollingUpdateDeploymentStrategyType
+}
+
+// MaxUnavailable returns the maximum unavailable pods a rolling deployment can take.
+func MaxUnavailable(deployment apps.Deployment) int32 {
+ if !IsRollingUpdate(&deployment) || *(deployment.Spec.Replicas) == 0 {
+ return int32(0)
+ }
+ // Error caught by validation
+ _, maxUnavailable, _ := ResolveFenceposts(deployment.Spec.Strategy.RollingUpdate.MaxSurge, deployment.Spec.Strategy.RollingUpdate.MaxUnavailable, *(deployment.Spec.Replicas))
+ if maxUnavailable > *deployment.Spec.Replicas {
+ return *deployment.Spec.Replicas
+ }
+ return maxUnavailable
+}
+
+// ResolveFenceposts resolves both maxSurge and maxUnavailable. This needs to happen in one
+// step. For example:
+//
+// 2 desired, max unavailable 1%, surge 0% - should scale old(-1), then new(+1), then old(-1), then new(+1)
+// 1 desired, max unavailable 1%, surge 0% - should scale old(-1), then new(+1)
+// 2 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1)
+// 1 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1)
+// 2 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1)
+// 1 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1)
+func ResolveFenceposts(maxSurge, maxUnavailable *intstrutil.IntOrString, desired int32) (int32, int32, error) {
+ surge, err := intstrutil.GetValueFromIntOrPercent(intstrutil.ValueOrDefault(maxSurge, intstrutil.FromInt(0)), int(desired), true)
+ if err != nil {
+ return 0, 0, err
+ }
+ unavailable, err := intstrutil.GetValueFromIntOrPercent(intstrutil.ValueOrDefault(maxUnavailable, intstrutil.FromInt(0)), int(desired), false)
+ if err != nil {
+ return 0, 0, err
+ }
+
+ if surge == 0 && unavailable == 0 {
+ // Validation should never allow the user to explicitly use zero values for both maxSurge
+ // maxUnavailable. Due to rounding down maxUnavailable though, it may resolve to zero.
+ // If both fenceposts resolve to zero, then we should set maxUnavailable to 1 on the
+ // theory that surge might not work due to quota.
+ unavailable = 1
+ }
+
+ return int32(surge), int32(unavailable), nil
+}
--- /dev/null
+/*
+Copyright © 2021 Nokia Bell Labs
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+ "fmt"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
+ "helm.sh/helm/v3/pkg/release"
+ "log"
+ "strings"
+ "time"
+)
+
+// Timeout used when deleting resources with a hook-delete-policy.
+const defaultHookDeleteTimeoutInSeconds = int64(60)
+
+// HookClient implements the Helm Hook interface
+type HookClient struct {
+ kubeNameSpace string
+ id string
+ dbStoreName string
+ dbTagInst string
+}
+
+type MultiCloudHook struct{
+ release.Hook
+ Group string
+ Version string
+}
+
+// NewHookClient returns a new instance of HookClient
+func NewHookClient(namespace, id, dbStoreName, dbTagInst string) *HookClient {
+ return &HookClient{
+ kubeNameSpace: namespace,
+ id: id,
+ dbStoreName: dbStoreName,
+ dbTagInst: dbTagInst,
+ }
+}
+
+func (hc *HookClient) getHookByEvent(hs []*helm.Hook, hook release.HookEvent) []*helm.Hook {
+ hooks := []*helm.Hook{}
+ for _, h := range hs {
+ for _, e := range h.Hook.Events {
+ if e == hook {
+ hooks = append(hooks, h)
+ }
+ }
+ }
+ return hooks
+}
+
+// Mimic function ExecHook in helm/pkg/tiller/release_server.go
+func (hc *HookClient) ExecHook(
+ k8sClient KubernetesClient,
+ hs []*helm.Hook,
+ hook release.HookEvent,
+ timeout int64,
+ startIndex int,
+ dbData *InstanceDbData) (error){
+ executingHooks := hc.getHookByEvent(hs, hook)
+ key := InstanceKey{
+ ID: hc.id,
+ }
+ log.Printf("Executing %d %s hook(s) for instance %s", len(executingHooks), hook, hc.id)
+ executingHooks = sortByHookWeight(executingHooks)
+
+ for index, h := range executingHooks {
+ if index < startIndex {
+ continue
+ }
+ // Set default delete policy to before-hook-creation
+ if h.Hook.DeletePolicies == nil || len(h.Hook.DeletePolicies) == 0 {
+ h.Hook.DeletePolicies = []release.HookDeletePolicy{release.HookBeforeHookCreation}
+ }
+ if err := hc.deleteHookByPolicy(h, release.HookBeforeHookCreation, k8sClient); err != nil {
+ return err
+ }
+ //update DB here before the creation of the hook, if the plugin quits
+ //-> when it comes back, it will continue from next hook and consider that this one is done
+ if dbData != nil {
+ dbData.HookProgress = fmt.Sprintf("%d/%d", index + 1, len(executingHooks))
+ err := db.DBconn.Update(hc.dbStoreName, key, hc.dbTagInst, dbData)
+ if err != nil {
+ return err
+ }
+ }
+ log.Printf(" Instance: %s, Creating %s hook %s, index %d", hc.id, hook, h.Hook.Name, index)
+ resTempl := helm.KubernetesResourceTemplate{
+ GVK: h.KRT.GVK,
+ FilePath: h.KRT.FilePath,
+ }
+ createdHook, err := k8sClient.CreateKind(resTempl, hc.kubeNameSpace)
+ if err != nil {
+ log.Printf(" Instance: %s, Warning: %s hook %s, filePath: %s, error: %s", hc.id, hook, h.Hook.Name, h.KRT.FilePath, err)
+ hc.deleteHookByPolicy(h, release.HookFailed, k8sClient)
+ return err
+ }
+ if hook != "crd-install" {
+ //timeout <= 0 -> do not wait
+ if timeout > 0 {
+ // Watch hook resources until they are completed
+ err = k8sClient.WatchHookUntilReady(time.Duration(timeout)*time.Second, hc.kubeNameSpace, createdHook)
+ if err != nil {
+ // If a hook is failed, check the annotation of the hook to determine whether the hook should be deleted
+ // under failed condition. If so, then clear the corresponding resource object in the hook
+ if err := hc.deleteHookByPolicy(h, release.HookFailed, k8sClient); err != nil {
+ return err
+ }
+ return err
+ }
+ }
+ } else {
+ //Do not handle CRD Hooks
+ }
+ }
+
+ for _, h := range executingHooks {
+ if err := hc.deleteHookByPolicy(h, release.HookSucceeded, k8sClient); err != nil {
+ log.Printf(" Instance: %s, Warning: Error deleting %s hook %s based on delete policy, continue", hc.id, hook, h.Hook.Name)
+ return err
+ }
+ }
+ log.Printf("%d %s hook(s) complete for release %s", len(executingHooks), hook, hc.id)
+ return nil
+}
+
+func (hc *HookClient) deleteHookByPolicy(h *helm.Hook, policy release.HookDeletePolicy, k8sClient KubernetesClient) error {
+ rss := helm.KubernetesResource{
+ GVK: h.KRT.GVK,
+ Name: h.Hook.Name,
+ }
+ if hookHasDeletePolicy(h, policy) {
+ log.Printf(" Instance: %s, Deleting hook %s due to %q policy", hc.id, h.Hook.Name, policy)
+ if errHookDelete := k8sClient.deleteResources(append([]helm.KubernetesResource{}, rss), hc.kubeNameSpace); errHookDelete != nil {
+ if strings.Contains(errHookDelete.Error(), "not found") {
+ return nil
+ } else {
+ log.Printf(" Instance: %s, Warning: hook %s, filePath %s could not be deleted: %s", hc.id, h.Hook.Name, h.KRT.FilePath ,errHookDelete)
+ return errHookDelete
+ }
+ } else {
+ //Verify that the rss is deleted
+ isDeleted := false
+ for !isDeleted {
+ log.Printf(" Instance: %s, Waiting on deleting hook %s for release %s due to %q policy", hc.id, h.Hook.Name, hc.id, policy)
+ if _, err := k8sClient.GetResourceStatus(rss, hc.kubeNameSpace); err != nil {
+ if strings.Contains(err.Error(), "not found") {
+ log.Printf(" Instance: %s, Deleted hook %s for release %s due to %q policy", hc.id, h.Hook.Name, hc.id, policy)
+ return nil
+ } else {
+ isDeleted = true
+ }
+ }
+ time.Sleep(5 * time.Second)
+ }
+ }
+ }
+ return nil
+}
+
+// hookHasDeletePolicy determines whether the defined hook deletion policy matches the hook deletion polices
+// supported by helm. If so, mark the hook as one should be deleted.
+func hookHasDeletePolicy(h *helm.Hook, policy release.HookDeletePolicy) bool {
+ for _, v := range h.Hook.DeletePolicies {
+ if policy == v {
+ return true
+ }
+ }
+ return false
+}
\ No newline at end of file
--- /dev/null
+/*
+Copyright © 2021 Nokia Bell Labs
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
+ sortLib "sort"
+)
+
+// sortByHookWeight does an in-place sort of hooks by their supplied weight.
+func sortByHookWeight(hooks []*helm.Hook) []*helm.Hook {
+ hs := newHookWeightSorter(hooks)
+ sortLib.Sort(hs)
+ return hs.hooks
+}
+
+type hookWeightSorter struct {
+ hooks []*helm.Hook
+}
+
+func newHookWeightSorter(h []*helm.Hook) *hookWeightSorter {
+ return &hookWeightSorter{
+ hooks: h,
+ }
+}
+
+func (hs *hookWeightSorter) Len() int { return len(hs.hooks) }
+
+func (hs *hookWeightSorter) Swap(i, j int) {
+ hs.hooks[i], hs.hooks[j] = hs.hooks[j], hs.hooks[i]
+}
+
+func (hs *hookWeightSorter) Less(i, j int) bool {
+ if hs.hooks[i].Hook.Weight == hs.hooks[j].Hook.Weight {
+ return hs.hooks[i].Hook.Name < hs.hooks[j].Hook.Name
+ }
+ return hs.hooks[i].Hook.Weight < hs.hooks[j].Hook.Weight
+}
+
--- /dev/null
+/*
+Copyright © 2021 Nokia Bell Labs.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+ "encoding/base64"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
+ "helm.sh/helm/v3/pkg/release"
+ "helm.sh/helm/v3/pkg/time"
+ "io/ioutil"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "testing"
+)
+
+func generateHookList() []*helm.Hook {
+ var hookList []*helm.Hook
+ preInstallHook1 := helm.Hook{
+ Hook: release.Hook{
+ Name : "preinstall1",
+ Kind : "Job",
+ Path : "",
+ Manifest : "",
+ Events : []release.HookEvent{release.HookPreInstall},
+ LastRun : release.HookExecution{
+ StartedAt: time.Now(),
+ CompletedAt: time.Now(),
+ Phase: "",
+ },
+ Weight : -5,
+ DeletePolicies : []release.HookDeletePolicy{},
+ },
+ KRT: helm.KubernetesResourceTemplate{
+ GVK: schema.GroupVersionKind{
+ Group: "batch",
+ Version: "v1",
+ Kind: "Job",
+ },
+ FilePath: "../../mock_files/mock_yamls/job.yaml",
+ },
+ }
+ preInstallHook2 := helm.Hook{
+ Hook: release.Hook{
+ Name : "preinstall2",
+ Kind : "Deployment",
+ Path : "",
+ Manifest : "",
+ Events : []release.HookEvent{release.HookPreInstall},
+ LastRun : release.HookExecution{
+ StartedAt: time.Now(),
+ CompletedAt: time.Now(),
+ Phase: "",
+ },
+ Weight : 0,
+ DeletePolicies : []release.HookDeletePolicy{},
+ },
+ KRT: helm.KubernetesResourceTemplate{
+ GVK: schema.GroupVersionKind{
+ Group: "batch",
+ Version: "v1",
+ Kind: "Job",
+ },
+ FilePath: "../../mock_files/mock_yamls/job.yaml",
+ },
+ }
+ postInstallHook := helm.Hook{
+ Hook: release.Hook{
+ Name : "postinstall",
+ Kind : "Job",
+ Path : "",
+ Manifest : "",
+ Events : []release.HookEvent{release.HookPostInstall},
+ LastRun : release.HookExecution{
+ StartedAt: time.Now(),
+ CompletedAt: time.Now(),
+ Phase: "",
+ },
+ Weight : -5,
+ DeletePolicies : []release.HookDeletePolicy{},
+ },
+ KRT: helm.KubernetesResourceTemplate{
+ GVK: schema.GroupVersionKind{
+ Group: "batch",
+ Version: "v1",
+ Kind: "Job",
+ },
+ FilePath: "../../mock_files/mock_yamls/job.yaml",
+ },
+ }
+ preDeleteHook := helm.Hook{
+ Hook: release.Hook{
+ Name : "predelete",
+ Kind : "Job",
+ Path : "",
+ Manifest : "",
+ Events : []release.HookEvent{release.HookPreDelete},
+ LastRun : release.HookExecution{
+ StartedAt: time.Now(),
+ CompletedAt: time.Now(),
+ Phase: "",
+ },
+ Weight : -5,
+ DeletePolicies : []release.HookDeletePolicy{},
+ },
+ KRT: helm.KubernetesResourceTemplate{
+ GVK: schema.GroupVersionKind{
+ Group: "batch",
+ Version: "v1",
+ Kind: "Job",
+ },
+ FilePath: "../../mock_files/mock_yamls/job.yaml",
+ },
+ }
+ postDeleteHook := helm.Hook{
+ Hook: release.Hook{
+ Name : "postdelete",
+ Kind : "Job",
+ Path : "",
+ Manifest : "",
+ Events : []release.HookEvent{release.HookPostDelete},
+ LastRun : release.HookExecution{
+ StartedAt: time.Now(),
+ CompletedAt: time.Now(),
+ Phase: "",
+ },
+ Weight : -5,
+ DeletePolicies : []release.HookDeletePolicy{},
+ },
+ KRT: helm.KubernetesResourceTemplate{
+ GVK: schema.GroupVersionKind{
+ Group: "batch",
+ Version: "v1",
+ Kind: "Job",
+ },
+ FilePath: "../../mock_files/mock_yamls/job.yaml",
+ },
+ }
+ hookList = append(hookList, &preInstallHook2)
+ hookList = append(hookList, &preInstallHook1)
+ hookList = append(hookList, &postInstallHook)
+ hookList = append(hookList, &preDeleteHook)
+ hookList = append(hookList, &postDeleteHook)
+
+ return hookList
+}
+
+func TestGetHookByEvent(t *testing.T) {
+ hookList := generateHookList()
+ hookClient := NewHookClient("test", "test", "rbdef", "instance")
+ t.Run("Get pre-install hook", func(t *testing.T) {
+ preinstallList := hookClient.getHookByEvent(hookList, release.HookPreInstall)
+ if len(preinstallList) != 2 {
+ t.Fatalf("TestGetHookByEvent error: expected=2 preinstall hook, result= %d", len(preinstallList))
+ }
+ if preinstallList[0].Hook.Name != "preinstall2" {
+ t.Fatalf("TestGetHookByEvent error: expect name of 1st preinstall hook is preinstall2, result= %s", preinstallList[0].Hook.Name)
+ }
+ if preinstallList[1].Hook.Name != "preinstall1" {
+ t.Fatalf("TestGetHookByEvent error: expect name of 2nd preinstall hook is preinstall1, result= %s", preinstallList[0].Hook.Name)
+ }
+ })
+ t.Run("Get post-install hook", func(t *testing.T) {
+ postinstallList := hookClient.getHookByEvent(hookList, release.HookPostInstall)
+ if len(postinstallList) != 1 {
+ t.Fatalf("TestGetHookByEvent error: expected=1 postinstall hook, result= %d", len(postinstallList))
+ }
+ if postinstallList[0].Hook.Name != "postinstall" {
+ t.Fatalf("TestGetHookByEvent error: expect name of 1st postinstall hook is postinstall, result= %s", postinstallList[0].Hook.Name)
+ }
+ })
+ t.Run("Get pre-delete hook", func(t *testing.T) {
+ predeleteList := hookClient.getHookByEvent(hookList, release.HookPreDelete)
+ if len(predeleteList) != 1 {
+ t.Fatalf("TestGetHookByEvent error: expected=1 predelete hook, result= %d", len(predeleteList))
+ }
+ if predeleteList[0].Hook.Name != "predelete" {
+ t.Fatalf("TestGetHookByEvent error: expect name of 1st predelete hook is predelete, result= %s", predeleteList[0].Hook.Name)
+ }
+ })
+ t.Run("Get post-delete hook", func(t *testing.T) {
+ postdeleteList := hookClient.getHookByEvent(hookList, release.HookPostDelete)
+ if len(postdeleteList) != 1 {
+ t.Fatalf("TestGetHookByEvent error: expected=1 postdelete hook, result= %d", len(postdeleteList))
+ }
+ if postdeleteList[0].Hook.Name != "postdelete" {
+ t.Fatalf("TestGetHookByEvent error: expect name of 1st postdelete hook is postdelete, result= %s", postdeleteList[0].Hook.Name)
+ }
+ })
+}
+
+func TestShortHook(t *testing.T) {
+ hookList := generateHookList()
+ hookClient := NewHookClient("test", "test", "rbdef", "instance")
+ preinstallList := hookClient.getHookByEvent(hookList, release.HookPreInstall)
+ t.Run("Short pre-install hook", func(t *testing.T) {
+ shortedHooks := sortByHookWeight(preinstallList)
+ if shortedHooks[0].Hook.Name != "preinstall1" {
+ t.Fatalf("TestShortHook error: expect name of 1st preinstall hook is preinstall1, result= %s", preinstallList[0].Hook.Name)
+ }
+ if shortedHooks[1].Hook.Name != "preinstall2" {
+ t.Fatalf("TestShortHook error: expect name of 2nd preinstall hook is preinstall2, result= %s", preinstallList[0].Hook.Name)
+ }
+ })
+}
+
+func TestExecHook(t *testing.T) {
+ hookList := generateHookList()
+ hookClient := NewHookClient("test", "test", "rbdef", "instance")
+ err := LoadMockPlugins(utils.LoadedPlugins)
+ if err != nil {
+ t.Fatalf("LoadMockPlugins returned an error (%s)", err)
+ }
+
+ // Load the mock kube config file into memory
+ fd, err := ioutil.ReadFile("../../mock_files/mock_configs/mock_kube_config")
+ if err != nil {
+ t.Fatal("Unable to read mock_kube_config")
+ }
+ db.DBconn = &db.MockDB{
+ Items: map[string]map[string][]byte{
+ connection.ConnectionKey{CloudRegion: "mock_connection"}.String(): {
+ "metadata": []byte(
+ "{\"cloud-region\":\"mock_connection\"," +
+ "\"cloud-owner\":\"mock_owner\"," +
+ "\"kubeconfig\": \"" + base64.StdEncoding.EncodeToString(fd) + "\"}"),
+ },
+ },
+ }
+
+ k8sClient := KubernetesClient{}
+ err = k8sClient.Init("mock_connection", "test")
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ err = hookClient.ExecHook(k8sClient, hookList, release.HookPreInstall,10,0, nil)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ err = hookClient.ExecHook(k8sClient, hookList, release.HookPostInstall,10,0, nil)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ err = hookClient.ExecHook(k8sClient, hookList, release.HookPreDelete,10,0, nil)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+ err = hookClient.ExecHook(k8sClient, hookList, release.HookPostDelete,10,0, nil)
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+}
\ No newline at end of file
/*
* Copyright 2018 Intel Corporation, Inc
* Copyright © 2021 Samsung Electronics
+ * Copyright © 2021 Orange
+ * Copyright © 2021 Nokia Bell Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
package app
import (
+ "context"
"encoding/json"
+ "log"
+ "strings"
+ "strconv"
+ "time"
+
+ appsv1 "k8s.io/api/apps/v1"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/cli-runtime/pkg/resource"
+
"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/namegenerator"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/rb"
- "k8s.io/apimachinery/pkg/runtime/schema"
- "log"
- "strings"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/statuscheck"
pkgerrors "github.com/pkg/errors"
+ "helm.sh/helm/v3/pkg/release"
)
// InstanceRequest contains the parameters needed for instantiation
Hooks []*helm.Hook `json:"-"`
}
+// InstanceDbData contains the data to put to Db
+type InstanceDbData struct {
+ ID string `json:"id"`
+ Request InstanceRequest `json:"request"`
+ Namespace string `json:"namespace"`
+ Status string `json:"status"`
+ ReleaseName string `json:"release-name"`
+ Resources []helm.KubernetesResource `json:"resources"`
+ Hooks []*helm.Hook `json:"hooks"`
+ HookProgress string `json:"hook-progress"`
+ PreInstallTimeout int64 `json:"PreInstallTimeout"`
+ PostInstallTimeout int64 `json:"PostInstallTimeout"`
+ PreDeleteTimeout int64 `json:"PreDeleteTimeout"`
+ PostDeleteTimeout int64 `json:"PostDeleteTimeout"`
+}
+
// InstanceMiniResponse contains the response from instantiation
// It does NOT include the created resources.
// Use the regular GET to get the created resources for a particular instance
type InstanceManager interface {
Create(i InstanceRequest) (InstanceResponse, error)
Get(id string) (InstanceResponse, error)
+ GetFull(id string) (InstanceDbData, error)
Status(id string) (InstanceStatus, error)
Query(id, apiVersion, kind, name, labels string) (InstanceStatus, error)
List(rbname, rbversion, profilename string) ([]InstanceMiniResponse, error)
Find(rbName string, ver string, profile string, labelKeys map[string]string) ([]InstanceMiniResponse, error)
Delete(id string) error
+ RecoverCreateOrDelete(id string) error
}
// InstanceKey is used as the primary key in the db
// InstanceClient implements the InstanceManager interface
// It will also be used to maintain some localized state
type InstanceClient struct {
- storeName string
- tagInst string
+ storeName string
+ tagInst string
}
// NewInstanceClient returns an instance of the InstanceClient
// which implements the InstanceManager
func NewInstanceClient() *InstanceClient {
+ //TODO: Call RecoverCreateOrDelete to perform recovery when the plugin restart.
+ //Not implement here now because We have issue with current test set (date race)
+
return &InstanceClient{
storeName: "rbdef",
tagInst: "instance",
// Create an instance of rb on the cluster in the database
func (v *InstanceClient) Create(i InstanceRequest) (InstanceResponse, error) {
-
// Name is required
if i.RBName == "" || i.RBVersion == "" || i.ProfileName == "" || i.CloudRegion == "" {
return InstanceResponse{},
//Convert override values from map to array of strings of the following format
//foo=bar
overrideValues := []string{}
+ var preInstallTimeOut, postInstallTimeOut, preDeleteTimeout, postDeleteTimeout int64
if i.OverrideValues != nil {
+ preInstallTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-pre-install-timeout"]
+ if !ok {
+ preInstallTimeOutStr = "60"
+ }
+ preInstallTimeOut,err = strconv.ParseInt(preInstallTimeOutStr, 10, 64)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-install-timeout")
+ }
+
+ postInstallTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-post-install-timeout"]
+ if !ok {
+ postInstallTimeOutStr = "600"
+ }
+ postInstallTimeOut,err = strconv.ParseInt(postInstallTimeOutStr, 10, 64)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-install-timeout")
+ }
+
+ preDeleteTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-pre-delete-timeout"]
+ if !ok {
+ preDeleteTimeOutStr = "60"
+ }
+ preDeleteTimeout,err = strconv.ParseInt(preDeleteTimeOutStr, 10, 64)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-pre-delete-timeout")
+ }
+
+ postDeleteTimeOutStr, ok := i.OverrideValues["k8s-rb-instance-post-delete-timeout"]
+ if !ok {
+ postDeleteTimeOutStr = "600"
+ }
+ postDeleteTimeout,err = strconv.ParseInt(postDeleteTimeOutStr, 10, 64)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Error parsing k8s-rb-instance-post-delete-timeout")
+ }
+
for k, v := range i.OverrideValues {
overrideValues = append(overrideValues, k+"="+v)
}
+ } else {
+ preInstallTimeOut = 60
+ postInstallTimeOut = 600
+ preDeleteTimeout = 60
+ postDeleteTimeout = 600
}
//Execute the kubernetes create command
return InstanceResponse{}, pkgerrors.Wrap(err, "Getting CloudRegion Information")
}
- createdResources, err := k8sClient.createResources(sortedTemplates, profile.Namespace)
+ log.Printf("Main rss info")
+ for _,t := range sortedTemplates {
+ log.Printf(" Path: %s", t.FilePath)
+ log.Printf(" Kind: %s", t.GVK.Kind)
+ }
+
+ log.Printf("Hook info")
+ for _,h := range hookList {
+ log.Printf(" Name: %s", h.Hook.Name)
+ log.Printf(" Events: %s", h.Hook.Events)
+ log.Printf(" Weight: %d", h.Hook.Weight)
+ log.Printf(" DeletePolicies: %s", h.Hook.DeletePolicies)
+ }
+ dbData := InstanceDbData{
+ ID: id,
+ Request: i,
+ Namespace: profile.Namespace,
+ ReleaseName: releaseName,
+ Status: "PRE-INSTALL",
+ Resources: []helm.KubernetesResource{},
+ Hooks: hookList,
+ HookProgress: "",
+ PreInstallTimeout: preInstallTimeOut,
+ PostInstallTimeout: postInstallTimeOut,
+ PreDeleteTimeout: preDeleteTimeout,
+ PostDeleteTimeout: postDeleteTimeout,
+ }
+
+ key := InstanceKey{
+ ID: id,
+ }
+ err = db.DBconn.Create(v.storeName, key, v.tagInst, dbData)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Instance DB Entry")
+ }
+
+ err = k8sClient.ensureNamespace(profile.Namespace)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Namespace")
+ }
+
+ hookClient := NewHookClient(profile.Namespace, id, v.storeName, v.tagInst)
+ if len(hookClient.getHookByEvent(hookList, release.HookPreInstall)) != 0 {
+ err = hookClient.ExecHook(k8sClient, hookList, release.HookPreInstall, preInstallTimeOut, 0, &dbData)
+ if err != nil {
+ log.Printf("Error running preinstall hooks for release %s, Error: %s. Stop here", releaseName, err)
+ err2 := db.DBconn.Delete(v.storeName, key, v.tagInst)
+ if err2 != nil {
+ log.Printf("Error cleaning failed instance in DB, please check DB.")
+ }
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Error running preinstall hooks")
+ }
+ }
+
+ dbData.Status = "CREATING"
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData)
+ if err != nil {
+ err = db.DBconn.Delete(v.storeName, key, v.tagInst)
+ if err != nil {
+ log.Printf("Delete Instance DB Entry for release %s has error.", releaseName)
+ }
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Update Instance DB Entry")
+ }
+
+ //Main rss creation is supposed to be very quick -> no need to support recover for main rss
+ createdResources, err := k8sClient.createResources(sortedTemplates, profile.Namespace);
if err != nil {
+ if len(createdResources) > 0 {
+ log.Printf("[Instance] Reverting created resources on Error: %s", err.Error())
+ k8sClient.deleteResources(createdResources, profile.Namespace)
+ }
+ log.Printf(" Instance: %s, Main rss are failed, skip post-install and remove instance in DB", id)
+ //main rss creation failed -> remove instance in DB
+ err = db.DBconn.Delete(v.storeName, key, v.tagInst)
+ if err != nil {
+ log.Printf("Delete Instance DB Entry for release %s has error.", releaseName)
+ }
return InstanceResponse{}, pkgerrors.Wrap(err, "Create Kubernetes Resources")
}
+ dbData.Status = "CREATED"
+ dbData.Resources = createdResources
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData)
+ if err != nil {
+ return InstanceResponse{}, pkgerrors.Wrap(err, "Update Instance DB Entry")
+ }
+
//Compose the return response
resp := InstanceResponse{
ID: id,
Hooks: hookList,
}
+ if len(hookClient.getHookByEvent(hookList, release.HookPostInstall)) != 0 {
+ go func() {
+ dbData.Status = "POST-INSTALL"
+ dbData.HookProgress = ""
+ err = hookClient.ExecHook(k8sClient, hookList, release.HookPostInstall, postInstallTimeOut, 0, &dbData)
+ if err != nil {
+ dbData.Status = "POST-INSTALL-FAILED"
+ log.Printf(" Instance: %s, Error running postinstall hooks error: %s", id, err)
+ } else {
+ dbData.Status = "DONE"
+ }
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", releaseName)
+ }
+ }()
+ } else {
+ dbData.Status = "DONE"
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, dbData)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", releaseName)
+ }
+ }
+
+ return resp, nil
+}
+
+// Get returns the full instance for corresponding ID
+func (v *InstanceClient) GetFull(id string) (InstanceDbData, error) {
key := InstanceKey{
ID: id,
}
- err = db.DBconn.Create(v.storeName, key, v.tagInst, resp)
+ value, err := db.DBconn.Read(v.storeName, key, v.tagInst)
if err != nil {
- return InstanceResponse{}, pkgerrors.Wrap(err, "Creating Instance DB Entry")
+ return InstanceDbData{}, pkgerrors.Wrap(err, "Get Instance")
}
- return resp, nil
+ //value is a byte array
+ if value != nil {
+ resp := InstanceDbData{}
+ err = db.DBconn.Unmarshal(value, &resp)
+ if err != nil {
+ return InstanceDbData{}, pkgerrors.Wrap(err, "Unmarshaling Instance Value")
+ }
+ //In case that we are communicating with an old db, some field will missing -> fill it with default value
+ if resp.Status == "" {
+ //For instance that is in Db -> consider it's DONE
+ resp.Status = "DONE"
+ }
+ if resp.PreInstallTimeout == 0 {
+ resp.PreInstallTimeout = 60
+ }
+ if resp.PostInstallTimeout == 0 {
+ resp.PostInstallTimeout = 600
+ }
+ if resp.PreDeleteTimeout == 0 {
+ resp.PreInstallTimeout = 60
+ }
+ if resp.PostDeleteTimeout == 0 {
+ resp.PostDeleteTimeout = 600
+ }
+ return resp, nil
+ }
+
+ return InstanceDbData{}, pkgerrors.New("Error getting Instance")
}
// Get returns the instance for corresponding ID
// Query returns state of instance's filtered resources
func (v *InstanceClient) Query(id, apiVersion, kind, name, labels string) (InstanceStatus, error) {
+ queryClient := NewQueryClient()
//Read the status from the DB
key := InstanceKey{
ID: id,
return InstanceStatus{}, pkgerrors.Wrap(err, "Unmarshaling Instance Value")
}
- k8sClient := KubernetesClient{}
- err = k8sClient.Init(resResp.Request.CloudRegion, id)
+ resources, err := queryClient.Query(resResp.Namespace, resResp.Request.CloudRegion, apiVersion, kind, name, labels, id)
if err != nil {
- return InstanceStatus{}, pkgerrors.Wrap(err, "Getting CloudRegion Information")
- }
-
- var resourcesStatus []ResourceStatus
- if labels != "" {
- resList, err := k8sClient.queryResources(apiVersion, kind, labels, resResp.Namespace)
- if err != nil {
- return InstanceStatus{}, pkgerrors.Wrap(err, "Querying Resources")
- }
- // If user specifies both label and name, we want to pick up only single resource from these matching label
- if name != "" {
- //Assigning 0-length, because we may actually not find matching name
- resourcesStatus = make([]ResourceStatus, 0)
- for _, res := range resList {
- if res.Name == name {
- resourcesStatus = append(resourcesStatus, res)
- break
- }
- }
- } else {
- resourcesStatus = resList
- }
- } else if name != "" {
- resIdentifier := helm.KubernetesResource{
- Name: name,
- GVK: schema.FromAPIVersionAndKind(apiVersion, kind),
- }
- res, err := k8sClient.GetResourceStatus(resIdentifier, resResp.Namespace)
- if err != nil {
- return InstanceStatus{}, pkgerrors.Wrap(err, "Querying Resource")
- }
- resourcesStatus = []ResourceStatus{res}
+ return InstanceStatus{}, pkgerrors.Wrap(err, "Querying Resources")
}
resp := InstanceStatus{
Request: resResp.Request,
- ResourceCount: int32(len(resourcesStatus)),
- ResourcesStatus: resourcesStatus,
+ ResourceCount: resources.ResourceCount,
+ ResourcesStatus: resources.ResourcesStatus,
}
return resp, nil
}
// Status returns the status for the instance
func (v *InstanceClient) Status(id string) (InstanceStatus, error) {
-
//Read the status from the DB
key := InstanceKey{
ID: id,
return InstanceStatus{}, pkgerrors.New("Status is not available")
}
- resResp := InstanceResponse{}
+ resResp := InstanceDbData{}
err = db.DBconn.Unmarshal(value, &resResp)
if err != nil {
return InstanceStatus{}, pkgerrors.Wrap(err, "Unmarshaling Instance Value")
cumulatedErrorMsg = append(cumulatedErrorMsg, err.Error())
}
+ isReady := true
generalStatus := make([]ResourceStatus, 0, len(resResp.Resources))
Main:
- for _, resource := range resResp.Resources {
+ for _, oneResource := range resResp.Resources {
for _, pod := range podsStatus {
- if resource.GVK == pod.GVK && resource.Name == pod.Name {
+ if oneResource.GVK == pod.GVK && oneResource.Name == pod.Name {
continue Main //Don't double check pods if someone decided to define pod explicitly in helm chart
}
}
- status, err := k8sClient.GetResourceStatus(resource, resResp.Namespace)
+ status, err := k8sClient.GetResourceStatus(oneResource, resResp.Namespace)
if err != nil {
cumulatedErrorMsg = append(cumulatedErrorMsg, err.Error())
+ isReady = false
} else {
generalStatus = append(generalStatus, status)
+ ready, err := v.checkRssStatus(oneResource, k8sClient, resResp.Namespace, status)
+
+ if !ready || err != nil {
+ isReady = false
+ if err != nil {
+ cumulatedErrorMsg = append(cumulatedErrorMsg, err.Error())
+ }
+ }
}
}
+ //We still need to iterate through rss list even the status is not DONE, to gather status of rss + pod for the response
resp := InstanceStatus{
Request: resResp.Request,
ResourceCount: int32(len(generalStatus) + len(podsStatus)),
- Ready: false, //FIXME To determine readiness, some parsing of status fields is necessary
+ Ready: isReady && resResp.Status == "DONE",
ResourcesStatus: append(generalStatus, podsStatus...),
}
return resp, nil
}
+func (v *InstanceClient) checkRssStatus(rss helm.KubernetesResource, k8sClient KubernetesClient, namespace string, status ResourceStatus) (bool, error) {
+ readyChecker := statuscheck.NewReadyChecker(k8sClient.clientSet, statuscheck.PausedAsReady(true), statuscheck.CheckJobs(true))
+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(60)*time.Second)
+ defer cancel()
+
+ apiVersion, kind := rss.GVK.ToAPIVersionAndKind()
+ log.Printf("apiVersion: %s, Kind: %s", apiVersion, kind)
+ restClient, err := k8sClient.getRestApi(apiVersion)
+ if err != nil {
+ return false, err
+ }
+ mapper := k8sClient.GetMapper()
+ mapping, err := mapper.RESTMapping(schema.GroupKind{
+ Group: rss.GVK.Group,
+ Kind: rss.GVK.Kind,
+ }, rss.GVK.Version)
+ resourceInfo := resource.Info{
+ Client: restClient,
+ Mapping: mapping,
+ Namespace: namespace,
+ Name: rss.Name,
+ Source: "",
+ Object: nil,
+ ResourceVersion: "",
+ }
+
+ var parsedRes runtime.Object
+ //TODO: Should we care about different api version for a same kind?
+ switch kind {
+ case "Pod":
+ parsedRes = new(corev1.Pod)
+ case "Job":
+ parsedRes = new(batchv1.Job)
+ case "Deployment":
+ parsedRes = new(appsv1.Deployment)
+ case "PersistentVolumeClaim":
+ parsedRes = new(corev1.PersistentVolume)
+ case "Service":
+ parsedRes = new(corev1.Service)
+ case "DaemonSet":
+ parsedRes = new(appsv1.DaemonSet)
+ case "CustomResourceDefinition":
+ parsedRes = new(apiextv1.CustomResourceDefinition)
+ case "StatefulSet":
+ parsedRes = new(appsv1.StatefulSet)
+ case "ReplicationController":
+ parsedRes = new(corev1.ReplicationController)
+ case "ReplicaSet":
+ parsedRes = new(appsv1.ReplicaSet)
+ default:
+ //For not listed resource, consider ready
+ return true, nil
+ }
+ err = runtime.DefaultUnstructuredConverter.FromUnstructured(status.Status.Object, parsedRes)
+ if err != nil {
+ return false, err
+ }
+ resourceInfo.Object = parsedRes
+ ready, err := readyChecker.IsReady(ctx, &resourceInfo)
+ return ready, err
+}
+
// List returns the instance for corresponding ID
// Empty string returns all
func (v *InstanceClient) List(rbname, rbversion, profilename string) ([]InstanceMiniResponse, error) {
for key, value := range dbres {
//value is a byte array
if value != nil {
- resp := InstanceResponse{}
+ resp := InstanceDbData{}
err = db.DBconn.Unmarshal(value, &resp)
if err != nil {
log.Printf("[Instance] Error: %s Unmarshaling Instance: %s", err.Error(), key)
continue
}
+ if resp.Status == "PRE-INSTALL" {
+ //DO not add instance which is in pre-install phase
+ continue
+ }
+
results = append(results, miniresp)
}
}
if add {
ret = append(ret, resp)
}
-
}
return ret, nil
// Delete the Instance from database
func (v *InstanceClient) Delete(id string) error {
- inst, err := v.Get(id)
+ inst, err := v.GetFull(id)
if err != nil {
return pkgerrors.Wrap(err, "Error getting Instance")
}
+ key := InstanceKey{
+ ID: id,
+ }
+ if inst.Status == "DELETED" {
+ //The instance is deleted when the plugin comes back -> just remove from Db
+ err = db.DBconn.Delete(v.storeName, key, v.tagInst)
+ if err != nil {
+ log.Printf("Delete Instance DB Entry for release %s has error.", inst.ReleaseName)
+ }
+ return nil
+ } else if inst.Status != "DONE"{
+ //Recover is ongoing, do nothing here
+ return nil
+ }
k8sClient := KubernetesClient{}
err = k8sClient.Init(inst.Request.CloudRegion, inst.ID)
if err != nil {
return pkgerrors.Wrap(err, "Getting CloudRegion Information")
}
+ inst.Status = "PRE-DELETE"
+ inst.HookProgress = ""
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, inst)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", inst.ReleaseName)
+ }
+
+ hookClient := NewHookClient(inst.Namespace, id, v.storeName, v.tagInst)
+ if len(hookClient.getHookByEvent(inst.Hooks, release.HookPreDelete)) != 0 {
+ err = hookClient.ExecHook(k8sClient, inst.Hooks, release.HookPreDelete, inst.PreDeleteTimeout, 0, &inst)
+ if err != nil {
+ log.Printf(" Instance: %s, Error running pre-delete hooks error: %s", id, err)
+ inst.Status = "PRE-DELETE-FAILED"
+ err2 := db.DBconn.Update(v.storeName, key, v.tagInst, inst)
+ if err2 != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", inst.ReleaseName)
+ }
+ return pkgerrors.Wrap(err, "Error running pre-delete hooks")
+ }
+ }
+
+ inst.Status = "DELETING"
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, inst)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", inst.ReleaseName)
+ }
err = k8sClient.deleteResources(inst.Resources, inst.Namespace)
if err != nil {
return pkgerrors.Wrap(err, "Deleting Instance Resources")
}
+ if len(hookClient.getHookByEvent(inst.Hooks, release.HookPostDelete)) != 0 {
+ go func() {
+ inst.HookProgress = ""
+ if err := v.runPostDelete(k8sClient, hookClient, &inst, 0, true); err != nil {
+ log.Printf(err.Error())
+ }
+ }()
+ } else {
+ err = db.DBconn.Delete(v.storeName, key, v.tagInst)
+ if err != nil {
+ return pkgerrors.Wrap(err, "Delete Instance")
+ }
+ }
+ return nil
+}
+
+//Continue the instantiation
+func (v *InstanceClient) RecoverCreateOrDelete(id string) error {
+ instance, err := v.GetFull(id)
+ if err != nil {
+ return pkgerrors.Wrap(err, "Error getting instance " + id + ", skip this instance. Error detail")
+ }
+ log.Printf("Instance " + id + ", status: " + instance.Status + ", HookProgress: " + instance.HookProgress)
+ //have to resolve again template for this instance because all templates are in /tmp -> will be deleted when container restarts
+ overrideValues := []string{}
+ if instance.Request.OverrideValues != nil {
+ for k, v := range instance.Request.OverrideValues {
+ overrideValues = append(overrideValues, k + "=" + v)
+ }
+ }
key := InstanceKey{
ID: id,
}
- err = db.DBconn.Delete(v.storeName, key, v.tagInst)
+ log.Printf(" Resolving template for release %s", instance.Request.ReleaseName)
+ _, hookList, _, err := rb.NewProfileClient().Resolve(instance.Request.RBName, instance.Request.RBVersion, instance.Request.ProfileName, overrideValues, instance.Request.ReleaseName)
+ instance.Hooks = hookList
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, instance)
+ if err != nil {
+ return pkgerrors.Wrap(err, "Update Instance DB Entry")
+ }
+
+ if strings.Contains(instance.Status, "FAILED"){
+ log.Printf(" This instance has failed during instantiation, not going to recover")
+ return nil
+ } else if !strings.Contains(instance.Status, "-INSTALL") && !strings.Contains(instance.Status, "-DELETE") {
+ log.Printf(" This instance is not in hook state, not going to recover")
+ return nil
+ }
+
+ splitHookProgress := strings.Split(instance.HookProgress,"/")
+ completedHooks,err := strconv.Atoi(splitHookProgress[0])
+ if err != nil {
+ return pkgerrors.Wrap(err, "Error getting completed PRE-INSTALL hooks for instance " + instance.ID + ", skip. Error detail")
+ }
+
+ //we can add an option to delete instances that will not be recovered from database to clean the db
+ if (instance.Status != "POST-INSTALL") && (instance.Status != "PRE-DELETE") && (instance.Status != "POST-DELETE") {
+ if instance.Status == "PRE-INSTALL" {
+ //Plugin quits during pre-install hooks -> Will do nothing because from SO point of view, there's no instance ID and will be reported as fail and be rolled back
+ log.Printf(" The plugin quits during pre-install hook of this instance, not going to recover")
+ }
+ return nil
+ }
+ k8sClient := KubernetesClient{}
+ err = k8sClient.Init(instance.Request.CloudRegion, id)
+ if err != nil {
+ log.Printf(" Error getting CloudRegion %s", instance.Request.CloudRegion)
+ return nil
+ }
+ hookClient := NewHookClient(instance.Namespace, id, v.storeName, v.tagInst)
+ switch instance.Status {
+ case "POST-INSTALL":
+ //Plugin quits during post-install hooks -> continue
+ go func() {
+ log.Printf(" The plugin quits during post-install hook of this instance, continue post-install hook")
+ err = hookClient.ExecHook(k8sClient, instance.Hooks, release.HookPostInstall, instance.PostInstallTimeout, completedHooks, &instance)
+ log.Printf("dbData.HookProgress %s", instance.HookProgress)
+ if err != nil {
+ instance.Status = "POST-INSTALL-FAILED"
+ log.Printf(" Instance: %s, Error running postinstall hooks error: %s", id, err)
+ } else {
+ instance.Status = "DONE"
+ }
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, instance)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", instance.ReleaseName)
+ }
+ }()
+ case "PRE-DELETE":
+ //Plugin quits during pre-delete hooks -> This already effects the instance -> should continue the deletion
+ go func() {
+ log.Printf(" The plugin quits during pre-delete hook of this instance, continue pre-delete hook")
+ err = hookClient.ExecHook(k8sClient, instance.Hooks, release.HookPreDelete, instance.PreDeleteTimeout, completedHooks, &instance)
+ if err != nil {
+ log.Printf(" Instance: %s, Error running pre-delete hooks error: %s", id, err)
+ instance.Status = "PRE-DELETE-FAILED"
+ err = db.DBconn.Update(v.storeName, key, v.tagInst, instance)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", instance.ReleaseName)
+ }
+ return
+ }
+
+ err = k8sClient.deleteResources(instance.Resources, instance.Namespace)
+ if err != nil {
+ log.Printf(" Error running deleting instance resources, error: %s", err)
+ return
+ }
+ //will not delete the instance in Db to avoid error when SO call delete again and there is not instance in DB
+ //the instance in DB will be deleted when SO call delete again.
+ instance.HookProgress = ""
+ if err := v.runPostDelete(k8sClient, hookClient, &instance, 0, false); err != nil {
+ log.Printf(err.Error())
+ }
+ }()
+ case "POST-DELETE":
+ //Plugin quits during post-delete hooks -> continue
+ go func() {
+ log.Printf(" The plugin quits during post-delete hook of this instance, continue post-delete hook")
+ if err := v.runPostDelete(k8sClient, hookClient, &instance, completedHooks, true); err != nil {
+ log.Printf(err.Error())
+ }
+ }()
+ default:
+ log.Printf(" This instance is not in hook state, not going to recover")
+ }
+
+ return nil
+}
+
+func (v *InstanceClient) runPostDelete(k8sClient KubernetesClient, hookClient *HookClient, instance *InstanceDbData, startIndex int, clearDb bool) error {
+ key := InstanceKey{
+ ID: instance.ID,
+ }
+ instance.Status = "POST-DELETE"
+ err := db.DBconn.Update(v.storeName, key, v.tagInst, instance)
if err != nil {
- return pkgerrors.Wrap(err, "Delete Instance")
+ log.Printf("Update Instance DB Entry for release %s has error.", instance.ReleaseName)
+ }
+ err = hookClient.ExecHook(k8sClient, instance.Hooks, release.HookPostDelete, instance.PostDeleteTimeout, startIndex, instance)
+ if err != nil {
+ //If this case happen, user should clean the cluster
+ log.Printf(" Instance: %s, Error running post-delete hooks error: %s", instance.ID, err)
+ instance.Status = "POST-DELETE-FAILED"
+ err2 := db.DBconn.Update(v.storeName, key, v.tagInst, instance)
+ if err2 != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", instance.ReleaseName)
+ return pkgerrors.Wrap(err2, "Delete Instance DB Entry")
+ }
+ return pkgerrors.Wrap(err, "Error running post-delete hooks")
}
+ if clearDb {
+ err = db.DBconn.Delete(v.storeName, key, v.tagInst)
+ if err != nil {
+ log.Printf("Delete Instance DB Entry for release %s has error.", instance.ReleaseName)
+ return pkgerrors.Wrap(err, "Delete Instance DB Entry")
+ }
+ } else {
+ instance.Status = "DELETED"
+ err := db.DBconn.Update(v.storeName, key, v.tagInst, instance)
+ if err != nil {
+ log.Printf("Update Instance DB Entry for release %s has error.", instance.ReleaseName)
+ return pkgerrors.Wrap(err, "Update Instance DB Entry")
+ }
+ }
+
+ go func() {
+ //Clear all hook rss that does not have delete-on-success deletion policy
+ log.Printf("Clean leftover hook resource")
+ var remainHookRss []helm.KubernetesResource
+ for _, h := range instance.Hooks {
+ res := helm.KubernetesResource{
+ GVK: h.KRT.GVK,
+ Name: h.Hook.Name,
+ }
+ if _, err := k8sClient.GetResourceStatus(res, hookClient.kubeNameSpace); err == nil {
+ remainHookRss = append(remainHookRss, res)
+ log.Printf(" Rss %s will be deleted.", res.Name)
+ }
+ }
+ if len(remainHookRss) > 0 {
+ err = k8sClient.deleteResources(remainHookRss, hookClient.kubeNameSpace)
+ if err != nil {
+ log.Printf("Error cleaning Hook Rss, please do it manually if needed. Error: %s", err.Error())
+ }
+ }
+ }()
return nil
}
/*
Copyright 2018 Intel Corporation.
+Copyright © 2021 Nokia Bell Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
import (
"encoding/base64"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
"io/ioutil"
"log"
"reflect"
"sort"
"testing"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/connection"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/db"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
}
})
}
+
+//TODO: add a test when pre-hook is failed (if possible)
+func TestInstanceWithHookCreate(t *testing.T) {
+ err := LoadMockPlugins(utils.LoadedPlugins)
+ if err != nil {
+ t.Fatalf("LoadMockPlugins returned an error (%s)", err)
+ }
+
+ // Load the mock kube config file into memory
+ fd, err := ioutil.ReadFile("../../mock_files/mock_configs/mock_kube_config")
+ if err != nil {
+ t.Fatal("Unable to read mock_kube_config")
+ }
+
+ t.Run("Successfully create Instance With Hook", func(t *testing.T) {
+ db.DBconn = &db.MockDB{
+ Items: map[string]map[string][]byte{
+ rb.ProfileKey{RBName: "test-rbdef-hook", RBVersion: "v1",
+ ProfileName: "profile1"}.String(): {
+ "profilemetadata": []byte(
+ "{\"profile-name\":\"profile1\"," +
+ "\"release-name\":\"testprofilereleasename\"," +
+ "\"namespace\":\"testnamespace\"," +
+ "\"rb-name\":\"test-rbdef\"," +
+ "\"rb-version\":\"v1\"," +
+ "\"kubernetesversion\":\"1.12.3\"}"),
+ // base64 encoding of vagrant/tests/vnfs/testrb/helm/profile
+ "profilecontent": []byte("H4sICCVd3FwAA3Byb2ZpbGUxLnRhcgDt1NEKgjAUxvFd7ylG98aWO" +
+ "sGXiYELxLRwJvj2rbyoIPDGiuD/uzmwM9iB7Vvruvrgw7CdXHsUn6Ejm2W3aopcP9eZL" +
+ "YRJM1voPN+ZndAm16kVSn9onheXMLheKeGqfdM0rq07/3bfUv9PJUkiR9+H+tSVajRym" +
+ "M6+lEqN7njxoVSbU+z2deX388r9nWzkr8fGSt5d79pnLOZfm0f+dRrzb7P4DZD/LyDJA" +
+ "AAAAAAAAAAAAAAA/+0Ksq1N5QAoAAA="),
+ },
+ rb.DefinitionKey{RBName: "test-rbdef-hook", RBVersion: "v1"}.String(): {
+ "defmetadata": []byte(
+ "{\"rb-name\":\"test-rbdef-hook\"," +
+ "\"rb-version\":\"v1\"," +
+ "\"chart-name\":\"test\"," +
+ "\"description\":\"testresourcebundle\"}"),
+ // base64 encoding of test helm package with hooks inside
+ "defcontent": []byte("H4sICE+Q8WAAA3Rlc3QudGFyAO1aW2+jOBTOM7/CYl6HlEsIq7xV24" +
+ "fVqluNdlYjrVajkQMnhS1gFjvZjbr972MDJYTQwGhMMmn9qVUaYx/o8TnfuRgGlF1NxoX" +
+ "J4Xmu+LQ812x+PmNiOXzEMe3ZfD4xLdO23QlyR36uAmvKcI7QhIXs6Ly+6xcKJvZ/g+M1" +
+ "0OkWJ/EY9xAbPJ/PXtx/m9tGtf+WOePjlu143gSZYzxMG298/9+hG1jhdcxQaQRoRXKU5" +
+ "WBEKVdMHEM+1d6hP8KIIv6D0Z/Xv90afE6CGYMAraIYxIQb8GOcAxeSR3gZczmMoCWgDF" +
+ "PKp0Up/8pCQAySLMbc6KYaDpIoXWgIhYQ8fAkgBgZfMhJH/naBdDFo0LXvAwQQvOey+E3" +
+ "BKIb9HDCLSKqfW3mvAIX//xzinI3m/r3+b7nzZ/83Z57gf9tyHeX/pwDOok+QU+5NC7Sx" +
+ "NJxl9VfdmppTU9cCoH4eZawYvEa/QJwgX1hMwRXCgKL0HiWcQyI/JutAS3ECi+KCtnkWV" +
+ "sjSzv3fKrRR+H/NyuNkgoPyv5npzRzxOxP+b9uOyv9Ogdb+BxgSklKQGg36+N+zZ7v9tw" +
+ "X/u3xM8f8p0OR/Tv70igeBhygNFuimMIWPwLQEGA4wwyJZK7n98RFNf+cZG6YwveMj6On" +
+ "JqE2nmkUz7POp+uPj3tRi+OlJ57NivISYCqlI3LtPLM3AF5Mpn+EzkpcLeSLqh7cNSYNk" +
+ "oToTraQ0/kWBeE/gQJH80apHFPBJynCUcuU+jxiV9uortfgowfdCV8s13S7Jf3p9gbKAJ" +
+ "8mI5WuoxxjbtkZ8kiRY7NlfOg31z9+y/y3/zwhlRpmLG3+TpRwW6PF/25l7Vf5nWZaIE9" +
+ "ac/6H8/xRo+v9SuNKOAH4ly4Gu37IaSy4DdEjHaUpYUQNWi/WQZ6VTGl6JAlFfoMaaw+v" +
+ "GvxDdh4xP042f9I7r1c3KYlQvn+pT2SMpqtbpYcmK/kf/rAkTD1wT1RL7D2S1uo2SiC2Q" +
+ "I490OjSyzz2Up+fwISc+UHq324kGaeQg7J59qOrtO9jUdHRIXDvqojFAZhwS2BEK26cns" +
+ "V5/z2sLU/+sGYahjWGA9qgGaMs0QPMV2J89tv31Wd+LttdlebawvHPT7g+DdvzPQXr474" +
+ "//7i7+25Yt8n/PVPH/JJBDv3tWIzv8HwjvJ996yWsM/gf6eOOxf08fskP/gXBZxneZgf9" +
+ "AHSruXzZa8Z9Cvol8kHsW1Nf/K+r/sv83dx3R/5u5rjr/PQla5z8l+X4srWAgAVc2I7nt" +
+ "B1lMtgmku75fRnJWLTOKLwtkces56AgOkXlutf8waPf/axVJpIDe/r9jtc5/XNszlf+fA" +
+ "kf6/ztvGXgAsFswNhV8xxFA8yFlnQE0ZV7YIUBH/V+9+XOy/v/M9qxd/PfMsv/vKv8/BY" +
+ "7F/2vfJ+vB7k9xUaJwC6oMaKh/dy0cVGXtph+p8d0R6iyptWvD3UbonLSky9PrxfZOWhp" +
+ "RzZOGQkbonrSkSzPAi+2ftBRyYQ2UtuV9Z87YVMhY+eOL95Bmi9YQW9Q7X2GWkNLuP6V8" +
+ "Sx2Q1B5La48yXFdq25XcHqS3qoKXg673f2QXAL3nf17j/M8U539zx1T5/0kg7/WLEfPYD" +
+ "vHDXsB4xZlsh07eeCrb0sgYLwF9czI71AgvM5vtUMmFpbPnpl8FBQUFBQUFBQUFBQUFBQ" +
+ "UFBQUFhdHwFf2f+3IAUAAA"),
+ },
+ connection.ConnectionKey{CloudRegion: "mock_connection"}.String(): {
+ "metadata": []byte(
+ "{\"cloud-region\":\"mock_connection\"," +
+ "\"cloud-owner\":\"mock_owner\"," +
+ "\"kubeconfig\": \"" + base64.StdEncoding.EncodeToString(fd) + "\"}"),
+ },
+ },
+ }
+
+ ic := NewInstanceClient()
+ input := InstanceRequest{
+ RBName: "test-rbdef-hook",
+ RBVersion: "v1",
+ ProfileName: "profile1",
+ CloudRegion: "mock_connection",
+ }
+
+ ir, err := ic.Create(input)
+ if err != nil {
+ t.Fatalf("TestInstanceWithHookCreate returned an error (%s)", err)
+ }
+
+ log.Println(ir)
+
+ if len(ir.Resources) == 0 {
+ t.Fatalf("TestInstanceWithHookCreate returned empty data (%+v)", ir)
+ }
+ })
+}
--- /dev/null
+/*
+ * Copyright 2018 Intel Corporation, Inc
+ * Copyright © 2021 Samsung Electronics
+ * Copyright © 2021 Orange
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package app
+
+import (
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+
+ pkgerrors "github.com/pkg/errors"
+)
+
+// QueryStatus is what is returned when status is queried for an instance
+type QueryStatus struct {
+ ResourceCount int32 `json:"resourceCount"`
+ ResourcesStatus []ResourceStatus `json:"resourcesStatus"`
+}
+
+// QueryManager is an interface exposes the instantiation functionality
+type QueryManager interface {
+ Query(namespace, cloudRegion, apiVersion, kind, name, labels, id string) (QueryStatus, error)
+}
+
+// QueryClient implements the InstanceManager interface
+// It will also be used to maintain some localized state
+type QueryClient struct {
+ storeName string
+ tagInst string
+}
+
+// NewQueryClient returns an instance of the QueryClient
+// which implements the InstanceManager
+func NewQueryClient() *QueryClient {
+ return &QueryClient{
+ storeName: "rbdef",
+ tagInst: "instance",
+ }
+}
+
+// Query returns state of instance's filtered resources
+func (v *QueryClient) Query(namespace, cloudRegion, apiVersion, kind, name, labels, id string) (QueryStatus, error) {
+
+ //Read the status from the DD
+
+ k8sClient := KubernetesClient{}
+ err := k8sClient.Init(cloudRegion, id)
+ if err != nil {
+ return QueryStatus{}, pkgerrors.Wrap(err, "Getting CloudRegion Information")
+ }
+
+ var resourcesStatus []ResourceStatus
+ if labels != "" {
+ resList, err := k8sClient.queryResources(apiVersion, kind, labels, namespace)
+ if err != nil {
+ return QueryStatus{}, pkgerrors.Wrap(err, "Querying Resources")
+ }
+ // If user specifies both label and name, we want to pick up only single resource from these matching label
+ if name != "" {
+ //Assigning 0-length, because we may actually not find matching name
+ resourcesStatus = make([]ResourceStatus, 0)
+ for _, res := range resList {
+ if res.Name == name {
+ resourcesStatus = append(resourcesStatus, res)
+ break
+ }
+ }
+ } else {
+ resourcesStatus = resList
+ }
+ } else if name != "" {
+ resIdentifier := helm.KubernetesResource{
+ Name: name,
+ GVK: schema.FromAPIVersionAndKind(apiVersion, kind),
+ }
+ res, err := k8sClient.GetResourceStatus(resIdentifier, namespace)
+ if err != nil {
+ return QueryStatus{}, pkgerrors.Wrap(err, "Querying Resource")
+ }
+ resourcesStatus = []ResourceStatus{res}
+ } else {
+ resList, err := k8sClient.queryResources(apiVersion, kind, labels, namespace)
+ if err != nil {
+ return QueryStatus{}, pkgerrors.Wrap(err, "Querying Resources")
+ }
+ resourcesStatus = resList
+ }
+
+ resp := QueryStatus{
+ ResourceCount: int32(len(resourcesStatus)),
+ ResourcesStatus: resourcesStatus,
+ }
+ return resp, nil
+}
// EtcdStore Interface needed for mocking
type EtcdStore interface {
Get(key string) ([]byte, error)
+ GetAll(key string) ([][]byte, error)
Put(key, value string) error
Delete(key string) error
}
}
getResp, err := e.cli.Get(context.Background(), key)
if err != nil {
- return nil, pkgerrors.Errorf("Error getitng etcd entry: %s", err.Error())
+ return nil, pkgerrors.Errorf("Error getting etcd entry: %s", err.Error())
}
if getResp.Count == 0 {
return nil, pkgerrors.Errorf("Key doesn't exist")
return getResp.Kvs[0].Value, nil
}
+// GetAll sub values from Etcd DB
+func (e EtcdClient) GetAll(key string) ([][]byte, error) {
+ if e.cli == nil {
+ return nil, pkgerrors.Errorf("Etcd Client not initialized")
+ }
+ getResp, err := e.cli.Get(context.Background(), key, clientv3.WithPrefix())
+ if err != nil {
+ return nil, pkgerrors.Errorf("Error getting etcd entry: %s", err.Error())
+ }
+ result := make([][]byte, 0)
+ for _, v := range getResp.Kvs {
+ result = append(result, v.Value)
+ }
+ return result, nil
+}
+
// Delete values from Etcd DB
func (e EtcdClient) Delete(key string) error {
package db
import (
+ "strings"
+
pkgerrors "github.com/pkg/errors"
)
return nil, pkgerrors.Errorf("Key doesn't exist")
}
+func (c *MockEtcdClient) GetAll(key string) ([][]byte, error) {
+ result := make([][]byte, 0)
+ for kvKey, kvValue := range c.Items {
+ if strings.HasPrefix(kvKey, key) {
+ result = append(result, []byte(kvValue))
+ }
+ }
+ return result, nil
+}
+
func (c *MockEtcdClient) Delete(key string) error {
delete(c.Items, key)
return c.Err
import (
"fmt"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
-
pkgerrors "github.com/pkg/errors"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
/*
* Copyright 2019 Intel Corporation, Inc
+ * Copyright © 2021 Nokia Bell Labs.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
package plugin
import (
+ "k8s.io/client-go/rest"
"log"
"strings"
+ "time"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/config"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
pkgerrors "github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
//Update kubernetes resource based on the groupVersionKind and resourceName provided in resource
Update(yamlFilePath string, namespace string, client KubernetesConnector) (string, error)
+
+
+ //WatchUntilReady a kubernetes resource until it's ready
+ WatchUntilReady(timeout time.Duration,
+ ns string,
+ res helm.KubernetesResource,
+ mapper meta.RESTMapper,
+ restClient rest.Interface,
+ objType runtime.Object,
+ clientSet kubernetes.Interface) error
}
// GetPluginByKind returns a plugin by the kind name
package plugin
import (
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
"testing"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/config"
)
import (
"archive/tar"
"compress/gzip"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
pkgerrors "github.com/pkg/errors"
"io"
"io/ioutil"
"os"
"path/filepath"
-
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
)
func isTarGz(r io.Reader) error {
--- /dev/null
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statuscheck // import "helm.sh/helm/v3/pkg/kube"
+
+import (
+ "sync"
+
+ apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+ apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/cli-runtime/pkg/resource"
+ "k8s.io/client-go/kubernetes/scheme"
+)
+
+var k8sNativeScheme *runtime.Scheme
+var k8sNativeSchemeOnce sync.Once
+
+// AsVersioned converts the given info into a runtime.Object with the correct
+// group and version set
+func AsVersioned(info *resource.Info) runtime.Object {
+ return convertWithMapper(info.Object, info.Mapping)
+}
+
+// convertWithMapper converts the given object with the optional provided
+// RESTMapping. If no mapping is provided, the default schema versioner is used
+func convertWithMapper(obj runtime.Object, mapping *meta.RESTMapping) runtime.Object {
+ s := kubernetesNativeScheme()
+ var gv = runtime.GroupVersioner(schema.GroupVersions(s.PrioritizedVersionsAllGroups()))
+ if mapping != nil {
+ gv = mapping.GroupVersionKind.GroupVersion()
+ }
+ if obj, err := runtime.ObjectConvertor(s).ConvertToVersion(obj, gv); err == nil {
+ return obj
+ }
+ return obj
+}
+
+// kubernetesNativeScheme returns a clean *runtime.Scheme with _only_ Kubernetes
+// native resources added to it. This is required to break free of custom resources
+// that may have been added to scheme.Scheme due to Helm being used as a package in
+// combination with e.g. a versioned kube client. If we would not do this, the client
+// may attempt to perform e.g. a 3-way-merge strategy patch for custom resources.
+func kubernetesNativeScheme() *runtime.Scheme {
+ k8sNativeSchemeOnce.Do(func() {
+ k8sNativeScheme = runtime.NewScheme()
+ scheme.AddToScheme(k8sNativeScheme)
+ // API extensions are not in the above scheme set,
+ // and must thus be added separately.
+ apiextensionsv1beta1.AddToScheme(k8sNativeScheme)
+ apiextensionsv1.AddToScheme(k8sNativeScheme)
+ })
+ return k8sNativeScheme
+}
--- /dev/null
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statuscheck // import "helm.sh/helm/v3/pkg/kube"
+
+import (
+ "context"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
+ appsv1 "k8s.io/api/apps/v1"
+ appsv1beta1 "k8s.io/api/apps/v1beta1"
+ appsv1beta2 "k8s.io/api/apps/v1beta2"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+ apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+ apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/util/intstr"
+ "k8s.io/cli-runtime/pkg/resource"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/kubernetes/scheme"
+ "log"
+)
+
+// ReadyCheckerOption is a function that configures a ReadyChecker.
+type ReadyCheckerOption func(*ReadyChecker)
+
+// PausedAsReady returns a ReadyCheckerOption that configures a ReadyChecker
+// to consider paused resources to be ready. For example a Deployment
+// with spec.paused equal to true would be considered ready.
+func PausedAsReady(pausedAsReady bool) ReadyCheckerOption {
+ return func(c *ReadyChecker) {
+ c.pausedAsReady = pausedAsReady
+ }
+}
+
+// CheckJobs returns a ReadyCheckerOption that configures a ReadyChecker
+// to consider readiness of Job resources.
+func CheckJobs(checkJobs bool) ReadyCheckerOption {
+ return func(c *ReadyChecker) {
+ c.checkJobs = checkJobs
+ }
+}
+
+// NewReadyChecker creates a new checker. Passed ReadyCheckerOptions can
+// be used to override defaults.
+func NewReadyChecker(cl kubernetes.Interface, opts ...ReadyCheckerOption) ReadyChecker {
+ c := ReadyChecker{
+ client: cl,
+ }
+
+ for _, opt := range opts {
+ opt(&c)
+ }
+
+ return c
+}
+
+// ReadyChecker is a type that can check core Kubernetes types for readiness.
+type ReadyChecker struct {
+ client kubernetes.Interface
+ checkJobs bool
+ pausedAsReady bool
+}
+
+// IsReady checks if v is ready. It supports checking readiness for pods,
+// deployments, persistent volume claims, services, daemon sets, custom
+// resource definitions, stateful sets, replication controllers, and replica
+// sets. All other resource kinds are always considered ready.
+//
+// IsReady will fetch the latest state of the object from the server prior to
+// performing readiness checks, and it will return any error encountered.
+func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, error) {
+ var (
+ // This defaults to true, otherwise we get to a point where
+ // things will always return false unless one of the objects
+ // that manages pods has been hit
+ ok = true
+ err error
+ )
+ switch value := AsVersioned(v).(type) {
+ case *corev1.Pod:
+ pod, err := c.client.CoreV1().Pods(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil || !c.isPodReady(pod) {
+ return false, err
+ }
+ case *batchv1.Job:
+ if c.checkJobs {
+ job, err := c.client.BatchV1().Jobs(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil || !c.jobReady(job) {
+ return false, err
+ }
+ }
+ case *appsv1.Deployment, *appsv1beta1.Deployment, *appsv1beta2.Deployment, *extensionsv1beta1.Deployment:
+ currentDeployment, err := c.client.AppsV1().Deployments(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+ // If paused deployment will never be ready
+ if currentDeployment.Spec.Paused {
+ return c.pausedAsReady, nil
+ }
+ // Find RS associated with deployment
+ newReplicaSet, err := utils.GetNewReplicaSet(currentDeployment, c.client.AppsV1())
+ if err != nil || newReplicaSet == nil {
+ return false, err
+ }
+ if !c.deploymentReady(newReplicaSet, currentDeployment) {
+ return false, nil
+ }
+ case *corev1.PersistentVolumeClaim:
+ claim, err := c.client.CoreV1().PersistentVolumeClaims(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+ if !c.volumeReady(claim) {
+ return false, nil
+ }
+ case *corev1.Service:
+ svc, err := c.client.CoreV1().Services(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+ if !c.serviceReady(svc) {
+ return false, nil
+ }
+ case *extensionsv1beta1.DaemonSet, *appsv1.DaemonSet, *appsv1beta2.DaemonSet:
+ ds, err := c.client.AppsV1().DaemonSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+ if !c.daemonSetReady(ds) {
+ return false, nil
+ }
+ case *apiextv1beta1.CustomResourceDefinition:
+ if err := v.Get(); err != nil {
+ return false, err
+ }
+ crd := &apiextv1beta1.CustomResourceDefinition{}
+ if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
+ return false, err
+ }
+ if !c.crdBetaReady(*crd) {
+ return false, nil
+ }
+ case *apiextv1.CustomResourceDefinition:
+ if err := v.Get(); err != nil {
+ return false, err
+ }
+ crd := &apiextv1.CustomResourceDefinition{}
+ if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
+ return false, err
+ }
+ if !c.crdReady(*crd) {
+ return false, nil
+ }
+ case *appsv1.StatefulSet, *appsv1beta1.StatefulSet, *appsv1beta2.StatefulSet:
+ sts, err := c.client.AppsV1().StatefulSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+ if !c.statefulSetReady(sts) {
+ return false, nil
+ }
+ case *corev1.ReplicationController, *extensionsv1beta1.ReplicaSet, *appsv1beta2.ReplicaSet, *appsv1.ReplicaSet:
+ ok, err = c.podsReadyForObject(ctx, v.Namespace, value)
+ }
+ if !ok || err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
+func (c *ReadyChecker) podsReadyForObject(ctx context.Context, namespace string, obj runtime.Object) (bool, error) {
+ pods, err := c.podsforObject(ctx, namespace, obj)
+ if err != nil {
+ return false, err
+ }
+ for _, pod := range pods {
+ if !c.isPodReady(&pod) {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+func (c *ReadyChecker) podsforObject(ctx context.Context, namespace string, obj runtime.Object) ([]corev1.Pod, error) {
+ selector, err := SelectorsForObject(obj)
+ if err != nil {
+ return nil, err
+ }
+ list, err := getPods(ctx, c.client, namespace, selector.String())
+ return list, err
+}
+
+// isPodReady returns true if a pod is ready; false otherwise.
+func (c *ReadyChecker) isPodReady(pod *corev1.Pod) bool {
+ for _, c := range pod.Status.Conditions {
+ if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
+ return true
+ }
+ }
+ log.Printf("Pod is not ready: %s/%s", pod.GetNamespace(), pod.GetName())
+ return false
+}
+
+func (c *ReadyChecker) jobReady(job *batchv1.Job) bool {
+ if job.Status.Failed > *job.Spec.BackoffLimit {
+ log.Printf("Job is failed: %s/%s", job.GetNamespace(), job.GetName())
+ return false
+ }
+ if job.Status.Succeeded < *job.Spec.Completions {
+ log.Printf("Job is not completed: %s/%s", job.GetNamespace(), job.GetName())
+ return false
+ }
+ return true
+}
+
+func (c *ReadyChecker) serviceReady(s *corev1.Service) bool {
+ // ExternalName Services are external to cluster so helm shouldn't be checking to see if they're 'ready' (i.e. have an IP Set)
+ if s.Spec.Type == corev1.ServiceTypeExternalName {
+ return true
+ }
+
+ // Ensure that the service cluster IP is not empty
+ if s.Spec.ClusterIP == "" {
+ log.Printf("Service does not have cluster IP address: %s/%s", s.GetNamespace(), s.GetName())
+ return false
+ }
+
+ // This checks if the service has a LoadBalancer and that balancer has an Ingress defined
+ if s.Spec.Type == corev1.ServiceTypeLoadBalancer {
+ // do not wait when at least 1 external IP is set
+ if len(s.Spec.ExternalIPs) > 0 {
+ log.Printf("Service %s/%s has external IP addresses (%v), marking as ready", s.GetNamespace(), s.GetName(), s.Spec.ExternalIPs)
+ return true
+ }
+
+ if s.Status.LoadBalancer.Ingress == nil {
+ log.Printf("Service does not have load balancer ingress IP address: %s/%s", s.GetNamespace(), s.GetName())
+ return false
+ }
+ }
+
+ return true
+}
+
+func (c *ReadyChecker) volumeReady(v *corev1.PersistentVolumeClaim) bool {
+ if v.Status.Phase != corev1.ClaimBound {
+ log.Printf("PersistentVolumeClaim is not bound: %s/%s", v.GetNamespace(), v.GetName())
+ return false
+ }
+ return true
+}
+
+func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool {
+ expectedReady := *dep.Spec.Replicas - utils.MaxUnavailable(*dep)
+ if !(rs.Status.ReadyReplicas >= expectedReady) {
+ log.Printf("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", dep.Namespace, dep.Name, rs.Status.ReadyReplicas, expectedReady)
+ return false
+ }
+ return true
+}
+
+func (c *ReadyChecker) daemonSetReady(ds *appsv1.DaemonSet) bool {
+ // If the update strategy is not a rolling update, there will be nothing to wait for
+ if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType {
+ return true
+ }
+
+ // Make sure all the updated pods have been scheduled
+ if ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled {
+ log.Printf("DaemonSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", ds.Namespace, ds.Name, ds.Status.UpdatedNumberScheduled, ds.Status.DesiredNumberScheduled)
+ return false
+ }
+ maxUnavailable, err := intstr.GetValueFromIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, int(ds.Status.DesiredNumberScheduled), true)
+ if err != nil {
+ // If for some reason the value is invalid, set max unavailable to the
+ // number of desired replicas. This is the same behavior as the
+ // `MaxUnavailable` function in deploymentutil
+ maxUnavailable = int(ds.Status.DesiredNumberScheduled)
+ }
+
+ expectedReady := int(ds.Status.DesiredNumberScheduled) - maxUnavailable
+ if !(int(ds.Status.NumberReady) >= expectedReady) {
+ log.Printf("DaemonSet is not ready: %s/%s. %d out of %d expected pods are ready", ds.Namespace, ds.Name, ds.Status.NumberReady, expectedReady)
+ return false
+ }
+ return true
+}
+
+// Because the v1 extensions API is not available on all supported k8s versions
+// yet and because Go doesn't support generics, we need to have a duplicate
+// function to support the v1beta1 types
+func (c *ReadyChecker) crdBetaReady(crd apiextv1beta1.CustomResourceDefinition) bool {
+ for _, cond := range crd.Status.Conditions {
+ switch cond.Type {
+ case apiextv1beta1.Established:
+ if cond.Status == apiextv1beta1.ConditionTrue {
+ return true
+ }
+ case apiextv1beta1.NamesAccepted:
+ if cond.Status == apiextv1beta1.ConditionFalse {
+ // This indicates a naming conflict, but it's probably not the
+ // job of this function to fail because of that. Instead,
+ // we treat it as a success, since the process should be able to
+ // continue.
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool {
+ for _, cond := range crd.Status.Conditions {
+ switch cond.Type {
+ case apiextv1.Established:
+ if cond.Status == apiextv1.ConditionTrue {
+ return true
+ }
+ case apiextv1.NamesAccepted:
+ if cond.Status == apiextv1.ConditionFalse {
+ // This indicates a naming conflict, but it's probably not the
+ // job of this function to fail because of that. Instead,
+ // we treat it as a success, since the process should be able to
+ // continue.
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool {
+ // If the update strategy is not a rolling update, there will be nothing to wait for
+ if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
+ return true
+ }
+
+ // Dereference all the pointers because StatefulSets like them
+ var partition int
+ // 1 is the default for replicas if not set
+ var replicas = 1
+ // For some reason, even if the update strategy is a rolling update, the
+ // actual rollingUpdate field can be nil. If it is, we can safely assume
+ // there is no partition value
+ if sts.Spec.UpdateStrategy.RollingUpdate != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
+ partition = int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition)
+ }
+ if sts.Spec.Replicas != nil {
+ replicas = int(*sts.Spec.Replicas)
+ }
+
+ // Because an update strategy can use partitioning, we need to calculate the
+ // number of updated replicas we should have. For example, if the replicas
+ // is set to 3 and the partition is 2, we'd expect only one pod to be
+ // updated
+ expectedReplicas := replicas - partition
+
+ // Make sure all the updated pods have been scheduled
+ if int(sts.Status.UpdatedReplicas) != expectedReplicas {
+ log.Printf("StatefulSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", sts.Namespace, sts.Name, sts.Status.UpdatedReplicas, expectedReplicas)
+ return false
+ }
+
+ if int(sts.Status.ReadyReplicas) != replicas {
+ log.Printf("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready", sts.Namespace, sts.Name, sts.Status.ReadyReplicas, replicas)
+ return false
+ }
+ return true
+}
+
+func getPods(ctx context.Context, client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) {
+ list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+ LabelSelector: selector,
+ })
+ return list.Items, err
+}
--- /dev/null
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statuscheck // import "helm.sh/helm/v3/pkg/kube"
+
+import (
+ "context"
+ "testing"
+
+ appsv1 "k8s.io/api/apps/v1"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/util/intstr"
+ "k8s.io/client-go/kubernetes/fake"
+)
+
+const defaultNamespace = metav1.NamespaceDefault
+
+func Test_ReadyChecker_deploymentReady(t *testing.T) {
+ type args struct {
+ rs *appsv1.ReplicaSet
+ dep *appsv1.Deployment
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ }{
+ {
+ name: "deployment is ready",
+ args: args{
+ rs: newReplicaSet("foo", 1, 1),
+ dep: newDeployment("foo", 1, 1, 0),
+ },
+ want: true,
+ },
+ {
+ name: "deployment is not ready",
+ args: args{
+ rs: newReplicaSet("foo", 0, 0),
+ dep: newDeployment("foo", 1, 1, 0),
+ },
+ want: false,
+ },
+ {
+ name: "deployment is ready when maxUnavailable is set",
+ args: args{
+ rs: newReplicaSet("foo", 2, 1),
+ dep: newDeployment("foo", 2, 1, 1),
+ },
+ want: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ c := NewReadyChecker(fake.NewSimpleClientset(), PausedAsReady(false), CheckJobs(false))
+ if got := c.deploymentReady(tt.args.rs, tt.args.dep); got != tt.want {
+ t.Errorf("deploymentReady() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func Test_ReadyChecker_daemonSetReady(t *testing.T) {
+ type args struct {
+ ds *appsv1.DaemonSet
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ }{
+ {
+ name: "daemonset is ready",
+ args: args{
+ ds: newDaemonSet("foo", 0, 1, 1, 1),
+ },
+ want: true,
+ },
+ {
+ name: "daemonset is not ready",
+ args: args{
+ ds: newDaemonSet("foo", 0, 0, 1, 1),
+ },
+ want: false,
+ },
+ {
+ name: "daemonset pods have not been scheduled successfully",
+ args: args{
+ ds: newDaemonSet("foo", 0, 0, 1, 0),
+ },
+ want: false,
+ },
+ {
+ name: "daemonset is ready when maxUnavailable is set",
+ args: args{
+ ds: newDaemonSet("foo", 1, 1, 2, 2),
+ },
+ want: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ c := NewReadyChecker(fake.NewSimpleClientset(), PausedAsReady(false), CheckJobs(false))
+ if got := c.daemonSetReady(tt.args.ds); got != tt.want {
+ t.Errorf("daemonSetReady() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func Test_ReadyChecker_statefulSetReady(t *testing.T) {
+ type args struct {
+ sts *appsv1.StatefulSet
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ }{
+ {
+ name: "statefulset is ready",
+ args: args{
+ sts: newStatefulSet("foo", 1, 0, 1, 1),
+ },
+ want: true,
+ },
+ {
+ name: "statefulset is not ready",
+ args: args{
+ sts: newStatefulSet("foo", 1, 0, 0, 1),
+ },
+ want: false,
+ },
+ {
+ name: "statefulset is ready when partition is specified",
+ args: args{
+ sts: newStatefulSet("foo", 2, 1, 2, 1),
+ },
+ want: true,
+ },
+ {
+ name: "statefulset is not ready when partition is set",
+ args: args{
+ sts: newStatefulSet("foo", 1, 1, 1, 1),
+ },
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ c := NewReadyChecker(fake.NewSimpleClientset(), PausedAsReady(false), CheckJobs(false))
+ if got := c.statefulSetReady(tt.args.sts); got != tt.want {
+ t.Errorf("statefulSetReady() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func Test_ReadyChecker_podsReadyForObject(t *testing.T) {
+ type args struct {
+ namespace string
+ obj runtime.Object
+ }
+ tests := []struct {
+ name string
+ args args
+ existPods []corev1.Pod
+ want bool
+ wantErr bool
+ }{
+ {
+ name: "pods ready for a replicaset",
+ args: args{
+ namespace: defaultNamespace,
+ obj: newReplicaSet("foo", 1, 1),
+ },
+ existPods: []corev1.Pod{
+ *newPodWithCondition("foo", corev1.ConditionTrue),
+ },
+ want: true,
+ wantErr: false,
+ },
+ {
+ name: "pods not ready for a replicaset",
+ args: args{
+ namespace: defaultNamespace,
+ obj: newReplicaSet("foo", 1, 1),
+ },
+ existPods: []corev1.Pod{
+ *newPodWithCondition("foo", corev1.ConditionFalse),
+ },
+ want: false,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ c := NewReadyChecker(fake.NewSimpleClientset(), PausedAsReady(false), CheckJobs(false))
+ for _, pod := range tt.existPods {
+ if _, err := c.client.CoreV1().Pods(defaultNamespace).Create(context.TODO(), &pod, metav1.CreateOptions{}); err != nil {
+ t.Errorf("Failed to create Pod error: %v", err)
+ return
+ }
+ }
+ got, err := c.podsReadyForObject(context.TODO(), tt.args.namespace, tt.args.obj)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("podsReadyForObject() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("podsReadyForObject() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func Test_ReadyChecker_jobReady(t *testing.T) {
+ type args struct {
+ job *batchv1.Job
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ }{
+ {
+ name: "job is completed",
+ args: args{job: newJob("foo", 1, 1, 1, 0)},
+ want: true,
+ },
+ {
+ name: "job is incomplete",
+ args: args{job: newJob("foo", 1, 1, 0, 0)},
+ want: false,
+ },
+ {
+ name: "job is failed",
+ args: args{job: newJob("foo", 1, 1, 0, 1)},
+ want: false,
+ },
+ {
+ name: "job is completed with retry",
+ args: args{job: newJob("foo", 1, 1, 1, 1)},
+ want: true,
+ },
+ {
+ name: "job is failed with retry",
+ args: args{job: newJob("foo", 1, 1, 0, 2)},
+ want: false,
+ },
+ {
+ name: "job is completed single run",
+ args: args{job: newJob("foo", 0, 1, 1, 0)},
+ want: true,
+ },
+ {
+ name: "job is failed single run",
+ args: args{job: newJob("foo", 0, 1, 0, 1)},
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ c := NewReadyChecker(fake.NewSimpleClientset(), PausedAsReady(false), CheckJobs(false))
+ if got := c.jobReady(tt.args.job); got != tt.want {
+ t.Errorf("jobReady() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func Test_ReadyChecker_volumeReady(t *testing.T) {
+ type args struct {
+ v *corev1.PersistentVolumeClaim
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ }{
+ {
+ name: "pvc is bound",
+ args: args{
+ v: newPersistentVolumeClaim("foo", corev1.ClaimBound),
+ },
+ want: true,
+ },
+ {
+ name: "pvc is not ready",
+ args: args{
+ v: newPersistentVolumeClaim("foo", corev1.ClaimPending),
+ },
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ c := NewReadyChecker(fake.NewSimpleClientset(), PausedAsReady(false), CheckJobs(false))
+ if got := c.volumeReady(tt.args.v); got != tt.want {
+ t.Errorf("volumeReady() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func newDaemonSet(name string, maxUnavailable, numberReady, desiredNumberScheduled, updatedNumberScheduled int) *appsv1.DaemonSet {
+ return &appsv1.DaemonSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ },
+ Spec: appsv1.DaemonSetSpec{
+ UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
+ Type: appsv1.RollingUpdateDaemonSetStrategyType,
+ RollingUpdate: &appsv1.RollingUpdateDaemonSet{
+ MaxUnavailable: func() *intstr.IntOrString { i := intstr.FromInt(maxUnavailable); return &i }(),
+ },
+ },
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"name": name}},
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{"name": name},
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Image: "nginx",
+ },
+ },
+ },
+ },
+ },
+ Status: appsv1.DaemonSetStatus{
+ DesiredNumberScheduled: int32(desiredNumberScheduled),
+ NumberReady: int32(numberReady),
+ UpdatedNumberScheduled: int32(updatedNumberScheduled),
+ },
+ }
+}
+
+func newStatefulSet(name string, replicas, partition, readyReplicas, updatedReplicas int) *appsv1.StatefulSet {
+ return &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ },
+ Spec: appsv1.StatefulSetSpec{
+ UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
+ Type: appsv1.RollingUpdateStatefulSetStrategyType,
+ RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
+ Partition: intToInt32(partition),
+ },
+ },
+ Replicas: intToInt32(replicas),
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"name": name}},
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{"name": name},
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Image: "nginx",
+ },
+ },
+ },
+ },
+ },
+ Status: appsv1.StatefulSetStatus{
+ UpdatedReplicas: int32(updatedReplicas),
+ ReadyReplicas: int32(readyReplicas),
+ },
+ }
+}
+
+func newDeployment(name string, replicas, maxSurge, maxUnavailable int) *appsv1.Deployment {
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ },
+ Spec: appsv1.DeploymentSpec{
+ Strategy: appsv1.DeploymentStrategy{
+ Type: appsv1.RollingUpdateDeploymentStrategyType,
+ RollingUpdate: &appsv1.RollingUpdateDeployment{
+ MaxUnavailable: func() *intstr.IntOrString { i := intstr.FromInt(maxUnavailable); return &i }(),
+ MaxSurge: func() *intstr.IntOrString { i := intstr.FromInt(maxSurge); return &i }(),
+ },
+ },
+ Replicas: intToInt32(replicas),
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"name": name}},
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{"name": name},
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Image: "nginx",
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func newReplicaSet(name string, replicas int, readyReplicas int) *appsv1.ReplicaSet {
+ d := newDeployment(name, replicas, 0, 0)
+ return &appsv1.ReplicaSet{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ Labels: d.Spec.Selector.MatchLabels,
+ OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(d, d.GroupVersionKind())},
+ },
+ Spec: appsv1.ReplicaSetSpec{
+ Selector: d.Spec.Selector,
+ Replicas: intToInt32(replicas),
+ Template: d.Spec.Template,
+ },
+ Status: appsv1.ReplicaSetStatus{
+ ReadyReplicas: int32(readyReplicas),
+ },
+ }
+}
+
+func newPodWithCondition(name string, podReadyCondition corev1.ConditionStatus) *corev1.Pod {
+ return &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ Labels: map[string]string{"name": name},
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Image: "nginx",
+ },
+ },
+ },
+ Status: corev1.PodStatus{
+ Conditions: []corev1.PodCondition{
+ {
+ Type: corev1.PodReady,
+ Status: podReadyCondition,
+ },
+ },
+ },
+ }
+}
+
+func newPersistentVolumeClaim(name string, phase corev1.PersistentVolumeClaimPhase) *corev1.PersistentVolumeClaim {
+ return &corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ },
+ Status: corev1.PersistentVolumeClaimStatus{
+ Phase: phase,
+ },
+ }
+}
+
+func newJob(name string, backoffLimit, completions, succeeded, failed int) *batchv1.Job {
+ return &batchv1.Job{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: defaultNamespace,
+ },
+ Spec: batchv1.JobSpec{
+ BackoffLimit: intToInt32(backoffLimit),
+ Completions: intToInt32(completions),
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{"name": name},
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Image: "nginx",
+ },
+ },
+ },
+ },
+ },
+ Status: batchv1.JobStatus{
+ Succeeded: int32(succeeded),
+ Failed: int32(failed),
+ },
+ }
+}
+
+func intToInt32(i int) *int32 {
+ i32 := int32(i)
+ return &i32
+}
--- /dev/null
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statuscheck // import "helm.sh/helm/v3/pkg/kube"
+
+import "k8s.io/cli-runtime/pkg/resource"
+
+// ResourceList provides convenience methods for comparing collections of Infos.
+type ResourceList []*resource.Info
+
+// Append adds an Info to the Result.
+func (r *ResourceList) Append(val *resource.Info) {
+ *r = append(*r, val)
+}
+
+// Visit implements resource.Visitor.
+func (r ResourceList) Visit(fn resource.VisitorFunc) error {
+ for _, i := range r {
+ if err := fn(i, nil); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Filter returns a new Result with Infos that satisfy the predicate fn.
+func (r ResourceList) Filter(fn func(*resource.Info) bool) ResourceList {
+ var result ResourceList
+ for _, i := range r {
+ if fn(i) {
+ result.Append(i)
+ }
+ }
+ return result
+}
+
+// Get returns the Info from the result that matches the name and kind.
+func (r ResourceList) Get(info *resource.Info) *resource.Info {
+ for _, i := range r {
+ if isMatchingInfo(i, info) {
+ return i
+ }
+ }
+ return nil
+}
+
+// Contains checks to see if an object exists.
+func (r ResourceList) Contains(info *resource.Info) bool {
+ for _, i := range r {
+ if isMatchingInfo(i, info) {
+ return true
+ }
+ }
+ return false
+}
+
+// Difference will return a new Result with objects not contained in rs.
+func (r ResourceList) Difference(rs ResourceList) ResourceList {
+ return r.Filter(func(info *resource.Info) bool {
+ return !rs.Contains(info)
+ })
+}
+
+// Intersect will return a new Result with objects contained in both Results.
+func (r ResourceList) Intersect(rs ResourceList) ResourceList {
+ return r.Filter(rs.Contains)
+}
+
+// isMatchingInfo returns true if infos match on Name and GroupVersionKind.
+func isMatchingInfo(a, b *resource.Info) bool {
+ return a.Name == b.Name && a.Namespace == b.Namespace && a.Mapping.GroupVersionKind.Kind == b.Mapping.GroupVersionKind.Kind
+}
--- /dev/null
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statuscheck // import "helm.sh/helm/v3/pkg/kube"
+
+import (
+ "testing"
+
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/cli-runtime/pkg/resource"
+)
+
+func TestResourceList(t *testing.T) {
+ mapping := &meta.RESTMapping{
+ Resource: schema.GroupVersionResource{Group: "group", Version: "version", Resource: "pod"},
+ }
+
+ info := func(name string) *resource.Info {
+ return &resource.Info{Name: name, Mapping: mapping}
+ }
+
+ var r1, r2 ResourceList
+ r1 = []*resource.Info{info("foo"), info("bar")}
+ r2 = []*resource.Info{info("bar")}
+
+ if r1.Get(info("bar")).Mapping.Resource.Resource != "pod" {
+ t.Error("expected get pod")
+ }
+
+ diff := r1.Difference(r2)
+ if len(diff) != 1 {
+ t.Error("expected 1 result")
+ }
+
+ if !diff.Contains(info("foo")) {
+ t.Error("expected diff to return foo")
+ }
+
+ inter := r1.Intersect(r2)
+ if len(inter) != 1 {
+ t.Error("expected 1 result")
+ }
+
+ if !inter.Contains(info("bar")) {
+ t.Error("expected intersect to return bar")
+ }
+}
--- /dev/null
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package statuscheck // import "helm.sh/helm/v3/pkg/kube"
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/pkg/errors"
+ appsv1 "k8s.io/api/apps/v1"
+ appsv1beta1 "k8s.io/api/apps/v1beta1"
+ appsv1beta2 "k8s.io/api/apps/v1beta2"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/runtime"
+
+ "k8s.io/apimachinery/pkg/util/wait"
+)
+
+type waiter struct {
+ c ReadyChecker
+ timeout time.Duration
+ log func(string, ...interface{})
+}
+
+// waitForResources polls to get the current status of all pods, PVCs, Services and
+// Jobs(optional) until all are ready or a timeout is reached
+func (w *waiter) waitForResources(created ResourceList) error {
+ w.log("beginning wait for %d resources with timeout of %v", len(created), w.timeout)
+
+ ctx, cancel := context.WithTimeout(context.Background(), w.timeout)
+ defer cancel()
+
+ return wait.PollImmediateUntil(2*time.Second, func() (bool, error) {
+ for _, v := range created {
+ ready, err := w.c.IsReady(ctx, v)
+ if !ready || err != nil {
+ return false, err
+ }
+ }
+ return true, nil
+ }, ctx.Done())
+}
+
+// SelectorsForObject returns the pod label selector for a given object
+//
+// Modified version of https://github.com/kubernetes/kubernetes/blob/v1.14.1/pkg/kubectl/polymorphichelpers/helpers.go#L84
+func SelectorsForObject(object runtime.Object) (selector labels.Selector, err error) {
+ switch t := object.(type) {
+ case *extensionsv1beta1.ReplicaSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1.ReplicaSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1beta2.ReplicaSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *corev1.ReplicationController:
+ selector = labels.SelectorFromSet(t.Spec.Selector)
+ case *appsv1.StatefulSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1beta1.StatefulSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1beta2.StatefulSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *extensionsv1beta1.DaemonSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1.DaemonSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1beta2.DaemonSet:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *extensionsv1beta1.Deployment:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1.Deployment:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1beta1.Deployment:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *appsv1beta2.Deployment:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *batchv1.Job:
+ selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
+ case *corev1.Service:
+ if t.Spec.Selector == nil || len(t.Spec.Selector) == 0 {
+ return nil, fmt.Errorf("invalid service '%s': Service is defined without a selector", t.Name)
+ }
+ selector = labels.SelectorFromSet(t.Spec.Selector)
+
+ default:
+ return nil, fmt.Errorf("selector for %T not implemented", object)
+ }
+
+ return selector, errors.Wrap(err, "invalid label selector")
+}
--- /dev/null
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "context"
+ "sort"
+
+ apps "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ apiequality "k8s.io/apimachinery/pkg/api/equality"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ intstrutil "k8s.io/apimachinery/pkg/util/intstr"
+ appsclient "k8s.io/client-go/kubernetes/typed/apps/v1"
+)
+
+// deploymentutil contains a copy of a few functions from Kubernetes controller code to avoid a dependency on k8s.io/kubernetes.
+// This code is copied from https://github.com/kubernetes/kubernetes/blob/e856613dd5bb00bcfaca6974431151b5c06cbed5/pkg/controller/deployment/util/deployment_util.go
+// No changes to the code were made other than removing some unused functions
+
+// RsListFunc returns the ReplicaSet from the ReplicaSet namespace and the List metav1.ListOptions.
+type RsListFunc func(string, metav1.ListOptions) ([]*apps.ReplicaSet, error)
+
+// ListReplicaSets returns a slice of RSes the given deployment targets.
+// Note that this does NOT attempt to reconcile ControllerRef (adopt/orphan),
+// because only the controller itself should do that.
+// However, it does filter out anything whose ControllerRef doesn't match.
+func ListReplicaSets(deployment *apps.Deployment, getRSList RsListFunc) ([]*apps.ReplicaSet, error) {
+ // TODO: Right now we list replica sets by their labels. We should list them by selector, i.e. the replica set's selector
+ // should be a superset of the deployment's selector, see https://github.com/kubernetes/kubernetes/issues/19830.
+ namespace := deployment.Namespace
+ selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
+ if err != nil {
+ return nil, err
+ }
+ options := metav1.ListOptions{LabelSelector: selector.String()}
+ all, err := getRSList(namespace, options)
+ if err != nil {
+ return nil, err
+ }
+ // Only include those whose ControllerRef matches the Deployment.
+ owned := make([]*apps.ReplicaSet, 0, len(all))
+ for _, rs := range all {
+ if metav1.IsControlledBy(rs, deployment) {
+ owned = append(owned, rs)
+ }
+ }
+ return owned, nil
+}
+
+// ReplicaSetsByCreationTimestamp sorts a list of ReplicaSet by creation timestamp, using their names as a tie breaker.
+type ReplicaSetsByCreationTimestamp []*apps.ReplicaSet
+
+func (o ReplicaSetsByCreationTimestamp) Len() int { return len(o) }
+func (o ReplicaSetsByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
+func (o ReplicaSetsByCreationTimestamp) Less(i, j int) bool {
+ if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) {
+ return o[i].Name < o[j].Name
+ }
+ return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp)
+}
+
+// FindNewReplicaSet returns the new RS this given deployment targets (the one with the same pod template).
+func FindNewReplicaSet(deployment *apps.Deployment, rsList []*apps.ReplicaSet) *apps.ReplicaSet {
+ sort.Sort(ReplicaSetsByCreationTimestamp(rsList))
+ for i := range rsList {
+ if EqualIgnoreHash(&rsList[i].Spec.Template, &deployment.Spec.Template) {
+ // In rare cases, such as after cluster upgrades, Deployment may end up with
+ // having more than one new ReplicaSets that have the same template as its template,
+ // see https://github.com/kubernetes/kubernetes/issues/40415
+ // We deterministically choose the oldest new ReplicaSet.
+ return rsList[i]
+ }
+ }
+ // new ReplicaSet does not exist.
+ return nil
+}
+
+// EqualIgnoreHash returns true if two given podTemplateSpec are equal, ignoring the diff in value of Labels[pod-template-hash]
+// We ignore pod-template-hash because:
+// 1. The hash result would be different upon podTemplateSpec API changes
+// (e.g. the addition of a new field will cause the hash code to change)
+// 2. The deployment template won't have hash labels
+func EqualIgnoreHash(template1, template2 *v1.PodTemplateSpec) bool {
+ t1Copy := template1.DeepCopy()
+ t2Copy := template2.DeepCopy()
+ // Remove hash labels from template.Labels before comparing
+ delete(t1Copy.Labels, apps.DefaultDeploymentUniqueLabelKey)
+ delete(t2Copy.Labels, apps.DefaultDeploymentUniqueLabelKey)
+ return apiequality.Semantic.DeepEqual(t1Copy, t2Copy)
+}
+
+// GetNewReplicaSet returns a replica set that matches the intent of the given deployment; get ReplicaSetList from client interface.
+// Returns nil if the new replica set doesn't exist yet.
+func GetNewReplicaSet(deployment *apps.Deployment, c appsclient.AppsV1Interface) (*apps.ReplicaSet, error) {
+ rsList, err := ListReplicaSets(deployment, RsListFromClient(c))
+ if err != nil {
+ return nil, err
+ }
+ return FindNewReplicaSet(deployment, rsList), nil
+}
+
+// RsListFromClient returns an rsListFunc that wraps the given client.
+func RsListFromClient(c appsclient.AppsV1Interface) RsListFunc {
+ return func(namespace string, options metav1.ListOptions) ([]*apps.ReplicaSet, error) {
+ rsList, err := c.ReplicaSets(namespace).List(context.Background(), options)
+ if err != nil {
+ return nil, err
+ }
+ var ret []*apps.ReplicaSet
+ for i := range rsList.Items {
+ ret = append(ret, &rsList.Items[i])
+ }
+ return ret, err
+ }
+}
+
+// IsRollingUpdate returns true if the strategy type is a rolling update.
+func IsRollingUpdate(deployment *apps.Deployment) bool {
+ return deployment.Spec.Strategy.Type == apps.RollingUpdateDeploymentStrategyType
+}
+
+// MaxUnavailable returns the maximum unavailable pods a rolling deployment can take.
+func MaxUnavailable(deployment apps.Deployment) int32 {
+ if !IsRollingUpdate(&deployment) || *(deployment.Spec.Replicas) == 0 {
+ return int32(0)
+ }
+ // Error caught by validation
+ _, maxUnavailable, _ := ResolveFenceposts(deployment.Spec.Strategy.RollingUpdate.MaxSurge, deployment.Spec.Strategy.RollingUpdate.MaxUnavailable, *(deployment.Spec.Replicas))
+ if maxUnavailable > *deployment.Spec.Replicas {
+ return *deployment.Spec.Replicas
+ }
+ return maxUnavailable
+}
+
+// ResolveFenceposts resolves both maxSurge and maxUnavailable. This needs to happen in one
+// step. For example:
+//
+// 2 desired, max unavailable 1%, surge 0% - should scale old(-1), then new(+1), then old(-1), then new(+1)
+// 1 desired, max unavailable 1%, surge 0% - should scale old(-1), then new(+1)
+// 2 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1)
+// 1 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1)
+// 2 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1)
+// 1 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1)
+func ResolveFenceposts(maxSurge, maxUnavailable *intstrutil.IntOrString, desired int32) (int32, int32, error) {
+ surge, err := intstrutil.GetValueFromIntOrPercent(intstrutil.ValueOrDefault(maxSurge, intstrutil.FromInt(0)), int(desired), true)
+ if err != nil {
+ return 0, 0, err
+ }
+ unavailable, err := intstrutil.GetValueFromIntOrPercent(intstrutil.ValueOrDefault(maxUnavailable, intstrutil.FromInt(0)), int(desired), false)
+ if err != nil {
+ return 0, 0, err
+ }
+
+ if surge == 0 && unavailable == 0 {
+ // Validation should never allow the user to explicitly use zero values for both maxSurge
+ // maxUnavailable. Due to rounding down maxUnavailable though, it may resolve to zero.
+ // If both fenceposts resolve to zero, then we should set maxUnavailable to 1 on the
+ // theory that surge might not work due to quota.
+ unavailable = 1
+ }
+
+ return int32(surge), int32(unavailable), nil
+}
},
{
label: "Successfully read YAML file",
- input: "../mock_files/mock_yamls/deployment.yaml",
+ input: "../../mock_files/mock_yamls/deployment.yaml",
expectedResult: &appsV1.Deployment{
ObjectMeta: metaV1.ObjectMeta{
Name: "mock-deployment",
/*
Copyright 2018 Intel Corporation.
+Copyright © 2021 Nokia Bell Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
import (
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
-
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
+ "time"
)
// ExportedVariable is what we will look for when calling the plugin
type mockPlugin struct {
}
+func (g mockPlugin) WatchUntilReady(
+ timeout time.Duration,
+ ns string,
+ res helm.KubernetesResource,
+ mapper meta.RESTMapper,
+ restClient rest.Interface,
+ objType runtime.Object,
+ clientSet kubernetes.Interface) error {
+ return nil
+}
+
// Create object in a specific Kubernetes resource
func (p mockPlugin) Create(yamlFilePath string, namespace string, client plugin.KubernetesConnector) (string, error) {
return "resource-name", nil
--- /dev/null
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: job
+ labels:
+ app: job
+ annotations:
+ "helm.sh/hook": post-install
+ "helm.sh/hook-weight": "-8"
+ "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+ backoffLimit: 3
+ template:
+ metadata:
+ labels:
+ app: job
+ spec:
+ shareProcessNamespace: true
+ restartPolicy: Never
+ containers:
+ - name: sleep-container
+ image: "busybox"
+ command: ["sleep", "10"]
\ No newline at end of file
/*
Copyright 2018 Intel Corporation.
+Copyright © 2021 Nokia Bell Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
import (
"context"
+ "fmt"
+ logger "log"
+ "time"
+
+ appsv1 "k8s.io/api/apps/v1"
+ "k8s.io/client-go/kubernetes"
+ //appsv1beta1 "k8s.io/api/apps/v1beta1"
+ //appsv1beta2 "k8s.io/api/apps/v1beta2"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/util/intstr"
+
pkgerrors "github.com/pkg/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/watch"
+ "k8s.io/client-go/rest"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/app"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/config"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
+ cachetools "k8s.io/client-go/tools/cache"
+ watchtools "k8s.io/client-go/tools/watch"
)
// Compile time check to see if genericPlugin implements the correct interface
type genericPlugin struct {
}
+func (g genericPlugin) WatchUntilReady(
+ timeout time.Duration,
+ ns string,
+ res helm.KubernetesResource,
+ mapper meta.RESTMapper,
+ restClient rest.Interface,
+ objType runtime.Object,
+ clientSet kubernetes.Interface) error {
+ selector, err := fields.ParseSelector(fmt.Sprintf("metadata.name=%s", res.Name))
+ if err != nil {
+ return err
+ }
+
+ mapping, err := mapper.RESTMapping(schema.GroupKind{
+ Group: res.GVK.Group,
+ Kind: res.GVK.Kind,
+ }, res.GVK.Version)
+ if err != nil {
+ return pkgerrors.Wrap(err, "Preparing mapper based on GVK")
+ }
+ lw := cachetools.NewListWatchFromClient(restClient, mapping.Resource.Resource, ns, selector)
+
+ // What we watch for depends on the Kind.
+ // - For a Job, we watch for completion.
+ // - For all else, we watch until Ready.
+ // In the future, we might want to add some special logic for types
+ // like Ingress, Volume, etc.
+ ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout)
+ defer cancel()
+
+ _, err = watchtools.UntilWithSync(ctx, lw, objType, nil, func(e watch.Event) (bool, error) {
+ obj := e.Object
+ switch e.Type {
+ case watch.Added, watch.Modified:
+ // For things like a secret or a config map, this is the best indicator
+ // we get. We care mostly about jobs, where what we want to see is
+ // the status go into a good state.
+ logger.Printf("Add/Modify event for %s: %v", res.Name, e.Type)
+ switch res.GVK.Kind {
+ case "Job":
+ return g.waitForJob(obj, res.Name)
+ case "Pod":
+ return g.waitForPodSuccess(obj, res.Name)
+ case "Deployment":
+ return g.waitForDeploymentSuccess(obj, res.Name, clientSet)
+ case "DaemonSet":
+ return g.waitForDaemonSetSuccess(obj, res.Name)
+ case "StatefulSet":
+ return g.waitForStatefulSetSuccess(obj, res.Name)
+ }
+ return true, nil
+ case watch.Deleted:
+ logger.Printf("Deleted event for %s", res.Name)
+ return true, nil
+ case watch.Error:
+ // Handle error and return with an error.
+ logger.Printf("Error event for %s", res.Name)
+ return true, pkgerrors.New("failed to deploy " + res.Name)
+ default:
+ return false, nil
+ }
+ })
+ if err != nil {
+ logger.Printf("Error in Rss %s", res.Name)
+ return err
+ } else {
+ logger.Printf("Done for %s", res.Name)
+ return nil
+ }
+}
+
+// waitForJob is a helper that waits for a job to complete.
+//
+// This operates on an event returned from a watcher.
+func (g genericPlugin) waitForJob(obj runtime.Object, name string) (bool, error) {
+ o, ok := obj.(*batchv1.Job)
+ if !ok {
+ return true, pkgerrors.New("expected " + name + " to be a *batch.Job, got " + obj.GetObjectKind().GroupVersionKind().Kind)
+ }
+
+ for _, c := range o.Status.Conditions {
+ if c.Type == batchv1.JobComplete && c.Status == "True" {
+ return true, nil
+ } else if c.Type == batchv1.JobFailed && c.Status == "True" {
+ return true, pkgerrors.New("job failed: " + c.Reason)
+ }
+ }
+
+ logger.Printf("%s: Jobs active: %d, jobs failed: %d, jobs succeeded: %d", name, o.Status.Active, o.Status.Failed, o.Status.Succeeded)
+ return false, nil
+}
+
+// waitForPodSuccess is a helper that waits for a pod to complete.
+//
+// This operates on an event returned from a watcher.
+func (g genericPlugin) waitForPodSuccess(obj runtime.Object, name string) (bool, error) {
+ o, ok := obj.(*corev1.Pod)
+ if !ok {
+ return true, pkgerrors.New("expected " + name + " to be a *v1.Pod, got " + obj.GetObjectKind().GroupVersionKind().Kind)
+ }
+
+ switch o.Status.Phase {
+ case corev1.PodSucceeded:
+ logger.Printf("Pod %s succeeded", o.Name)
+ return true, nil
+ case corev1.PodFailed:
+ return true, pkgerrors.New("pod " + o.Name + " failed")
+ case corev1.PodPending:
+ logger.Printf("Pod %s pending", o.Name)
+ case corev1.PodRunning:
+ logger.Printf("Pod %s running", o.Name)
+ }
+
+ return false, nil
+}
+
+// waitForDeploymentSuccess is a helper that waits for a deployment to run.
+//
+// This operates on an event returned from a watcher.
+func (g genericPlugin) waitForDeploymentSuccess(obj runtime.Object, name string, clientSet kubernetes.Interface) (bool, error) {
+ o, ok := obj.(*appsv1.Deployment)
+ if !ok {
+ return true, pkgerrors.New("expected " + name + " to be a *apps.Deployment, got " + obj.GetObjectKind().GroupVersionKind().Kind)
+ }
+
+ // If paused deployment will never be ready -> consider ready
+ if o.Spec.Paused {
+ logger.Printf("Depoyment %s is paused, consider ready", o.Name)
+ return true, nil
+ }
+
+ // Find RS associated with deployment
+ newReplicaSet, err := app.GetNewReplicaSet(o, clientSet.AppsV1())
+ if err != nil || newReplicaSet == nil {
+ return false, err
+ }
+ expectedReady := *o.Spec.Replicas - app.MaxUnavailable(*o)
+ if !(newReplicaSet.Status.ReadyReplicas >= expectedReady) {
+ logger.Printf("Deployment is not ready: %s/%s. %d out of %d expected pods are ready", o.Namespace, o.Name, newReplicaSet.Status.ReadyReplicas, expectedReady)
+ return false, nil
+ }
+ return true, nil
+}
+
+// waitForDaemonSetSuccess is a helper that waits for a daemonSet to run.
+//
+// This operates on an event returned from a watcher.
+func (g genericPlugin) waitForDaemonSetSuccess(obj runtime.Object, name string) (bool, error) {
+ o, ok := obj.(*appsv1.DaemonSet)
+ if !ok {
+ return true, pkgerrors.New("expected " + name + " to be a *apps.DaemonSet, got " + obj.GetObjectKind().GroupVersionKind().Kind)
+ }
+
+ // If the update strategy is not a rolling update, there will be nothing to wait for
+ if o.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType {
+ return true, nil
+ }
+
+ // Make sure all the updated pods have been scheduled
+ if o.Status.UpdatedNumberScheduled != o.Status.DesiredNumberScheduled {
+ logger.Printf("DaemonSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", o.Namespace, o.Name, o.Status.UpdatedNumberScheduled, o.Status.DesiredNumberScheduled)
+ return false, nil
+ }
+ maxUnavailable, err := intstr.GetValueFromIntOrPercent(o.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, int(o.Status.DesiredNumberScheduled), true)
+ if err != nil {
+ // If for some reason the value is invalid, set max unavailable to the
+ // number of desired replicas. This is the same behavior as the
+ // `MaxUnavailable` function in deploymentutil
+ maxUnavailable = int(o.Status.DesiredNumberScheduled)
+ }
+
+ expectedReady := int(o.Status.DesiredNumberScheduled) - maxUnavailable
+ if !(int(o.Status.NumberReady) >= expectedReady) {
+ logger.Printf("DaemonSet is not ready: %s/%s. %d out of %d expected pods are ready", o.Namespace, o.Name, o.Status.NumberReady, expectedReady)
+ return false, nil
+ }
+ return true, nil
+}
+
+// waitForStatefulSetSuccess is a helper that waits for a statefulSet to run.
+//
+// This operates on an event returned from a watcher.
+func (g genericPlugin) waitForStatefulSetSuccess(obj runtime.Object, name string) (bool, error) {
+ o, ok := obj.(*appsv1.StatefulSet)
+ if !ok {
+ return true, pkgerrors.New("expected " + name + " to be a *apps.StatefulSet, got " + obj.GetObjectKind().GroupVersionKind().Kind)
+ }
+
+ // If the update strategy is not a rolling update, there will be nothing to wait for
+ if o.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
+ return true, nil
+ }
+
+ // Dereference all the pointers because StatefulSets like them
+ var partition int
+ // 1 is the default for replicas if not set
+ var replicas = 1
+ // For some reason, even if the update strategy is a rolling update, the
+ // actual rollingUpdate field can be nil. If it is, we can safely assume
+ // there is no partition value
+ if o.Spec.UpdateStrategy.RollingUpdate != nil && o.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
+ partition = int(*o.Spec.UpdateStrategy.RollingUpdate.Partition)
+ }
+ if o.Spec.Replicas != nil {
+ replicas = int(*o.Spec.Replicas)
+ }
+
+ // Because an update strategy can use partitioning, we need to calculate the
+ // number of updated replicas we should have. For example, if the replicas
+ // is set to 3 and the partition is 2, we'd expect only one pod to be
+ // updated
+ expectedReplicas := replicas - partition
+
+ // Make sure all the updated pods have been scheduled
+ if int(o.Status.UpdatedReplicas) != expectedReplicas {
+ logger.Printf("StatefulSet is not ready: %s/%s. %d out of %d expected pods have been scheduled", o.Namespace, o.Name, o.Status.UpdatedReplicas, expectedReplicas)
+ return false, nil
+ }
+
+ if int(o.Status.ReadyReplicas) != replicas {
+ logger.Printf("StatefulSet is not ready: %s/%s. %d out of %d expected pods are ready", o.Namespace, o.Name, o.Status.ReadyReplicas, replicas)
+ return false, nil
+ }
+ return true, nil
+}
+
// Create generic object in a specific Kubernetes cluster
func (g genericPlugin) Create(yamlFilePath string, namespace string, client plugin.KubernetesConnector) (string, error) {
if namespace == "" {
/*
Copyright 2018 Intel Corporation.
+Copyright © 2021 Nokia Bell Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
import (
"context"
"log"
+ "time"
pkgerrors "github.com/pkg/errors"
coreV1 "k8s.io/api/core/v1"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
)
// Compile time check to see if namespacePlugin implements the correct interface
type namespacePlugin struct {
}
+func (g namespacePlugin) WatchUntilReady(
+ timeout time.Duration,
+ ns string,
+ res helm.KubernetesResource,
+ mapper meta.RESTMapper,
+ restClient rest.Interface,
+ objType runtime.Object,
+ clientSet kubernetes.Interface) error {
+ return pkgerrors.Errorf("This function is not implemented in this plugin")
+}
+
// Create a namespace object in a specific Kubernetes cluster
func (p namespacePlugin) Create(yamlFilePath string, namespace string, client plugin.KubernetesConnector) (string, error) {
namespaceObj := &coreV1.Namespace{
/*
Copyright 2018 Intel Corporation.
+Copyright © 2021 Nokia Bell Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
import (
"context"
"log"
+ "time"
pkgerrors "github.com/pkg/errors"
coreV1 "k8s.io/api/core/v1"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
- utils "github.com/onap/multicloud-k8s/src/k8splugin/internal"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/config"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/helm"
"github.com/onap/multicloud-k8s/src/k8splugin/internal/plugin"
+ "github.com/onap/multicloud-k8s/src/k8splugin/internal/utils"
)
// Compile time check to see if servicePlugin implements the correct interface
type servicePlugin struct {
}
+func (g servicePlugin) WatchUntilReady(
+ timeout time.Duration,
+ ns string,
+ res helm.KubernetesResource,
+ mapper meta.RESTMapper,
+ restClient rest.Interface,
+ objType runtime.Object,
+ clientSet kubernetes.Interface) error {
+ return pkgerrors.Errorf("This function is not implemented in this plugin")
+}
+
// Create a service object in a specific Kubernetes cluster
func (p servicePlugin) Create(yamlFilePath string, namespace string, client plugin.KubernetesConnector) (string, error) {
if namespace == "" {
Code Review / multicloud / k8s.git / commitdiff