From c3ef7c3f40f6aa4a14a98301ae12bfa11b1a12c3 Mon Sep 17 00:00:00 2001 From: Srivahni Date: Thu, 29 Aug 2019 12:06:51 -0700 Subject: [PATCH] Visualization operator - update/delete datasource Added support in the visualization operator to update or delete an existing grafana datasource dynamically. Issue-ID: ONAPARC-393 Signed-off-by: Srivahni Change-Id: I814cca8e5b4f7f5832a948449cf545cecd25b1f9 --- vnfs/DAaaS/README.md | 3 +- .../crds/onap_v1alpha1_grafanadatasource_crd.yaml | 4 - .../grafana/prometheus_grafanadatasource_cr.yaml | 2 +- .../microservices/visualization-operator/go.mod | 1 + .../apis/onap/v1alpha1/grafanadatasource_types.go | 6 - .../pkg/apis/onap/v1alpha1/zz_generated.openapi.go | 3 +- .../grafanadatasource_controller.go | 181 +++++++++++++++++++-- .../pkg/controller/utils/visualizationutils.go | 26 +++ 8 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go diff --git a/vnfs/DAaaS/README.md b/vnfs/DAaaS/README.md index 68d0401a..302cdd6e 100644 --- a/vnfs/DAaaS/README.md +++ b/vnfs/DAaaS/README.md @@ -195,7 +195,8 @@ kubectl create -f edge1 [PLUGIN_NAME3]_collectdplugin_cr.yaml #### Configure Grafana Datasources Using the sample [prometheus_grafanadatasource_cr.yaml](microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml), Configure the GrafanaDataSource CR by running the command below ```yaml -kubectl create -f [DATASOURCE_NAME]_grafanadatasource_cr.yaml +kubectl create -f [DATASOURCE_NAME1]_grafanadatasource_cr.yaml +kubectl create -f [DATASOURCE_NAME2]_grafanadatasource_cr.yaml ... ``` diff --git a/vnfs/DAaaS/microservices/visualization-operator/deploy/crds/onap_v1alpha1_grafanadatasource_crd.yaml b/vnfs/DAaaS/microservices/visualization-operator/deploy/crds/onap_v1alpha1_grafanadatasource_crd.yaml index 39b78d9a..22925cfe 100644 --- a/vnfs/DAaaS/microservices/visualization-operator/deploy/crds/onap_v1alpha1_grafanadatasource_crd.yaml +++ b/vnfs/DAaaS/microservices/visualization-operator/deploy/crds/onap_v1alpha1_grafanadatasource_crd.yaml @@ -30,10 +30,6 @@ spec: spec: properties: datasources: - description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - Important: Run "operator-sdk generate k8s" to regenerate code after - modifying this file Add custom validation using kubebuilder tags: - https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html' items: properties: access: diff --git a/vnfs/DAaaS/microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml b/vnfs/DAaaS/microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml index b8086366..2820baf7 100644 --- a/vnfs/DAaaS/microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml +++ b/vnfs/DAaaS/microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml @@ -8,7 +8,7 @@ spec: type: prometheus url: http://cp-prometheus-prometheus:9090 isDefault: true - access: proxy + access: direct withCredentials: true basicAuth: true basicAuthUser: user diff --git a/vnfs/DAaaS/microservices/visualization-operator/go.mod b/vnfs/DAaaS/microservices/visualization-operator/go.mod index af334b07..a3ed23bd 100644 --- a/vnfs/DAaaS/microservices/visualization-operator/go.mod +++ b/vnfs/DAaaS/microservices/visualization-operator/go.mod @@ -2,6 +2,7 @@ module visualization-operator require ( github.com/NYTimes/gziphandler v1.0.1 // indirect + github.com/go-logr/logr v0.1.0 github.com/operator-framework/operator-sdk v0.9.1-0.20190805223000-66e78cc576ef github.com/spf13/pflag v1.0.3 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect diff --git a/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/grafanadatasource_types.go b/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/grafanadatasource_types.go index 60820067..47a372e3 100644 --- a/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/grafanadatasource_types.go +++ b/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/grafanadatasource_types.go @@ -4,15 +4,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // GrafanaDataSourceSpec defines the desired state of GrafanaDataSource // +k8s:openapi-gen=true type GrafanaDataSourceSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file - // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html Datasources []Datasource `json:"datasources"` Grafana map[string]string `json:"grafana"` } diff --git a/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/zz_generated.openapi.go b/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/zz_generated.openapi.go index 7f74f2f7..9d66c4d0 100644 --- a/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/zz_generated.openapi.go +++ b/vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/zz_generated.openapi.go @@ -174,8 +174,7 @@ func schema_pkg_apis_onap_v1alpha1_GrafanaDataSourceSpec(ref common.ReferenceCal Properties: map[string]spec.Schema{ "datasources": { SchemaProps: spec.SchemaProps{ - Description: "INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run \"operator-sdk generate k8s\" to regenerate code after modifying this file Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html", - Type: []string{"array"}, + Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ diff --git a/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/grafanadatasource/grafanadatasource_controller.go b/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/grafanadatasource/grafanadatasource_controller.go index f46cf1b4..88b8a925 100644 --- a/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/grafanadatasource/grafanadatasource_controller.go +++ b/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/grafanadatasource/grafanadatasource_controller.go @@ -1,11 +1,16 @@ package grafanadatasource import ( + logr "github.com/go-logr/logr" + "bytes" "context" "encoding/json" + "fmt" + "io/ioutil" onapv1alpha1 "visualization-operator/pkg/apis/onap/v1alpha1" + visualizationutils "visualization-operator/pkg/controller/utils" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -77,13 +82,51 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon // Request object not found, could have been deleted after reconcile request. // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. // Return and don't requeue + reqLogger.Info("GrafanaDatasource object not found") return reconcile.Result{}, nil } // Error reading the object - requeue the request. - + reqLogger.Info("Error reading the Grafanadatasource object, Requeing") return reconcile.Result{}, err } + //Check if deletion timestamp is set. If yes, delete the GrafanaDataSource object + isBeingDeleted := checkDeletionTimestamp(reqLogger, instance) + if isBeingDeleted { + //Delete the datasource from grafana + err := deleteDatasource(instance) + if err != nil { + reqLogger.Error(err, "Unable to delete datasource") + return reconcile.Result{}, err + } + //remove Finalizer after deletion + if visualizationutils.Contains(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer) { + if err := removeFinalizer(reqLogger, instance); err != nil { + return reconcile.Result{}, err + } + err := r.client.Update(context.TODO(), instance) + if err != nil { + reqLogger.Error(err, "Unable to update instance") + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + } + + //Add finalizer for the CR object + if !visualizationutils.Contains(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer) { + reqLogger.Info("Adding finalizer for GrafanaDatasource") + if err := addFinalizer(reqLogger, instance); err != nil { + return reconcile.Result{}, err + } + err := r.client.Update(context.TODO(), instance) + if err != nil { + reqLogger.Error(err, "Unable to update instance") + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + datasources := instance.Spec.Datasources grafana := instance.Spec.Grafana @@ -110,6 +153,25 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon return reconcile.Result{}, err } + respBody, err := ioutil.ReadAll(getResp.Body) + if err != nil { + reqLogger.Error(err, "Response data not read properly") + return reconcile.Result{}, err + } + + respBodyBytes := []byte(respBody) + var respData map[string]interface{} + + if err := json.Unmarshal(respBodyBytes, &respData); err != nil { + reqLogger.Error(err, "JSON unmarshalling error") + return reconcile.Result{}, err + } + + respURL := fmt.Sprintf("%v", respData["url"]) + respID := fmt.Sprintf("%v", respData["id"]) + respIsDefault := respData["isDefault"] + respAccess := fmt.Sprintf("%v", respData["access"]) + defer getResp.Body.Close() //add datasource if datasource does not exist already @@ -120,19 +182,24 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon return reconcile.Result{}, err } } else if getResp.StatusCode == http.StatusOK { - //if datasource already exists + //if datasource already exists and there is any change in the spec - update it reqLogger.V(1).Info("datasource already exists", "datasource", datasource.Name) - } else { - reqLogger.Error(err, "unknown error", datasource.Name) + if respURL != datasource.URL || respIsDefault.(bool) != datasource.IsDefault || respAccess != datasource.Access { + if err := updateDatasource(grafana, datasource, respID); err != nil { + return reconcile.Result{}, err + } + } else { + reqLogger.Info("No creation/updation of datasource needed") + return reconcile.Result{}, nil + } } - } return reconcile.Result{}, nil } func createDataSource(grafana map[string]string, datasource onapv1alpha1.Datasource) error { reqLogger := log.WithValues("Datasource name", datasource.Name, "Datasource URL", datasource.URL) - reqLogger.Info("creating datasource") + reqLogger.Info("Creating datasource") grafanaURL := grafana["url"] + "/api/datasources" grafanaUsername := grafana["username"] @@ -145,19 +212,19 @@ func createDataSource(grafana map[string]string, datasource onapv1alpha1.Datasou return err } - req, err := http.NewRequest("POST", grafanaURL, bytes.NewBuffer(postBody)) + postReq, err := http.NewRequest("POST", grafanaURL, bytes.NewBuffer(postBody)) if err != nil { reqLogger.Error(err, "POST REQUEST error") return err } - req.Header.Set("Content-Type", "application/json") - req.SetBasicAuth(grafanaUsername, grafanaPassword) - postResp, err := client.Do(req) + postReq.Header.Set("Content-Type", "application/json") + postReq.SetBasicAuth(grafanaUsername, grafanaPassword) + postResp, err := client.Do(postReq) if err != nil { reqLogger.Error(err, "POST RESPONSE error") return err } - defer req.Body.Close() + defer postReq.Body.Close() if postResp.StatusCode == http.StatusOK { reqLogger.Info("Datasource created") @@ -165,3 +232,95 @@ func createDataSource(grafana map[string]string, datasource onapv1alpha1.Datasou } return err } + +func updateDatasource(grafana map[string]string, datasource onapv1alpha1.Datasource, datasourceID string) error { + reqLogger := log.WithValues("Datasource name", datasource.Name, "Datasource URL", datasource.URL) + reqLogger.Info("Updating datasource") + + grafanaURL := grafana["url"] + "/api/datasources/" + datasourceID + grafanaUsername := grafana["username"] + grafanaPassword := grafana["password"] + + client := &http.Client{} + putBody, err := json.Marshal(datasource) + if err != nil { + reqLogger.Error(err, "JSON Marshalling error") + return err + } + putReq, err := http.NewRequest("PUT", grafanaURL, bytes.NewBuffer(putBody)) + if err != nil { + reqLogger.Error(err, "PUT REQUEST error") + return err + } + putReq.Header.Set("Content-Type", "application/json") + putReq.Header.Set("Accept", "application/json") + putReq.SetBasicAuth(grafanaUsername, grafanaPassword) + + putResp, err := client.Do(putReq) + if err != nil { + reqLogger.Error(err, "PUT RESPONSE error") + return err + } + defer putReq.Body.Close() + + if putResp.StatusCode == http.StatusOK { + reqLogger.Info("Datasource updated") + return nil + } + return err +} + +func deleteDatasource(instance *onapv1alpha1.GrafanaDataSource) error { + + datasources := instance.Spec.Datasources + grafana := instance.Spec.Grafana + + for _, datasource := range datasources { + + reqLogger := log.WithValues("Datasource name", datasource.Name, "Datasource URL", datasource.URL) + reqLogger.Info("Deleting datasource") + + grafanaURL := grafana["url"] + "/api/datasources/name/" + datasource.Name + grafanaUsername := grafana["username"] + grafanaPassword := grafana["password"] + + client := &http.Client{} + deleteReq, err := http.NewRequest("DELETE", grafanaURL, nil) + if err != nil { + reqLogger.Error(err, "DELETE request error") + return err + } + + deleteReq.SetBasicAuth(grafanaUsername, grafanaPassword) + + deleteResp, err := client.Do(deleteReq) + if err != nil { + reqLogger.Error(err, "DELETE RESPONSE error") + return err + } + + if deleteResp.StatusCode == http.StatusOK { + reqLogger.Info("Datasource deleted") + return nil + } + return err + } + return nil +} + +func checkDeletionTimestamp(reqlogger logr.Logger, instance *onapv1alpha1.GrafanaDataSource) bool { + isMarkedForDeletion := instance.GetDeletionTimestamp() != nil + return isMarkedForDeletion +} + +func addFinalizer(reqlogger logr.Logger, instance *onapv1alpha1.GrafanaDataSource) error { + reqlogger.Info("Adding finalizer for the GrafanaDatasource") + instance.SetFinalizers(append(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer)) + return nil +} + +func removeFinalizer(reqlogger logr.Logger, instance *onapv1alpha1.GrafanaDataSource) error { + reqlogger.Info("Removing finalizer for the GrafanaDatasource") + instance.SetFinalizers(visualizationutils.Remove(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer)) + return nil +} diff --git a/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go b/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go new file mode 100644 index 00000000..dc7e1f1b --- /dev/null +++ b/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go @@ -0,0 +1,26 @@ +package visualizationutils + +// Define the GrafanaDatasource finalizer for handling deletion +const ( + VisualizationFinalizer = "finalizer.visualization.onap.org" +) + +// Contains checks if a string is contained in a list of strings +func Contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// Remove checks and removes a string from a list of strings +func Remove(list []string, s string) []string { + for i, v := range list { + if v == s { + list = append(list[:i], list[i+1:]...) + } + } + return list +} -- 2.16.6