Visualization operator - update/delete datasource 33/94633/4
authorSrivahni <srivahni.chivukula@intel.com>
Thu, 29 Aug 2019 19:06:51 +0000 (12:06 -0700)
committerSrivahni Chivukula <srivahni.chivukula@intel.com>
Fri, 6 Sep 2019 17:22:06 +0000 (17:22 +0000)
Added support in the visualization operator to update
or delete an existing grafana datasource dynamically.

Issue-ID: ONAPARC-393
Signed-off-by: Srivahni <srivahni.chivukula@intel.com>
Change-Id: I814cca8e5b4f7f5832a948449cf545cecd25b1f9

vnfs/DAaaS/README.md
vnfs/DAaaS/microservices/visualization-operator/deploy/crds/onap_v1alpha1_grafanadatasource_crd.yaml
vnfs/DAaaS/microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml
vnfs/DAaaS/microservices/visualization-operator/go.mod
vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/grafanadatasource_types.go
vnfs/DAaaS/microservices/visualization-operator/pkg/apis/onap/v1alpha1/zz_generated.openapi.go
vnfs/DAaaS/microservices/visualization-operator/pkg/controller/grafanadatasource/grafanadatasource_controller.go
vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go [new file with mode: 0644]

index 68d0401..302cdd6 100644 (file)
@@ -195,7 +195,8 @@ kubectl create -f edge1 [PLUGIN_NAME3]_collectdplugin_cr.yaml
 #### Configure Grafana Datasources
 Using the sample [prometheus_grafanadatasource_cr.yaml](microservices/visualization-operator/examples/grafana/prometheus_grafanadatasource_cr.yaml), Configure the GrafanaDataSource CR by running the command below
 ```yaml
-kubectl create -f [DATASOURCE_NAME]_grafanadatasource_cr.yaml
+kubectl create -f [DATASOURCE_NAME1]_grafanadatasource_cr.yaml
+kubectl create -f [DATASOURCE_NAME2]_grafanadatasource_cr.yaml
 ...
 ```
 
index 39b78d9..22925cf 100644 (file)
@@ -30,10 +30,6 @@ spec:
         spec:
           properties:
             datasources:
-              description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-                Important: Run "operator-sdk generate k8s" to regenerate code after
-                modifying this file Add custom validation using kubebuilder tags:
-                https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html'
               items:
                 properties:
                   access:
index b808636..2820baf 100644 (file)
@@ -8,7 +8,7 @@ spec:
       type: prometheus
       url: http://cp-prometheus-prometheus:9090
       isDefault: true
-      access: proxy
+      access: direct 
       withCredentials: true
       basicAuth: true
       basicAuthUser: user
index af334b0..a3ed23b 100644 (file)
@@ -2,6 +2,7 @@ module visualization-operator
 
 require (
        github.com/NYTimes/gziphandler v1.0.1 // indirect
+       github.com/go-logr/logr v0.1.0
        github.com/operator-framework/operator-sdk v0.9.1-0.20190805223000-66e78cc576ef
        github.com/spf13/pflag v1.0.3
        golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
index 6082006..47a372e 100644 (file)
@@ -4,15 +4,9 @@ import (
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
 // GrafanaDataSourceSpec defines the desired state of GrafanaDataSource
 // +k8s:openapi-gen=true
 type GrafanaDataSourceSpec struct {
-       // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-       // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
-       // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html
        Datasources []Datasource      `json:"datasources"`
        Grafana     map[string]string `json:"grafana"`
 }
index 7f74f2f..9d66c4d 100644 (file)
@@ -174,8 +174,7 @@ func schema_pkg_apis_onap_v1alpha1_GrafanaDataSourceSpec(ref common.ReferenceCal
                                Properties: map[string]spec.Schema{
                                        "datasources": {
                                                SchemaProps: spec.SchemaProps{
-                                                       Description: "INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run \"operator-sdk generate k8s\" to regenerate code after modifying this file Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html",
-                                                       Type:        []string{"array"},
+                                                       Type: []string{"array"},
                                                        Items: &spec.SchemaOrArray{
                                                                Schema: &spec.Schema{
                                                                        SchemaProps: spec.SchemaProps{
index f46cf1b..88b8a92 100644 (file)
@@ -1,11 +1,16 @@
 package grafanadatasource
 
 import (
+       logr "github.com/go-logr/logr"
+
        "bytes"
        "context"
        "encoding/json"
+       "fmt"
+       "io/ioutil"
 
        onapv1alpha1 "visualization-operator/pkg/apis/onap/v1alpha1"
+       visualizationutils "visualization-operator/pkg/controller/utils"
 
        "k8s.io/apimachinery/pkg/api/errors"
        "k8s.io/apimachinery/pkg/runtime"
@@ -77,13 +82,51 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon
                        // Request object not found, could have been deleted after reconcile request.
                        // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
                        // Return and don't requeue
+                       reqLogger.Info("GrafanaDatasource object not found")
                        return reconcile.Result{}, nil
                }
                // Error reading the object - requeue the request.
-
+               reqLogger.Info("Error reading the Grafanadatasource object, Requeing")
                return reconcile.Result{}, err
        }
 
+       //Check if deletion timestamp is set. If yes, delete the GrafanaDataSource object
+       isBeingDeleted := checkDeletionTimestamp(reqLogger, instance)
+       if isBeingDeleted {
+               //Delete the datasource from grafana
+               err := deleteDatasource(instance)
+               if err != nil {
+                       reqLogger.Error(err, "Unable to delete datasource")
+                       return reconcile.Result{}, err
+               }
+               //remove Finalizer after deletion
+               if visualizationutils.Contains(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer) {
+                       if err := removeFinalizer(reqLogger, instance); err != nil {
+                               return reconcile.Result{}, err
+                       }
+                       err := r.client.Update(context.TODO(), instance)
+                       if err != nil {
+                               reqLogger.Error(err, "Unable to update instance")
+                               return reconcile.Result{}, err
+                       }
+                       return reconcile.Result{}, nil
+               }
+       }
+
+       //Add finalizer for the CR object
+       if !visualizationutils.Contains(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer) {
+               reqLogger.Info("Adding finalizer for GrafanaDatasource")
+               if err := addFinalizer(reqLogger, instance); err != nil {
+                       return reconcile.Result{}, err
+               }
+               err := r.client.Update(context.TODO(), instance)
+               if err != nil {
+                       reqLogger.Error(err, "Unable to update instance")
+                       return reconcile.Result{}, err
+               }
+               return reconcile.Result{}, nil
+       }
+
        datasources := instance.Spec.Datasources
        grafana := instance.Spec.Grafana
 
@@ -110,6 +153,25 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon
                        return reconcile.Result{}, err
                }
 
+               respBody, err := ioutil.ReadAll(getResp.Body)
+               if err != nil {
+                       reqLogger.Error(err, "Response data not read properly")
+                       return reconcile.Result{}, err
+               }
+
+               respBodyBytes := []byte(respBody)
+               var respData map[string]interface{}
+
+               if err := json.Unmarshal(respBodyBytes, &respData); err != nil {
+                       reqLogger.Error(err, "JSON unmarshalling error")
+                       return reconcile.Result{}, err
+               }
+
+               respURL := fmt.Sprintf("%v", respData["url"])
+               respID := fmt.Sprintf("%v", respData["id"])
+               respIsDefault := respData["isDefault"]
+               respAccess := fmt.Sprintf("%v", respData["access"])
+
                defer getResp.Body.Close()
 
                //add datasource if datasource does not exist already
@@ -120,19 +182,24 @@ func (r *ReconcileGrafanaDataSource) Reconcile(request reconcile.Request) (recon
                                return reconcile.Result{}, err
                        }
                } else if getResp.StatusCode == http.StatusOK {
-                       //if datasource already exists
+                       //if datasource already exists and there is any change in the spec - update it
                        reqLogger.V(1).Info("datasource already exists", "datasource", datasource.Name)
-               } else {
-                       reqLogger.Error(err, "unknown error", datasource.Name)
+                       if respURL != datasource.URL || respIsDefault.(bool) != datasource.IsDefault || respAccess != datasource.Access {
+                               if err := updateDatasource(grafana, datasource, respID); err != nil {
+                                       return reconcile.Result{}, err
+                               }
+                       } else {
+                               reqLogger.Info("No creation/updation of datasource needed")
+                               return reconcile.Result{}, nil
+                       }
                }
-
        }
        return reconcile.Result{}, nil
 }
 
 func createDataSource(grafana map[string]string, datasource onapv1alpha1.Datasource) error {
        reqLogger := log.WithValues("Datasource name", datasource.Name, "Datasource URL", datasource.URL)
-       reqLogger.Info("creating datasource")
+       reqLogger.Info("Creating datasource")
 
        grafanaURL := grafana["url"] + "/api/datasources"
        grafanaUsername := grafana["username"]
@@ -145,19 +212,19 @@ func createDataSource(grafana map[string]string, datasource onapv1alpha1.Datasou
                return err
        }
 
-       req, err := http.NewRequest("POST", grafanaURL, bytes.NewBuffer(postBody))
+       postReq, err := http.NewRequest("POST", grafanaURL, bytes.NewBuffer(postBody))
        if err != nil {
                reqLogger.Error(err, "POST REQUEST error")
                return err
        }
-       req.Header.Set("Content-Type", "application/json")
-       req.SetBasicAuth(grafanaUsername, grafanaPassword)
-       postResp, err := client.Do(req)
+       postReq.Header.Set("Content-Type", "application/json")
+       postReq.SetBasicAuth(grafanaUsername, grafanaPassword)
+       postResp, err := client.Do(postReq)
        if err != nil {
                reqLogger.Error(err, "POST RESPONSE error")
                return err
        }
-       defer req.Body.Close()
+       defer postReq.Body.Close()
 
        if postResp.StatusCode == http.StatusOK {
                reqLogger.Info("Datasource created")
@@ -165,3 +232,95 @@ func createDataSource(grafana map[string]string, datasource onapv1alpha1.Datasou
        }
        return err
 }
+
+func updateDatasource(grafana map[string]string, datasource onapv1alpha1.Datasource, datasourceID string) error {
+       reqLogger := log.WithValues("Datasource name", datasource.Name, "Datasource URL", datasource.URL)
+       reqLogger.Info("Updating datasource")
+
+       grafanaURL := grafana["url"] + "/api/datasources/" + datasourceID
+       grafanaUsername := grafana["username"]
+       grafanaPassword := grafana["password"]
+
+       client := &http.Client{}
+       putBody, err := json.Marshal(datasource)
+       if err != nil {
+               reqLogger.Error(err, "JSON Marshalling error")
+               return err
+       }
+       putReq, err := http.NewRequest("PUT", grafanaURL, bytes.NewBuffer(putBody))
+       if err != nil {
+               reqLogger.Error(err, "PUT REQUEST error")
+               return err
+       }
+       putReq.Header.Set("Content-Type", "application/json")
+       putReq.Header.Set("Accept", "application/json")
+       putReq.SetBasicAuth(grafanaUsername, grafanaPassword)
+
+       putResp, err := client.Do(putReq)
+       if err != nil {
+               reqLogger.Error(err, "PUT RESPONSE error")
+               return err
+       }
+       defer putReq.Body.Close()
+
+       if putResp.StatusCode == http.StatusOK {
+               reqLogger.Info("Datasource updated")
+               return nil
+       }
+       return err
+}
+
+func deleteDatasource(instance *onapv1alpha1.GrafanaDataSource) error {
+
+       datasources := instance.Spec.Datasources
+       grafana := instance.Spec.Grafana
+
+       for _, datasource := range datasources {
+
+               reqLogger := log.WithValues("Datasource name", datasource.Name, "Datasource URL", datasource.URL)
+               reqLogger.Info("Deleting datasource")
+
+               grafanaURL := grafana["url"] + "/api/datasources/name/" + datasource.Name
+               grafanaUsername := grafana["username"]
+               grafanaPassword := grafana["password"]
+
+               client := &http.Client{}
+               deleteReq, err := http.NewRequest("DELETE", grafanaURL, nil)
+               if err != nil {
+                       reqLogger.Error(err, "DELETE request error")
+                       return err
+               }
+
+               deleteReq.SetBasicAuth(grafanaUsername, grafanaPassword)
+
+               deleteResp, err := client.Do(deleteReq)
+               if err != nil {
+                       reqLogger.Error(err, "DELETE RESPONSE error")
+                       return err
+               }
+
+               if deleteResp.StatusCode == http.StatusOK {
+                       reqLogger.Info("Datasource deleted")
+                       return nil
+               }
+               return err
+       }
+       return nil
+}
+
+func checkDeletionTimestamp(reqlogger logr.Logger, instance *onapv1alpha1.GrafanaDataSource) bool {
+       isMarkedForDeletion := instance.GetDeletionTimestamp() != nil
+       return isMarkedForDeletion
+}
+
+func addFinalizer(reqlogger logr.Logger, instance *onapv1alpha1.GrafanaDataSource) error {
+       reqlogger.Info("Adding finalizer for the GrafanaDatasource")
+       instance.SetFinalizers(append(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer))
+       return nil
+}
+
+func removeFinalizer(reqlogger logr.Logger, instance *onapv1alpha1.GrafanaDataSource) error {
+       reqlogger.Info("Removing finalizer for the GrafanaDatasource")
+       instance.SetFinalizers(visualizationutils.Remove(instance.GetFinalizers(), visualizationutils.VisualizationFinalizer))
+       return nil
+}
diff --git a/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go b/vnfs/DAaaS/microservices/visualization-operator/pkg/controller/utils/visualizationutils.go
new file mode 100644 (file)
index 0000000..dc7e1f1
--- /dev/null
@@ -0,0 +1,26 @@
+package visualizationutils
+
+// Define the GrafanaDatasource finalizer for handling deletion
+const (
+       VisualizationFinalizer = "finalizer.visualization.onap.org"
+)
+
+// Contains checks if a string is contained in a list of strings
+func Contains(list []string, s string) bool {
+       for _, v := range list {
+               if v == s {
+                       return true
+               }
+       }
+       return false
+}
+
+// Remove checks and removes a string from a list of strings
+func Remove(list []string, s string) []string {
+       for i, v := range list {
+               if v == s {
+                       list = append(list[:i], list[i+1:]...)
+               }
+       }
+       return list
+}