vnfs/DAaaS/prometheus-operator/hack/sync_grafana_dashboards.py

   1 #!/usr/bin/env python3
   2 """Fetch dashboards from provided urls into this chart."""
   3 import json
   4 import textwrap
   5 from os import makedirs, path
   6
   7 import requests
   8 import yaml
   9 from yaml.representer import SafeRepresenter
  10
  11
  12 # https://stackoverflow.com/a/20863889/961092
  13 class LiteralStr(str):
  14     pass
  15
  16
  17 def change_style(style, representer):
  18     def new_representer(dumper, data):
  19         scalar = representer(dumper, data)
  20         scalar.style = style
  21         return scalar
  22
  23     return new_representer
  24
  25
  26 # Source files list
  27 charts = [
  28     {
  29         'source': 'https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml',
  30         'destination': '../templates/grafana/dashboards',
  31         'type': 'yaml',
  32     },
  33     {
  34         'source': 'https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json',
  35         'destination': '../templates/grafana/dashboards',
  36         'type': 'json',
  37     },
  38     {
  39         'source': 'https://raw.githubusercontent.com/helm/charts/master/stable/prometheus-operator/dashboards/grafana-coredns-k8s.json',
  40         'destination': '../templates/grafana/dashboards',
  41         'type': 'json',
  42     },
  43 ]
  44
  45 # Additional conditions map
  46 condition_map = {
  47     'grafana-coredns-k8s': ' .Values.coreDns.enabled',
  48     'etcd': ' .Values.kubeEtcd.enabled',
  49 }
  50
  51 # standard header
  52 header = '''# Generated from '%(name)s' from %(url)s
  53 {{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled%(condition)s }}
  54 apiVersion: v1
  55 kind: ConfigMap
  56 metadata:
  57   name: {{ printf "%%s-%%s" (include "prometheus-operator.fullname" $) "%(name)s" | trunc 63 | trimSuffix "-" }}
  58   labels:
  59     {{- if $.Values.grafana.sidecar.dashboards.label }}
  60     {{ $.Values.grafana.sidecar.dashboards.label }}: "1"
  61     {{- end }}
  62     app: {{ template "prometheus-operator.name" $ }}-grafana
  63 {{ include "prometheus-operator.labels" $ | indent 4 }}
  64 data:
  65 '''
  66
  67
  68 def init_yaml_styles():
  69     represent_literal_str = change_style('|', SafeRepresenter.represent_str)
  70     yaml.add_representer(LiteralStr, represent_literal_str)
  71
  72
  73 def escape(s):
  74     return s.replace("{{", "{{`{{").replace("}}", "}}`}}")
  75
  76
  77 def yaml_str_repr(struct, indent=2):
  78     """represent yaml as a string"""
  79     text = yaml.dump(
  80         struct,
  81         width=1000,  # to disable line wrapping
  82         default_flow_style=False  # to disable multiple items on single line
  83     )
  84     text = escape(text)  # escape {{ and }} for helm
  85     text = textwrap.indent(text, ' ' * indent)
  86     return text
  87
  88
  89 def write_group_to_file(resource_name, content, url, destination):
  90     # initialize header
  91     lines = header % {
  92         'name': resource_name,
  93         'url': url,
  94         'condition': condition_map.get(resource_name, ''),
  95     }
  96
  97     filename_struct = {resource_name + '.json': (LiteralStr(content))}
  98     # rules themselves
  99     lines += yaml_str_repr(filename_struct)
 100
 101     # footer
 102     lines += '{{- end }}'
 103
 104     filename = resource_name + '.yaml'
 105     new_filename = "%s/%s" % (destination, filename)
 106
 107     # make sure directories to store the file exist
 108     makedirs(destination, exist_ok=True)
 109
 110     # recreate the file
 111     with open(new_filename, 'w') as f:
 112         f.write(lines)
 113
 114     print("Generated %s" % new_filename)
 115
 116
 117 def main():
 118     init_yaml_styles()
 119     # read the rules, create a new template file per group
 120     for chart in charts:
 121         print("Generating rules from %s" % chart['source'])
 122         raw_text = requests.get(chart['source']).text
 123         if chart['type'] == 'yaml':
 124             yaml_text = yaml.load(raw_text)
 125             groups = yaml_text['items']
 126             for group in groups:
 127                 for resource, content in group['data'].items():
 128                     write_group_to_file(resource.replace('.json', ''), content, chart['source'], chart['destination'])
 129         elif chart['type'] == 'json':
 130             json_text = json.loads(raw_text)
 131             # is it already a dashboard structure or is it nested (etcd case)?
 132             flat_structure = bool(json_text.get('annotations'))
 133             if flat_structure:
 134                 resource = path.basename(chart['source']).replace('.json', '')
 135                 write_group_to_file(resource, json.dumps(json_text, indent=4), chart['source'], chart['destination'])
 136             else:
 137                 for resource, content in json_text.items():
 138                     write_group_to_file(resource.replace('.json', ''), json.dumps(content, indent=4), chart['source'], chart['destination'])
 139     print("Finished")
 140
 141
 142 if __name__ == '__main__':
 143     main()