kubernetes/sdnc/components/sdnc-prom/resources/bin/sdnc.monitor

   1 #!/usr/bin/env python3
   2 {{/*
   3
   4 # encoding: utf-8
   5
   6 # Copyright © 2018 Amdocs
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License");
   9 # you may not use this file except in compliance with the License.
  10 # You may obtain a copy of the License at
  11 #
  12 #       http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS,
  16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 # See the License for the specific language governing permissions and
  18 # limitations under the License.
  19 */}}
  20
  21 import sys
  22 import os
  23 import json
  24 import requests
  25 from datetime import datetime
  26
  27 consul_server = "consul-server:8500"
  28 message_router = "message-router:3904"
  29 topic = '{{.Values.config.messageRouterTopic}}'
  30 log_file='/app/monitor.log'
  31 status_file='/app/.health'
  32 logEnabled=False
  33
  34 siteName='sdnc01'
  35 if os.environ.get('SDNC_IS_PRIMARY_CLUSTER', 'true') == 'false':
  36     siteName='sdnc02'
  37
  38 debug=False
  39 if len(sys.argv) > 1 and sys.argv[1] == '--debug':
  40     debug=True
  41
  42 def get_state(healthcheck):
  43     response = requests.get("http://" + consul_server + "/v1/health/checks/" + healthcheck)
  44     if response.status_code != 200:
  45         raise RuntimeError("HTTP " + str(response.status_code))
  46     data = response.json()
  47     if len(data) == 0:
  48         raise RuntimeError(healthcheck + " not found")
  49     if len(data) > 1:
  50         raise RuntimeError("Multiple states for " + healthcheck + " found")
  51
  52     return data[0]
  53
  54
  55 def log(message):
  56     if logEnabled:
  57         with open(log_file, 'a') as f:
  58             f.write(str(datetime.now()) + " " + message + "\n")
  59
  60 def healthcheck(checks, failFirst=True):
  61     if len(checks) == 0:
  62         return True
  63
  64     for check in checks:
  65         if type(check) is list:
  66             passing = healthcheck(check, False)
  67         else:
  68             state = get_state(check)
  69             status = state['Status']
  70             passing = status == "passing" or status == "warning"
  71             log(check + " " + status)
  72             if debug:
  73                 if status == "passing":
  74                     color = "\033[32m" # green
  75                 elif status == "warning":
  76                     color = "\033[33m" # yellow
  77                 else:
  78                     color = "\033[31m" # red
  79                 print check, color + status + "\033[0m"
  80                 if not passing:
  81                     print "\tCause:", state['Output']
  82
  83
  84         if passing:
  85             if not failFirst:
  86                 # found a passing check so can stop here
  87                 return True
  88         else:
  89             if failFirst:
  90                 # found a failing check so can stop here
  91                 return False
  92
  93     return failFirst
  94
  95
  96 try:
  97     with open("/app/config/healthchecks.json") as f:
  98         checks = json.load(f)
  99
 100     try:
 101         with open(status_file) as f:
 102            previous_result = f.read()
 103     except IOError:
 104         # file doesn't exist
 105         previous_result = 'unknown'
 106
 107     if healthcheck(checks):
 108         result = "healthy"
 109     else:
 110         result = "unhealthy"
 111
 112     print result
 113
 114     # save current result to file
 115     with open(status_file, 'w') as f:
 116         f.write(result)
 117
 118     if previous_result != 'unknown' and result != previous_result:
 119         payload = { 'type' : 'health-change', 'status': result, 'site': siteName, 'deployment': '{{.Values.config.deployment}}', 'timestamp': str(datetime.now())  }
 120         log("Posting event " + str(payload))
 121         try:
 122             requests.post("http://" + message_router + "/events/" + topic, data=json.dumps(payload), headers={ 'Content-Type' : 'application/json' } )
 123         except Exception:
 124             # events are best-effort
 125             pass
 126
 127 except Exception as e:
 128     sys.exit(str(e))