2 # -*- coding: utf-8 -*-
4 # COPYRIGHT NOTICE STARTS HERE
6 # Copyright 2019 © Samsung Electronics Co., Ltd.
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
10 # You may obtain a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS,
16 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 # See the License for the specific language governing permissions and
18 # limitations under the License.
20 # COPYRIGHT NOTICE ENDS HERE
23 from __future__ import print_function
28 from subprocess import Popen,STDOUT,PIPE,check_output
30 from time import sleep
31 from os.path import expanduser
32 from itertools import chain
34 from requests.packages.urllib3.exceptions import InsecureRequestWarning
35 from base64 import b64decode
36 from tempfile import NamedTemporaryFile
38 def add_resource_kind(resources, kind):
39 for item in resources:
43 def pods_by_parent(pods, parent):
45 if pod['metadata']['labels']['app'] == parent:
48 def k8s_controller_ready(k8s_controller):
49 if k8s_controller['kind'] == 'Job':
50 return k8s_controller['status'].get('succeeded', 0) == k8s_controller['spec']['completions']
51 return k8s_controller['status'].get('readyReplicas', 0) == k8s_controller['spec']['replicas']
53 def get_not_ready(data):
54 return [x for x in data if not k8s_controller_ready(x)]
57 return [x['metadata']['labels']['app'] for x in data]
60 return [x['metadata']['name'] for x in data]
64 return [x['status'] for x in pod['status']['conditions']
65 if x['type'] == 'Ready'][0] == 'True'
66 except (KeyError, IndexError):
69 def not_ready_pods(pods):
71 if not pod_ready(pod):
74 def analyze_k8s_controllers(resources_data):
75 resources = {'total_count': len(resources_data)}
76 resources['not_ready_list'] = get_apps(get_not_ready(resources_data))
77 resources['ready_count'] = resources['total_count'] - len(resources['not_ready_list'])
81 def get_k8s_controllers(k8s):
84 k8s_controllers['deployments'] = {'data': k8s.get_resources(
85 'apis/apps/v1', 'deployments')}
86 k8s_controllers['deployments'].update(analyze_k8s_controllers(
87 k8s_controllers['deployments']['data']))
89 k8s_controllers['statefulsets'] = {'data': k8s.get_resources(
90 'apis/apps/v1', 'statefulsets')}
91 k8s_controllers['statefulsets'].update(analyze_k8s_controllers(
92 k8s_controllers['statefulsets']['data']))
94 k8s_controllers['jobs'] = {'data': k8s.get_resources(
95 'apis/batch/v1', 'jobs')}
96 k8s_controllers['jobs'].update(analyze_k8s_controllers(
97 k8s_controllers['jobs']['data']))
99 not_ready_controllers = chain.from_iterable(
100 k8s_controllers[x]['not_ready_list'] for x in k8s_controllers)
102 return k8s_controllers, list(not_ready_controllers)
104 def exec_healthcheck(hp_script, namespace, hp_mode):
105 # spawn healthcheck script and redirect it's stderr to stdout
106 hc = Popen(['sh',hp_script,namespace,hp_mode],stdout=PIPE,stderr=STDOUT)
107 # Trace the output of subprocess until it has finished
108 for line in iter(hc.stdout.readline, ''):
110 hc.poll() # set returncode in Popen object
113 def check_readiness(k8s, verbosity):
114 k8s_controllers, not_ready_controllers = get_k8s_controllers(k8s)
116 # check pods only when it is explicitly wanted (judging readiness by deployment status)
118 pods = k8s.get_resources('api/v1', 'pods')
119 unready_pods = chain.from_iterable(
120 get_names(not_ready_pods(
121 pods_by_parent(pods, x)))
122 for x in not_ready_controllers)
126 print_status(verbosity, k8s_controllers, unready_pods)
127 return not not_ready_controllers
129 def check_in_loop(k8s, max_time, sleep_time, verbosity):
130 max_end_time = datetime.datetime.now() + datetime.timedelta(minutes=max_time)
132 while datetime.datetime.now() < max_end_time:
133 ready = check_readiness(k8s, verbosity)
139 def check_helm_releases():
140 helm = check_output(['helm', 'ls'])
142 sys.exit('No Helm releases detected.')
143 helm_releases = csv.DictReader(
144 map(lambda x: x.replace(' ', ''), helm.split('\n')),
146 failed_releases = [release['NAME'] for release in helm_releases
147 if release['STATUS'] == 'FAILED']
148 return helm, failed_releases
151 def create_ready_string(ready, total, prefix):
152 return '{:12} {}/{}'.format(prefix, ready, total)
154 def print_status(verbosity, resources, not_ready_pods):
156 ready = {k: v['ready_count'] for k,v in resources.items()}
157 count = {k: v['total_count'] for k,v in resources.items()}
160 create_ready_string(ready[k], count[k], k.capitalize()) for k in ready
162 total_ready = sum(ready.values())
163 total_count = sum(count.values())
164 ready_strings.append(create_ready_string(total_ready, total_count, 'Ready'))
165 status_strings = ['\n'.join(ready_strings)]
168 status_strings.append('\nWaiting for pods:\n{}'.format('\n'.join(not_ready_pods)))
170 status_strings.append('\nAll pods are ready!')
171 print('\n'.join(status_strings), '\n')
174 parser = argparse.ArgumentParser(description='Monitor ONAP deployment progress',
175 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
176 parser.add_argument('--namespace', '-n', default='onap',
177 help='Kubernetes namespace of ONAP')
178 parser.add_argument('--server', '-s', help='address of Kubernetes cluster')
179 parser.add_argument('--kubeconfig', '-c',
180 default=expanduser('~') + '/.kube/config',
181 help='path to .kube/config file')
182 parser.add_argument('--health-path', '-hp', help='path to ONAP robot ete-k8s.sh')
183 parser.add_argument('--health-mode', '-hm', default='health', help='healthcheck mode')
184 parser.add_argument('--no-helm', action='store_true', help='Do not check Helm')
185 parser.add_argument('--check-frequency', '-w', default=300, type=int,
186 help='time between readiness checks in seconds')
187 parser.add_argument('--max-time', '-t', default=120, type=int,
188 help='max time to run readiness checks in minutes')
189 parser.add_argument('--single-run', '-1', action='store_true',
190 help='run check loop only once')
191 parser.add_argument('-v', dest='verbosity', action='count', default=0,
192 help='increase output verbosity, e.g. -vv is more verbose than -v')
193 parser.add_argument('--no-ssl-auth', action='store_true',
194 help='Disable SSL certificate based authentication while connecting to server')
196 return parser.parse_args()
199 '''Class exposing get_resources() routine for connecting to kube API.
200 It keeps all attributes required by that call as an internal
203 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
205 def __init__(self,args):
207 self.config = args.kubeconfig
208 self.url = args.server if args.server is not None else \
210 self.no_ssl_auth = args.no_ssl_auth
211 self.certs = self._get_k8s_certs() if not self.no_ssl_auth else {}
212 self.namespace = args.namespace
214 # Setup tmp file with ca chain only if certs were gathered successfully
215 # and --no-ssl-auth wasn't set
216 if self.certs and not self.no_ssl_auth:
217 self._setup_cert_files()
219 def get_resources(self, api, kind):
220 '''Performs actual API call'''
221 url = '/'.join([self.url, api, 'namespaces', self.namespace, kind])
224 req = requests.get(url, verify=False)
226 req = requests.get(url, verify=self.crt_tmp_file.name, cert=self.crt_tmp_file.name)
227 except requests.exceptions.ConnectionError:
228 sys.exit('Error: Could not connect to {}'.format(self.url))
229 if req.status_code == 200:
231 # kind is <resource>List in response so [:-4] removes 'List' from value
232 return add_resource_kind(json['items'], json['kind'][:-4])
233 elif (req.status_code == 401):
234 sys.exit('Error: Server replied with "401 Unauthorized" while making connection')
236 sys.exit("Error: There's been an unspecified issue while making a request to the API")
238 def _setup_cert_files(self):
239 '''Helper funtion to setup named file for requests.get() call
240 in self.get_resources() which is able read certificate only
242 ca_chain = NamedTemporaryFile()
243 for crt in self.certs.values():
245 ca_chain.read() # flush the file buffer
246 self.crt_tmp_file = ca_chain
248 def _get_k8s_url(self):
249 # TODO: Get login info
250 with open(self.config) as f:
251 config = yaml.load(f)
252 # TODO: Support cluster by name
253 return config['clusters'][0]['cluster']['server']
255 def _get_k8s_certs(self):
256 '''Helper function to read and decode certificates from kube config'''
257 with open(self.config) as f:
258 config = yaml.load(f)
261 certs.update(dict(ca_cert=b64decode(
262 config['clusters'][0]['cluster']['certificate-authority-data'])))
263 certs.update(dict(client_cert=b64decode(
264 config['users'][0]['user']['client-certificate-data'])))
265 certs.update(dict(client_key=b64decode(
266 config['users'][0]['user']['client-key-data'])))
268 print('Warning: could not get Kubernetes config for certificates. ' \
269 'Turning off SSL authentication.')
270 self.no_ssl_auth = True
278 helm_output, failed_releases = check_helm_releases()
280 print('Deployment of {} failed.'.format(','.join(failed_releases)))
282 elif args.verbosity > 1:
284 except IOError as err:
285 sys.exit(err.strerror)
287 k8s = Kubernetes(args)
291 ready = check_readiness(k8s, args.verbosity)
293 if not check_in_loop(k8s, args.max_time, args.check_frequency, args.verbosity):
294 # Double-check last 5 minutes and write verbosely in case it is not ready
295 ready = check_readiness(k8s, 2)
297 if args.health_path is not None:
298 hc_rc = exec_healthcheck(args.health_path, args.namespace, args.health_mode)
303 sys.exit('Deployment is not ready')
305 if __name__ == '__main__':