src/onaptests/steps/cloud/check_status.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # SPDX-License-Identifier: Apache-2.0
   4 import json
   5 import logging
   6 import os
   7 import re
   8 from pathlib import Path
   9
  10 from jinja2 import Environment, PackageLoader, select_autoescape
  11 from kubernetes import client, config
  12 from kubernetes.stream import stream
  13 from natural.date import delta
  14 from urllib3.exceptions import MaxRetryError, NewConnectionError
  15 from xtesting.core import testcase
  16
  17 from onapsdk.configuration import settings
  18 from onaptests.utils.exceptions import StatusCheckException
  19
  20 from ..base import BaseStep
  21 from .resources import (ConfigMap, Container, DaemonSet, Deployment, Ingress,
  22                         Job, Pod, Pvc, ReplicaSet, Secret, Service,
  23                         StatefulSet)
  24
  25 NAMESPACE = settings.K8S_ONAP_NAMESPACE
  26
  27
  28 class CheckK8sResourcesStep(BaseStep):
  29
  30     __logger = logging.getLogger(__name__)
  31
  32     def __init__(self, resource_type: str, **kwargs):
  33         """Init CheckK8sResourcesStep."""
  34         super().__init__(cleanup=False)
  35         self.core = client.CoreV1Api()
  36         self.batch = client.BatchV1Api()
  37         self.app = client.AppsV1Api()
  38         self.networking = client.NetworkingV1Api()
  39
  40         if settings.STATUS_RESULTS_DIRECTORY:
  41             self.res_dir = f"{settings.STATUS_RESULTS_DIRECTORY}"
  42         else:
  43             self.res_dir = f"{testcase.TestCase.dir_results}/kubernetes-status"
  44
  45         self.failing = False
  46         self.resource_type = resource_type
  47         self.k8s_resources = []
  48         self.all_resources = []
  49         self.failing_resources = []
  50         self.jinja_env = Environment(autoescape=select_autoescape(['html']),
  51                                      loader=PackageLoader('onaptests.templates', 'status'))
  52
  53     @property
  54     def component(self) -> str:
  55         """Component name."""
  56         return "ALL"
  57
  58     @property
  59     def description(self) -> str:
  60         """Step description."""
  61         return f"Check status of all k8s {self.resource_type}s in the {NAMESPACE} namespace."
  62
  63     def _init_resources(self):
  64         self.__logger.debug(f"Loading all k8s {self.resource_type}s in the {NAMESPACE} namespace")
  65
  66     def _parse_resources(self):
  67         """Parse the resources."""
  68         return []
  69
  70     def _add_failing_resource(self, resource):
  71         if (resource.labels and settings.EXCLUDED_LABELS
  72                 and (resource.labels.keys() and settings.EXCLUDED_LABELS.keys())):
  73             for label in resource.labels.items():
  74                 for waived_label in settings.EXCLUDED_LABELS.items():
  75                     if label[0] in waived_label[0] and label[1] in waived_label[1]:
  76                         return
  77         self.__logger.warning("a {} is in error: {}".format(self.resource_type, resource.name))
  78         self.failing_resources.append(resource)
  79         self.failing = True
  80
  81     def execute(self):
  82         super().execute()
  83         os.makedirs(self.res_dir, exist_ok=True)
  84         try:
  85             self._init_resources()
  86             if len(self.k8s_resources) > 0:
  87                 self.__logger.info("%4s %ss in the namespace",
  88                                    len(self.k8s_resources),
  89                                    self.resource_type)
  90                 self._parse_resources()
  91                 self.__logger.info("%4s %ss parsed, %s failing",
  92                                    len(self.all_resources),
  93                                    self.resource_type,
  94                                    len(self.failing_resources))
  95         except (ConnectionRefusedError, MaxRetryError, NewConnectionError):
  96             self.__logger.error("Test of k8s %ss failed.", self.resource_type)
  97             self.__logger.error("Cannot connect to Kubernetes.")
  98
  99
 100 class CheckBasicK8sResourcesStep(CheckK8sResourcesStep):
 101
 102     __logger = logging.getLogger(__name__)
 103
 104     def __init__(self, resource_type: str, k8s_res_class, cleanup: bool = False, **kwargs):
 105         """Init CheckBasicK8sResourcesStep."""
 106         super().__init__(resource_type)
 107         self.k8s_res_class = k8s_res_class
 108
 109     def _parse_resources(self):
 110         """Parse simple k8s resources."""
 111         super()._parse_resources()
 112         for k8s in self.k8s_resources:
 113             resource = self.k8s_res_class(k8s=k8s)
 114             self.all_resources.append(resource)
 115
 116     @BaseStep.store_state
 117     def execute(self):
 118         super().execute()
 119
 120
 121 class CheckK8sConfigMapsStep(CheckBasicK8sResourcesStep):
 122
 123     __logger = logging.getLogger(__name__)
 124
 125     def __init__(self, cleanup: bool = False, **kwargs):
 126         """Init CheckK8sConfigMapsStep."""
 127         super().__init__("configmap", ConfigMap)
 128
 129     def _init_resources(self):
 130         super()._init_resources()
 131         self.k8s_resources = self.core.list_namespaced_config_map(NAMESPACE).items
 132
 133
 134 class CheckK8sSecretsStep(CheckBasicK8sResourcesStep):
 135
 136     __logger = logging.getLogger(__name__)
 137
 138     def __init__(self, cleanup: bool = False, **kwargs):
 139         """Init CheckK8sSecretsStep."""
 140         super().__init__("secret", Secret)
 141
 142     def _init_resources(self):
 143         super()._init_resources()
 144         self.k8s_resources = self.core.list_namespaced_secret(NAMESPACE).items
 145
 146
 147 class CheckK8sIngressesStep(CheckBasicK8sResourcesStep):
 148
 149     __logger = logging.getLogger(__name__)
 150
 151     def __init__(self, cleanup: bool = False, **kwargs):
 152         """Init CheckK8sIngressesStep."""
 153         super().__init__("ingress", Ingress)
 154
 155     def _init_resources(self):
 156         super()._init_resources()
 157         self.k8s_resources = self.networking.list_namespaced_ingress(NAMESPACE).items
 158
 159
 160 class CheckK8sPvcsStep(CheckK8sResourcesStep):
 161
 162     __logger = logging.getLogger(__name__)
 163
 164     def __init__(self, cleanup: bool = False, **kwargs):
 165         """Init CheckK8sPvcsStep."""
 166         super().__init__("pvc")
 167
 168     def _init_resources(self):
 169         super()._init_resources()
 170         self.k8s_resources = self.core.list_namespaced_persistent_volume_claim(NAMESPACE).items
 171
 172     def _parse_resources(self):
 173         """Parse the jobs.
 174         Return a list of Pods that were created to perform jobs.
 175         """
 176         super()._parse_resources()
 177         for k8s in self.k8s_resources:
 178             pvc = Pvc(k8s=k8s)
 179             field_selector = (f"involvedObject.name={pvc.name},"
 180                               "involvedObject.kind=PersistentVolumeClaim")
 181             pvc.events = self.core.list_namespaced_event(
 182                 NAMESPACE,
 183                 field_selector=field_selector).items
 184
 185             if k8s.status.phase != "Bound":
 186                 self._add_failing_resource(pvc)
 187             self.all_resources.append(pvc)
 188
 189     @BaseStep.store_state
 190     def execute(self):
 191         super().execute()
 192
 193
 194 class CheckK8sResourcesUsingPodsStep(CheckK8sResourcesStep):
 195
 196     __logger = logging.getLogger(__name__)
 197
 198     def __init__(self, resource_type: str, pods_source, cleanup: bool = False, **kwargs):
 199         """Init CheckK8sResourcesUsingPodsStep."""
 200         super().__init__(resource_type)
 201         self.pods_source = pods_source
 202
 203     def _get_used_pods(self):
 204         pods = []
 205         if self.pods_source is not None:
 206             pods = self.pods_source.all_resources
 207         return pods
 208
 209     def _find_child_pods(self, selector):
 210         pods_used = self._get_used_pods()
 211         pods_list = []
 212         failed_pods = 0
 213         if selector:
 214             raw_selector = ''
 215             for key, value in selector.items():
 216                 raw_selector += key + '=' + value + ','
 217             raw_selector = raw_selector[:-1]
 218             pods = self.core.list_namespaced_pod(
 219                 NAMESPACE, label_selector=raw_selector).items
 220             for pod in pods:
 221                 for known_pod in pods_used:
 222                     if known_pod.name == pod.metadata.name:
 223                         pods_list.append(known_pod)
 224                         if not known_pod.ready():
 225                             failed_pods += 1
 226         return (pods_list, failed_pods)
 227
 228     @BaseStep.store_state
 229     def execute(self):
 230         super().execute()
 231
 232
 233 class CheckK8sJobsStep(CheckK8sResourcesUsingPodsStep):
 234
 235     __logger = logging.getLogger(__name__)
 236
 237     def __init__(self, cleanup: bool = False, **kwargs):
 238         """Init CheckK8sJobsStep."""
 239         super().__init__("job", None)
 240
 241     def _init_resources(self):
 242         super()._init_resources()
 243         self.k8s_resources = self.batch.list_namespaced_job(NAMESPACE).items
 244
 245     def _parse_resources(self):
 246         """Parse the jobs.
 247         Return a list of Pods that were created to perform jobs.
 248         """
 249         super()._parse_resources()
 250         jobs_pods = []
 251         for k8s in self.k8s_resources:
 252             job = Job(k8s=k8s)
 253             job_pods = []
 254
 255             if k8s.spec.selector and k8s.spec.selector.match_labels:
 256                 (job.pods, job.failed_pods) = self._find_child_pods(
 257                     k8s.spec.selector.match_labels)
 258                 job_pods += job.pods
 259             field_selector = "involvedObject.name={}".format(job.name)
 260             field_selector += ",involvedObject.kind=Job"
 261             job.events = self.core.list_namespaced_event(
 262                 NAMESPACE,
 263                 field_selector=field_selector).items
 264
 265             self.jinja_env.get_template('job.html.j2').stream(job=job).dump(
 266                 '{}/job-{}.html'.format(self.res_dir, job.name))
 267
 268             # timemout job
 269             if not k8s.status.completion_time:
 270                 if any(waiver_elt not in job.name for waiver_elt in settings.WAIVER_LIST):
 271                     self._add_failing_resource(job)
 272             # completed job
 273             if any(waiver_elt not in job.name for waiver_elt in settings.WAIVER_LIST):
 274                 self.all_resources.append(job)
 275             jobs_pods += job_pods
 276
 277
 278 class CheckK8sPodsStep(CheckK8sResourcesUsingPodsStep):
 279
 280     __logger = logging.getLogger(__name__)
 281
 282     def __init__(self, pods, cleanup: bool = False, **kwargs):
 283         """Init CheckK8sPodsStep."""
 284         super().__init__("pod", pods)
 285
 286     def _init_resources(self):
 287         super()._init_resources()
 288         self.k8s_resources = self.core.list_namespaced_pod(NAMESPACE).items
 289
 290     def _parse_resources(self):  # noqa
 291         """Parse the pods."""
 292         super()._parse_resources()
 293         excluded_pods = self._get_used_pods()
 294         pod_versions = []
 295         containers = {}
 296         for k8s in self.k8s_resources:
 297             pod = Pod(k8s=k8s)
 298
 299             # check version firstly
 300             if settings.CHECK_POD_VERSIONS:
 301                 pod_component = k8s.metadata.name
 302                 if 'app' in k8s.metadata.labels:
 303                     pod_component = k8s.metadata.labels['app']
 304                 else:
 305                     if 'app.kubernetes.io/name' in k8s.metadata.labels:
 306                         pod_component = k8s.metadata.labels[
 307                             'app.kubernetes.io/name']
 308                     else:
 309                         self.__logger.error("pod %s has no 'app' or 'app.kubernetes.io/name' "
 310                                             "in metadata: %s", pod_component, k8s.metadata.labels)
 311
 312                 # looks for docker version
 313                 for container in k8s.spec.containers:
 314                     pod_version = {}
 315                     pod_container_version = container.image.rsplit(":", 1)
 316                     pod_container_image = pod_container_version[0]
 317                     pod_container_tag = "latest"
 318                     if len(pod_container_version) > 1:
 319                         pod_container_tag = pod_container_version[1]
 320
 321                     pod_version.update({
 322                         'container': container.name,
 323                         'component': pod_component,
 324                         'image': pod_container_image,
 325                         'version': pod_container_tag
 326                     })
 327                     pod_versions.append(pod_version)
 328
 329                     search_rule = "^(?P<source>[^/]*)/*(?P<container>[^:]*):*(?P<version>.*)$"
 330                     search = re.search(search_rule, container.image)
 331                     name = "{}/{}".format(search.group('source'),
 332                                           search.group('container'))
 333                     version = search.group('version')
 334                     if name[-1] == '/':
 335                         name = name[0:-1]
 336                     source = "default"
 337                     if search.group('source') in settings.DOCKER_REPOSITORIES:
 338                         source = search.group('source')
 339                         name = search.group('container')
 340                     container_search_rule = "^library/(?P<real_container>[^:]*)$"
 341                     container_search = re.search(container_search_rule, name)
 342                     if container_search:
 343                         name = container_search.group('real_container')
 344                     for common_component in settings.GENERIC_NAMES.keys():
 345                         if name in settings.GENERIC_NAMES[common_component]:
 346                             version = "{}:{}".format(name, version)
 347                             name = common_component
 348                             break
 349
 350                     repository = settings.DOCKER_REPOSITORIES_NICKNAMES[source]
 351                     if name in containers:
 352                         if version in containers[name]['versions']:
 353                             if not (pod_component in containers[name]['versions']
 354                                     [version]['components']):
 355                                 containers[name]['versions'][version][
 356                                     'components'].append(pod_component)
 357                                 containers[name]['number_components'] += 1
 358                             if not (repository in containers[name]['versions']
 359                                     [version]['repositories']):
 360                                 containers[name]['versions'][version][
 361                                     'repositories'].append(repository)
 362                         else:
 363                             containers[name]['versions'][version] = {
 364                                 'repositories': [repository],
 365                                 'components': [pod_component]
 366                             }
 367                             containers[name]['number_components'] += 1
 368                     else:
 369                         containers[name] = {
 370                             'versions': {
 371                                 version: {
 372                                     'repositories': [repository],
 373                                     'components': [pod_component]
 374                                 }
 375                             },
 376                             'number_components': 1
 377                         }
 378             # pod version check end
 379             if excluded_pods and pod in excluded_pods:
 380                 continue
 381
 382             if k8s.status.init_container_statuses:
 383                 for k8s_container in k8s.status.init_container_statuses:
 384                     pod.runned_init_containers += self._parse_container(
 385                         pod, k8s_container, init=True)
 386             if k8s.status.container_statuses:
 387                 for k8s_container in k8s.status.container_statuses:
 388                     pod.running_containers += self._parse_container(
 389                         pod, k8s_container)
 390             pod.events = self.core.list_namespaced_event(
 391                 NAMESPACE,
 392                 field_selector="involvedObject.name={}".format(pod.name)).items
 393             self.jinja_env.get_template('pod.html.j2').stream(pod=pod).dump(
 394                 '{}/pod-{}.html'.format(self.res_dir, pod.name))
 395             if any(waiver_elt in pod.name for waiver_elt in settings.WAIVER_LIST):
 396                 self.__logger.warn("Waiver pattern found in pod, exclude %s", pod.name)
 397             else:
 398                 self.all_resources.append(pod)
 399
 400         if settings.CHECK_POD_VERSIONS:
 401             self.jinja_env.get_template('version.html.j2').stream(
 402                 pod_versions=pod_versions).dump('{}/versions.html'.format(
 403                     self.res_dir))
 404             self.jinja_env.get_template('container_versions.html.j2').stream(
 405                 containers=containers).dump('{}/container_versions.html'.format(
 406                     self.res_dir))
 407             # create a json file for version tracking
 408             with open(self.res_dir + "/onap_versions.json", "w") as write_file:
 409                 json.dump(pod_versions, write_file)
 410
 411     def _get_container_logs(self, pod, container, full=True, previous=False):
 412         logs = ""
 413         limit_bytes = settings.MAX_LOG_BYTES
 414         if full:
 415             limit_bytes = settings.UNLIMITED_LOG_BYTES
 416         try:
 417             logs = self.core.read_namespaced_pod_log(
 418                 pod.name,
 419                 NAMESPACE,
 420                 container=container.name,
 421                 limit_bytes=limit_bytes,
 422                 previous=previous
 423             )
 424         except UnicodeDecodeError:
 425             logs = "{0} has an unicode decode error...".format(pod.name)
 426             self.__logger.error(
 427                 "{0} has an unicode decode error in the logs...", pod.name,
 428             )
 429         return logs
 430
 431     def _parse_container(self, pod, k8s_container, init=False):  # noqa
 432         """Get the logs of a container."""
 433         logs = ""
 434         old_logs = ""
 435         prefix = ""
 436         containers_list = pod.containers
 437         container = Container(name=k8s_container.name)
 438         container.restart_count = k8s_container.restart_count
 439         container.set_status(k8s_container.state)
 440         container.ready = k8s_container.ready
 441         container.image = k8s_container.image
 442         if init:
 443             prefix = "init "
 444             containers_list = pod.init_containers
 445             if container.restart_count > pod.init_restart_count:
 446                 pod.init_restart_count = container.restart_count
 447             if not container.ready:
 448                 pod.init_done = False
 449         else:
 450             if container.restart_count > pod.restart_count:
 451                 pod.restart_count = container.restart_count
 452         if settings.STORE_ARTIFACTS:
 453             try:
 454                 log_files = {}
 455                 logs = self._get_container_logs(pod=pod, container=container, full=False)
 456                 with open(
 457                         "{}/pod-{}-{}.log".format(self.res_dir,
 458                                                   pod.name, container.name),
 459                         'w') as log_result:
 460                     log_result.write(logs)
 461                 if (not container.ready) and container.restart_count > 0:
 462                     old_logs = self._get_container_logs(pod=pod, container=container,
 463                                                         previous=True)
 464                     with open(
 465                             "{}/pod-{}-{}.old.log".format(self.res_dir,
 466                                                           pod.name,
 467                                                           container.name),
 468                             'w') as log_result:
 469                         log_result.write(old_logs)
 470                 if (container.name in settings.FULL_LOGS_CONTAINERS):
 471                     logs = self._get_container_logs(pod=pod, container=container)
 472                     with open(
 473                             "{}/pod-{}-{}.log".format(self.res_dir,
 474                                                       pod.name, container.name),
 475                             'w') as log_result:
 476                         log_result.write(logs)
 477                 if (container.name in settings.SPECIFIC_LOGS_CONTAINERS):
 478                     for log_file in settings.SPECIFIC_LOGS_CONTAINERS[container.name]:
 479                         exec_command = ['/bin/sh', '-c', "cat {}".format(log_file)]
 480                         log_files[log_file] = stream(
 481                             self.core.connect_get_namespaced_pod_exec,
 482                             pod.name,
 483                             NAMESPACE,
 484                             container=container.name,
 485                             command=exec_command,
 486                             stderr=True,
 487                             stdin=False,
 488                             stdout=True,
 489                             tty=False)
 490                         log_file_slug = log_file.split('.')[0].split('/')[-1]
 491                         with open(
 492                                 "{}/pod-{}-{}-{}.log".format(
 493                                     self.res_dir, pod.name,
 494                                     container.name, log_file_slug),
 495                                 'w') as log_result:
 496                             log_result.write(log_files[log_file])
 497             except client.rest.ApiException as exc:
 498                 self.__logger.warning("%scontainer %s of pod %s has an exception: %s",
 499                                       prefix, container.name, pod.name, exc.reason)
 500             self.jinja_env.get_template('container_log.html.j2').stream(
 501                 container=container,
 502                 pod_name=pod.name,
 503                 logs=logs,
 504                 old_logs=old_logs,
 505                 log_files=log_files).dump('{}/pod-{}-{}-logs.html'.format(
 506                     self.res_dir, pod.name, container.name))
 507         if any(waiver_elt in container.name for waiver_elt in settings.WAIVER_LIST):
 508             self.__logger.warn(
 509                 "Waiver pattern found in container, exclude %s", container.name)
 510         else:
 511             containers_list.append(container)
 512             if k8s_container.ready:
 513                 return 1
 514         return 0
 515
 516
 517 class CheckK8sServicesStep(CheckK8sResourcesUsingPodsStep):
 518
 519     __logger = logging.getLogger(__name__)
 520
 521     def __init__(self, pods, cleanup: bool = False, **kwargs):
 522         """Init CheckK8sServicesStep."""
 523         super().__init__("service", pods)
 524
 525     def _init_resources(self):
 526         super()._init_resources()
 527         self.k8s_resources = self.core.list_namespaced_service(NAMESPACE).items
 528
 529     def _parse_resources(self):
 530         """Parse the services."""
 531         super()._parse_resources()
 532         for k8s in self.k8s_resources:
 533             service = Service(k8s=k8s)
 534
 535             (service.pods,
 536              service.failed_pods) = self._find_child_pods(k8s.spec.selector)
 537
 538             self.jinja_env.get_template('service.html.j2').stream(
 539                 service=service).dump('{}/service-{}.html'.format(
 540                     self.res_dir, service.name))
 541             self.all_resources.append(service)
 542
 543
 544 class CheckK8sDeploymentsStep(CheckK8sResourcesUsingPodsStep):
 545
 546     __logger = logging.getLogger(__name__)
 547
 548     def __init__(self, pods, cleanup: bool = False, **kwargs):
 549         """Init CheckK8sDeploymentsStep."""
 550         super().__init__("deployment", pods)
 551
 552     def _init_resources(self):
 553         super()._init_resources()
 554         self.k8s_resources = self.app.list_namespaced_deployment(NAMESPACE).items
 555
 556     def _parse_resources(self):
 557         """Parse the deployments."""
 558         super()._parse_resources()
 559         for k8s in self.k8s_resources:
 560             deployment = Deployment(k8s=k8s)
 561
 562             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 563                 continue
 564             if k8s.spec.selector and k8s.spec.selector.match_labels:
 565                 (deployment.pods,
 566                  deployment.failed_pods) = self._find_child_pods(
 567                      k8s.spec.selector.match_labels)
 568             field_selector = "involvedObject.name={}".format(deployment.name)
 569             field_selector += ",involvedObject.kind=Deployment"
 570             deployment.events = self.core.list_namespaced_event(
 571                 NAMESPACE,
 572                 field_selector=field_selector).items
 573
 574             self.jinja_env.get_template('deployment.html.j2').stream(
 575                 deployment=deployment).dump('{}/deployment-{}.html'.format(
 576                     self.res_dir, deployment.name))
 577
 578             if k8s.status.unavailable_replicas:
 579                 self._add_failing_resource(deployment)
 580
 581             self.all_resources.append(deployment)
 582
 583
 584 class CheckK8sResplicaSetsStep(CheckK8sResourcesUsingPodsStep):
 585
 586     __logger = logging.getLogger(__name__)
 587
 588     def __init__(self, pods, cleanup: bool = False, **kwargs):
 589         """Init CheckK8sResplicaSetsStep."""
 590         super().__init__("replicaset", pods)
 591
 592     def _init_resources(self):
 593         super()._init_resources()
 594         self.k8s_resources = self.app.list_namespaced_replica_set(NAMESPACE).items
 595
 596     def _parse_resources(self):
 597         """Parse the replicasets."""
 598         super()._parse_resources()
 599         for k8s in self.k8s_resources:
 600             replicaset = ReplicaSet(k8s=k8s)
 601
 602             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 603                 continue
 604
 605             if k8s.spec.selector and k8s.spec.selector.match_labels:
 606                 (replicaset.pods,
 607                  replicaset.failed_pods) = self._find_child_pods(
 608                      k8s.spec.selector.match_labels)
 609             field_selector = "involvedObject.name={}".format(replicaset.name)
 610             field_selector += ",involvedObject.kind=ReplicaSet"
 611             replicaset.events = self.core.list_namespaced_event(
 612                 NAMESPACE,
 613                 field_selector=field_selector).items
 614
 615             self.jinja_env.get_template('replicaset.html.j2').stream(
 616                 replicaset=replicaset).dump('{}/replicaset-{}.html'.format(
 617                     self.res_dir, replicaset.name))
 618
 619             if (not k8s.status.ready_replicas or
 620                     (k8s.status.ready_replicas < k8s.status.replicas)):
 621                 self._add_failing_resource(replicaset)
 622
 623             self.all_resources.append(replicaset)
 624
 625
 626 class CheckK8sStatefulSetsStep(CheckK8sResourcesUsingPodsStep):
 627
 628     __logger = logging.getLogger(__name__)
 629
 630     def __init__(self, pods, cleanup: bool = False, **kwargs):
 631         """Init CheckK8sStatefulSetsStep."""
 632         super().__init__("statefulset", pods)
 633
 634     def _init_resources(self):
 635         super()._init_resources()
 636         self.k8s_resources = self.app.list_namespaced_stateful_set(NAMESPACE).items
 637
 638     def _parse_resources(self):
 639         """Parse the statefulsets."""
 640         super()._parse_resources()
 641         for k8s in self.k8s_resources:
 642             statefulset = StatefulSet(k8s=k8s)
 643
 644             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 645                 continue
 646
 647             if k8s.spec.selector and k8s.spec.selector.match_labels:
 648                 (statefulset.pods,
 649                  statefulset.failed_pods) = self._find_child_pods(
 650                      k8s.spec.selector.match_labels)
 651             field_selector = "involvedObject.name={}".format(statefulset.name)
 652             field_selector += ",involvedObject.kind=StatefulSet"
 653             statefulset.events = self.core.list_namespaced_event(
 654                 NAMESPACE,
 655                 field_selector=field_selector).items
 656
 657             self.jinja_env.get_template('statefulset.html.j2').stream(
 658                 statefulset=statefulset).dump('{}/statefulset-{}.html'.format(
 659                     self.res_dir, statefulset.name))
 660
 661             if ((not k8s.status.ready_replicas)
 662                     or (k8s.status.ready_replicas < k8s.status.replicas)):
 663                 self._add_failing_resource(statefulset)
 664
 665             self.all_resources.append(statefulset)
 666
 667
 668 class CheckK8sDaemonSetsStep(CheckK8sResourcesUsingPodsStep):
 669
 670     __logger = logging.getLogger(__name__)
 671
 672     def __init__(self, pods, cleanup: bool = False, **kwargs):
 673         """Init CheckK8sDaemonSetsStep."""
 674         super().__init__("daemonset", pods)
 675
 676     def _init_resources(self):
 677         super()._init_resources()
 678         self.k8s_resources = self.app.list_namespaced_daemon_set(NAMESPACE).items
 679
 680     def _parse_resources(self):
 681         """Parse the daemonsets."""
 682         super()._parse_resources()
 683         for k8s in self.k8s_resources:
 684             daemonset = DaemonSet(k8s=k8s)
 685
 686             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 687                 continue
 688
 689             if k8s.spec.selector and k8s.spec.selector.match_labels:
 690                 (daemonset.pods,
 691                  daemonset.failed_pods) = self._find_child_pods(
 692                      k8s.spec.selector.match_labels)
 693             field_selector = "involvedObject.name={}".format(daemonset.name)
 694             field_selector += ",involvedObject.kind=DaemonSet"
 695             daemonset.events = self.core.list_namespaced_event(
 696                 NAMESPACE,
 697                 field_selector=field_selector).items
 698
 699             self.jinja_env.get_template('daemonset.html.j2').stream(
 700                 daemonset=daemonset).dump('{}/daemonset-{}.html'.format(
 701                     self.res_dir, daemonset.name))
 702
 703             if (k8s.status.number_ready < k8s.status.desired_number_scheduled):
 704                 self._add_failing_resource(daemonset)
 705
 706             self.all_resources.append(daemonset)
 707
 708
 709 class CheckNamespaceStatusStep(CheckK8sResourcesStep):
 710     """Check status of all k8s resources in the selected namespace."""
 711
 712     __logger = logging.getLogger(__name__)
 713
 714     def __init__(self, cleanup: bool = False, **kwargs):
 715         """Init CheckNamespaceStatusStep."""
 716         super().__init__("")
 717         self.__logger.debug("%s namespace status test init started", NAMESPACE)
 718         if settings.IN_CLUSTER:
 719             config.load_incluster_config()
 720         else:
 721             config.load_kube_config(config_file=settings.K8S_CONFIG)
 722
 723         self.job_list_step = CheckK8sJobsStep()
 724         self.pod_list_step = CheckK8sPodsStep(self.job_list_step)
 725         self.service_list_step = CheckK8sServicesStep(self.pod_list_step)
 726         self.deployment_list_step = CheckK8sDeploymentsStep(self.pod_list_step)
 727         self.replicaset_list_step = CheckK8sResplicaSetsStep(self.pod_list_step)
 728         self.statefulset_list_step = CheckK8sStatefulSetsStep(self.pod_list_step)
 729         self.daemonset_list_step = CheckK8sDaemonSetsStep(self.pod_list_step)
 730         self.configmap_list_step = CheckK8sConfigMapsStep()
 731         self.secret_list_step = CheckK8sSecretsStep()
 732         self.ingress_list_step = CheckK8sIngressesStep()
 733         self.pvc_list_step = CheckK8sPvcsStep()
 734         self.add_step(self.job_list_step)
 735         self.add_step(self.pod_list_step)
 736         self.add_step(self.service_list_step)
 737         self.add_step(self.deployment_list_step)
 738         self.add_step(self.replicaset_list_step)
 739         self.add_step(self.statefulset_list_step)
 740         self.add_step(self.daemonset_list_step)
 741         self.add_step(self.configmap_list_step)
 742         self.add_step(self.secret_list_step)
 743         self.add_step(self.ingress_list_step)
 744         self.add_step(self.pvc_list_step)
 745
 746     @property
 747     def description(self) -> str:
 748         """Step description."""
 749         return "Check status of all k8s resources in the selected namespace."
 750
 751     @property
 752     def component(self) -> str:
 753         """Component name."""
 754         return "ALL"
 755
 756     @BaseStep.store_state
 757     def execute(self):
 758         """Check status of all k8s resources in the selected namespace.
 759
 760         Use settings values:
 761          - K8S_ONAP_NAMESPACE
 762          - STATUS_RESULTS_DIRECTORY
 763          - STORE_ARTIFACTS
 764          - CHECK_POD_VERSIONS
 765         """
 766         super().execute()
 767
 768         self.pods = self.pod_list_step.all_resources
 769         self.services = self.service_list_step.all_resources
 770         self.jobs = self.job_list_step.all_resources
 771         self.deployments = self.deployment_list_step.all_resources
 772         self.replicasets = self.replicaset_list_step.all_resources
 773         self.statefulsets = self.statefulset_list_step.all_resources
 774         self.daemonsets = self.daemonset_list_step.all_resources
 775         self.pvcs = self.pvc_list_step.all_resources
 776         self.configmaps = self.configmap_list_step.all_resources
 777         self.secrets = self.secret_list_step.all_resources
 778         self.ingresses = self.ingress_list_step.all_resources
 779
 780         self.failing_statefulsets = self.statefulset_list_step.failing_resources
 781         self.failing_jobs = self.job_list_step.failing_resources
 782         self.failing_deployments = self.deployment_list_step.failing_resources
 783         self.failing_replicasets = self.replicaset_list_step.failing_resources
 784         self.failing_daemonsets = self.daemonset_list_step.failing_resources
 785         self.failing_pvcs = self.pvc_list_step.failing_resources
 786
 787         self.jinja_env.get_template('index.html.j2').stream(
 788             ns=self,
 789             delta=delta).dump('{}/index.html'.format(self.res_dir))
 790         self.jinja_env.get_template('raw_output.txt.j2').stream(
 791             ns=self, namespace=NAMESPACE).dump('{}/onap-k8s.log'.format(
 792                 self.res_dir))
 793
 794         details = {}
 795         for step in self._steps:
 796             if step.failing:
 797                 self.failing = True
 798                 self.__logger.info("%s failing: %s",
 799                                    step.resource_type,
 800                                    len(step.failing_resources))
 801             details[step.resource_type] = {
 802                 'number_all': len(step.all_resources),
 803                 'number_failing': len(step.failing_resources),
 804                 'all': self.map_by_name(step.all_resources),
 805                 'failing': self.map_by_name(step.failing_resources)
 806             }
 807         with (Path(self.res_dir).joinpath(settings.STATUS_DETAILS_JSON)).open('w') as file:
 808             json.dump(details, file, indent=4)
 809         if self.failing:
 810             raise StatusCheckException
 811
 812     def map_by_name(self, resources):
 813         return list(map(lambda resource: resource.name, resources))