src/onaptests/steps/cloud/check_status.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # SPDX-License-Identifier: Apache-2.0
   4 import json
   5 import logging
   6 import os
   7 import re
   8 from pathlib import Path
   9
  10 from jinja2 import Environment, PackageLoader, select_autoescape
  11 from kubernetes import client, config
  12 from kubernetes.stream import stream
  13 from natural.date import delta
  14 from urllib3.exceptions import MaxRetryError, NewConnectionError
  15 from xtesting.core import testcase
  16
  17 from onapsdk.configuration import settings
  18 from onaptests.utils.exceptions import StatusCheckException
  19
  20 from ..base import BaseStep
  21 from .resources import (ConfigMap, Container, DaemonSet, Deployment, Ingress,
  22                         Job, Pod, Pvc, ReplicaSet, Secret, Service,
  23                         StatefulSet)
  24
  25 NAMESPACE = settings.K8S_ONAP_NAMESPACE
  26
  27
  28 class CheckK8sResourcesStep(BaseStep):
  29
  30     __logger = logging.getLogger(__name__)
  31
  32     def __init__(self, resource_type: str, break_on_error=False):
  33         """Init CheckK8sResourcesStep."""
  34         super().__init__(cleanup=BaseStep.HAS_NO_CLEANUP, break_on_error=break_on_error)
  35         self.core = client.CoreV1Api()
  36         self.batch = client.BatchV1Api()
  37         self.app = client.AppsV1Api()
  38         self.networking = client.NetworkingV1Api()
  39
  40         if settings.STATUS_RESULTS_DIRECTORY:
  41             self.res_dir = f"{settings.STATUS_RESULTS_DIRECTORY}"
  42         else:
  43             self.res_dir = f"{testcase.TestCase.dir_results}/kubernetes-status"
  44
  45         self.failing = False
  46         self.resource_type = resource_type
  47         self.k8s_resources = []
  48         self.all_resources = []
  49         self.failing_resources = []
  50         self.jinja_env = Environment(autoescape=select_autoescape(['html']),
  51                                      loader=PackageLoader('onaptests.templates', 'status'))
  52
  53     @property
  54     def component(self) -> str:
  55         """Component name."""
  56         return "ALL"
  57
  58     @property
  59     def description(self) -> str:
  60         """Step description."""
  61         return f"Check status of all k8s {self.resource_type}s in the {NAMESPACE} namespace."
  62
  63     def _init_resources(self):
  64         if self.resource_type != "":
  65             self.__logger.debug(f"Loading all k8s {self.resource_type}s"
  66                                 " in the {NAMESPACE} namespace")
  67
  68     def _parse_resources(self):
  69         """Parse the resources."""
  70         return []
  71
  72     def _add_failing_resource(self, resource):
  73         if (resource.labels and settings.EXCLUDED_LABELS
  74                 and (resource.labels.keys() and settings.EXCLUDED_LABELS.keys())):
  75             for label in resource.labels.items():
  76                 for waived_label in settings.EXCLUDED_LABELS.items():
  77                     if label[0] in waived_label[0] and label[1] in waived_label[1]:
  78                         return
  79         self.__logger.warning("a {} is in error: {}".format(self.resource_type, resource.name))
  80         self.failing_resources.append(resource)
  81         self.failing = True
  82
  83     def execute(self):
  84         super().execute()
  85         os.makedirs(self.res_dir, exist_ok=True)
  86         try:
  87             self._init_resources()
  88             if len(self.k8s_resources) > 0:
  89                 self.__logger.info("%4s %ss in the namespace",
  90                                    len(self.k8s_resources),
  91                                    self.resource_type)
  92                 self._parse_resources()
  93                 self.__logger.info("%4s %ss parsed, %s failing",
  94                                    len(self.all_resources),
  95                                    self.resource_type,
  96                                    len(self.failing_resources))
  97                 if self.failing:
  98                     raise StatusCheckException(f"{self.resource_type} test failed")
  99         except (ConnectionRefusedError, MaxRetryError, NewConnectionError) as e:
 100             self.__logger.error("Test of k8s %ss failed.", self.resource_type)
 101             self.__logger.error("Cannot connect to Kubernetes.")
 102             raise StatusCheckException from e
 103
 104
 105 class CheckBasicK8sResourcesStep(CheckK8sResourcesStep):
 106
 107     def __init__(self, resource_type: str, k8s_res_class):
 108         """Init CheckBasicK8sResourcesStep."""
 109         super().__init__(resource_type)
 110         self.k8s_res_class = k8s_res_class
 111
 112     def _parse_resources(self):
 113         """Parse simple k8s resources."""
 114         super()._parse_resources()
 115         for k8s in self.k8s_resources:
 116             resource = self.k8s_res_class(k8s=k8s)
 117             self.all_resources.append(resource)
 118
 119     @BaseStep.store_state
 120     def execute(self):
 121         super().execute()
 122
 123
 124 class CheckK8sConfigMapsStep(CheckBasicK8sResourcesStep):
 125
 126     def __init__(self):
 127         """Init CheckK8sConfigMapsStep."""
 128         super().__init__("configmap", ConfigMap)
 129
 130     def _init_resources(self):
 131         super()._init_resources()
 132         self.k8s_resources = self.core.list_namespaced_config_map(NAMESPACE).items
 133
 134
 135 class CheckK8sSecretsStep(CheckBasicK8sResourcesStep):
 136
 137     def __init__(self):
 138         """Init CheckK8sSecretsStep."""
 139         super().__init__("secret", Secret)
 140
 141     def _init_resources(self):
 142         super()._init_resources()
 143         self.k8s_resources = self.core.list_namespaced_secret(NAMESPACE).items
 144
 145
 146 class CheckK8sIngressesStep(CheckBasicK8sResourcesStep):
 147
 148     def __init__(self):
 149         """Init CheckK8sIngressesStep."""
 150         super().__init__("ingress", Ingress)
 151
 152     def _init_resources(self):
 153         super()._init_resources()
 154         self.k8s_resources = self.networking.list_namespaced_ingress(NAMESPACE).items
 155
 156
 157 class CheckK8sPvcsStep(CheckK8sResourcesStep):
 158
 159     def __init__(self):
 160         """Init CheckK8sPvcsStep."""
 161         super().__init__("pvc")
 162
 163     def _init_resources(self):
 164         super()._init_resources()
 165         self.k8s_resources = self.core.list_namespaced_persistent_volume_claim(NAMESPACE).items
 166
 167     def _parse_resources(self):
 168         """Parse the jobs.
 169         Return a list of Pods that were created to perform jobs.
 170         """
 171         super()._parse_resources()
 172         for k8s in self.k8s_resources:
 173             pvc = Pvc(k8s=k8s)
 174             field_selector = (f"involvedObject.name={pvc.name},"
 175                               "involvedObject.kind=PersistentVolumeClaim")
 176             pvc.events = self.core.list_namespaced_event(
 177                 NAMESPACE,
 178                 field_selector=field_selector).items
 179
 180             if k8s.status.phase != "Bound":
 181                 self._add_failing_resource(pvc)
 182             self.all_resources.append(pvc)
 183
 184     @BaseStep.store_state
 185     def execute(self):
 186         super().execute()
 187
 188
 189 class CheckK8sResourcesUsingPodsStep(CheckK8sResourcesStep):
 190
 191     def __init__(self, resource_type: str, pods_source):
 192         """Init CheckK8sResourcesUsingPodsStep."""
 193         super().__init__(resource_type)
 194         self.pods_source = pods_source
 195
 196     def _get_used_pods(self):
 197         pods = []
 198         if self.pods_source is not None:
 199             pods = self.pods_source.all_resources
 200         return pods
 201
 202     def _find_child_pods(self, selector):
 203         pods_used = self._get_used_pods()
 204         pods_list = []
 205         failed_pods = 0
 206         if selector:
 207             raw_selector = ''
 208             for key, value in selector.items():
 209                 raw_selector += key + '=' + value + ','
 210             raw_selector = raw_selector[:-1]
 211             pods = self.core.list_namespaced_pod(
 212                 NAMESPACE, label_selector=raw_selector).items
 213             for pod in pods:
 214                 for known_pod in pods_used:
 215                     if known_pod.name == pod.metadata.name:
 216                         pods_list.append(known_pod)
 217                         if not known_pod.ready():
 218                             failed_pods += 1
 219         return (pods_list, failed_pods)
 220
 221     @BaseStep.store_state
 222     def execute(self):
 223         super().execute()
 224
 225
 226 class CheckK8sJobsStep(CheckK8sResourcesUsingPodsStep):
 227
 228     def __init__(self):
 229         """Init CheckK8sJobsStep."""
 230         super().__init__("job", None)
 231
 232     def _init_resources(self):
 233         super()._init_resources()
 234         self.k8s_resources = self.batch.list_namespaced_job(NAMESPACE).items
 235
 236     def _parse_resources(self):
 237         """Parse the jobs.
 238         Return a list of Pods that were created to perform jobs.
 239         """
 240         super()._parse_resources()
 241         jobs_pods = []
 242         for k8s in self.k8s_resources:
 243             job = Job(k8s=k8s)
 244             job_pods = []
 245
 246             if k8s.spec.selector and k8s.spec.selector.match_labels:
 247                 (job.pods, job.failed_pods) = self._find_child_pods(
 248                     k8s.spec.selector.match_labels)
 249                 job_pods += job.pods
 250             field_selector = "involvedObject.name={}".format(job.name)
 251             field_selector += ",involvedObject.kind=Job"
 252             job.events = self.core.list_namespaced_event(
 253                 NAMESPACE,
 254                 field_selector=field_selector).items
 255
 256             self.jinja_env.get_template('job.html.j2').stream(job=job).dump(
 257                 '{}/job-{}.html'.format(self.res_dir, job.name))
 258
 259             # timemout job
 260             if not k8s.status.completion_time:
 261                 if any(waiver_elt not in job.name for waiver_elt in settings.WAIVER_LIST):
 262                     self._add_failing_resource(job)
 263             # completed job
 264             if any(waiver_elt not in job.name for waiver_elt in settings.WAIVER_LIST):
 265                 self.all_resources.append(job)
 266             jobs_pods += job_pods
 267
 268
 269 class CheckK8sPodsStep(CheckK8sResourcesUsingPodsStep):
 270
 271     __logger = logging.getLogger(__name__)
 272
 273     def __init__(self, pods):
 274         """Init CheckK8sPodsStep."""
 275         super().__init__("pod", pods)
 276
 277     def _init_resources(self):
 278         super()._init_resources()
 279         self.k8s_resources = self.core.list_namespaced_pod(NAMESPACE).items
 280
 281     def _parse_resources(self):  # noqa
 282         """Parse the pods."""
 283         super()._parse_resources()
 284         excluded_pods = self._get_used_pods()
 285         pod_versions = []
 286         containers = {}
 287         for k8s in self.k8s_resources:
 288             pod = Pod(k8s=k8s)
 289
 290             # check version firstly
 291             if settings.CHECK_POD_VERSIONS:
 292                 pod_component = k8s.metadata.name
 293                 if 'app' in k8s.metadata.labels:
 294                     pod_component = k8s.metadata.labels['app']
 295                 else:
 296                     if 'app.kubernetes.io/name' in k8s.metadata.labels:
 297                         pod_component = k8s.metadata.labels[
 298                             'app.kubernetes.io/name']
 299                     else:
 300                         self.__logger.error("pod %s has no 'app' or 'app.kubernetes.io/name' "
 301                                             "in metadata: %s", pod_component, k8s.metadata.labels)
 302
 303                 # looks for docker version
 304                 for container in k8s.spec.containers:
 305                     pod_version = {}
 306                     pod_container_version = container.image.rsplit(":", 1)
 307                     pod_container_image = pod_container_version[0]
 308                     pod_container_tag = "latest"
 309                     if len(pod_container_version) > 1:
 310                         pod_container_tag = pod_container_version[1]
 311
 312                     pod_version.update({
 313                         'container': container.name,
 314                         'component': pod_component,
 315                         'image': pod_container_image,
 316                         'version': pod_container_tag
 317                     })
 318                     pod_versions.append(pod_version)
 319
 320                     search_rule = "^(?P<source>[^/]*)/*(?P<container>[^:]*):*(?P<version>.*)$"
 321                     search = re.search(search_rule, container.image)
 322                     name = "{}/{}".format(search.group('source'),
 323                                           search.group('container'))
 324                     version = search.group('version')
 325                     if name[-1] == '/':
 326                         name = name[0:-1]
 327                     source = "default"
 328                     if search.group('source') in settings.DOCKER_REPOSITORIES:
 329                         source = search.group('source')
 330                         name = search.group('container')
 331                     container_search_rule = "^library/(?P<real_container>[^:]*)$"
 332                     container_search = re.search(container_search_rule, name)
 333                     if container_search:
 334                         name = container_search.group('real_container')
 335                     for common_component in settings.GENERIC_NAMES.keys():
 336                         if name in settings.GENERIC_NAMES[common_component]:
 337                             version = "{}:{}".format(name, version)
 338                             name = common_component
 339                             break
 340
 341                     repository = settings.DOCKER_REPOSITORIES_NICKNAMES[source]
 342                     if name in containers:
 343                         if version in containers[name]['versions']:
 344                             if not (pod_component in containers[name]['versions']
 345                                     [version]['components']):
 346                                 containers[name]['versions'][version][
 347                                     'components'].append(pod_component)
 348                                 containers[name]['number_components'] += 1
 349                             if not (repository in containers[name]['versions']
 350                                     [version]['repositories']):
 351                                 containers[name]['versions'][version][
 352                                     'repositories'].append(repository)
 353                         else:
 354                             containers[name]['versions'][version] = {
 355                                 'repositories': [repository],
 356                                 'components': [pod_component]
 357                             }
 358                             containers[name]['number_components'] += 1
 359                     else:
 360                         containers[name] = {
 361                             'versions': {
 362                                 version: {
 363                                     'repositories': [repository],
 364                                     'components': [pod_component]
 365                                 }
 366                             },
 367                             'number_components': 1
 368                         }
 369             # pod version check end
 370             if excluded_pods and pod in excluded_pods:
 371                 continue
 372
 373             if k8s.status.init_container_statuses:
 374                 for k8s_container in k8s.status.init_container_statuses:
 375                     pod.runned_init_containers += self._parse_container(
 376                         pod, k8s_container, init=True)
 377             if k8s.status.container_statuses:
 378                 for k8s_container in k8s.status.container_statuses:
 379                     pod.running_containers += self._parse_container(
 380                         pod, k8s_container)
 381             pod.events = self.core.list_namespaced_event(
 382                 NAMESPACE,
 383                 field_selector="involvedObject.name={}".format(pod.name)).items
 384             self.jinja_env.get_template('pod.html.j2').stream(pod=pod).dump(
 385                 '{}/pod-{}.html'.format(self.res_dir, pod.name))
 386             if any(waiver_elt in pod.name for waiver_elt in settings.WAIVER_LIST):
 387                 self.__logger.warn("Waiver pattern found in pod, exclude %s", pod.name)
 388             else:
 389                 self.all_resources.append(pod)
 390
 391         if settings.CHECK_POD_VERSIONS:
 392             self.jinja_env.get_template('version.html.j2').stream(
 393                 pod_versions=pod_versions).dump('{}/versions.html'.format(
 394                     self.res_dir))
 395             self.jinja_env.get_template('container_versions.html.j2').stream(
 396                 containers=containers).dump('{}/container_versions.html'.format(
 397                     self.res_dir))
 398             # create a json file for version tracking
 399             with open(self.res_dir + "/onap_versions.json", "w") as write_file:
 400                 json.dump(pod_versions, write_file)
 401
 402     def _get_container_logs(self, pod, container, full=True, previous=False):
 403         logs = ""
 404         limit_bytes = settings.MAX_LOG_BYTES
 405         if full:
 406             limit_bytes = settings.UNLIMITED_LOG_BYTES
 407         try:
 408             logs = self.core.read_namespaced_pod_log(
 409                 pod.name,
 410                 NAMESPACE,
 411                 container=container.name,
 412                 limit_bytes=limit_bytes,
 413                 previous=previous
 414             )
 415         except UnicodeDecodeError:
 416             logs = "{0} has an unicode decode error...".format(pod.name)
 417             self.__logger.error(
 418                 "{0} has an unicode decode error in the logs...", pod.name,
 419             )
 420         return logs
 421
 422     def _parse_container(self, pod, k8s_container, init=False):  # noqa
 423         """Get the logs of a container."""
 424         logs = ""
 425         old_logs = ""
 426         prefix = ""
 427         containers_list = pod.containers
 428         container = Container(name=k8s_container.name)
 429         container.restart_count = k8s_container.restart_count
 430         container.set_status(k8s_container.state)
 431         container.ready = k8s_container.ready
 432         container.image = k8s_container.image
 433         if init:
 434             prefix = "init "
 435             containers_list = pod.init_containers
 436             if container.restart_count > pod.init_restart_count:
 437                 pod.init_restart_count = container.restart_count
 438             if not container.ready:
 439                 pod.init_done = False
 440         else:
 441             if container.restart_count > pod.restart_count:
 442                 pod.restart_count = container.restart_count
 443         if settings.STORE_ARTIFACTS:
 444             try:
 445                 log_files = {}
 446                 logs = self._get_container_logs(pod=pod, container=container, full=False)
 447                 with open(
 448                         "{}/pod-{}-{}.log".format(self.res_dir,
 449                                                   pod.name, container.name),
 450                         'w') as log_result:
 451                     log_result.write(logs)
 452                 if (not container.ready) and container.restart_count > 0:
 453                     old_logs = self._get_container_logs(pod=pod, container=container,
 454                                                         previous=True)
 455                     with open(
 456                             "{}/pod-{}-{}.old.log".format(self.res_dir,
 457                                                           pod.name,
 458                                                           container.name),
 459                             'w') as log_result:
 460                         log_result.write(old_logs)
 461                 if (container.name in settings.FULL_LOGS_CONTAINERS):
 462                     logs = self._get_container_logs(pod=pod, container=container)
 463                     with open(
 464                             "{}/pod-{}-{}.log".format(self.res_dir,
 465                                                       pod.name, container.name),
 466                             'w') as log_result:
 467                         log_result.write(logs)
 468                 if (container.name in settings.SPECIFIC_LOGS_CONTAINERS):
 469                     for log_file in settings.SPECIFIC_LOGS_CONTAINERS[container.name]:
 470                         exec_command = ['/bin/sh', '-c', "cat {}".format(log_file)]
 471                         log_files[log_file] = stream(
 472                             self.core.connect_get_namespaced_pod_exec,
 473                             pod.name,
 474                             NAMESPACE,
 475                             container=container.name,
 476                             command=exec_command,
 477                             stderr=True,
 478                             stdin=False,
 479                             stdout=True,
 480                             tty=False)
 481                         log_file_slug = log_file.split('.')[0].split('/')[-1]
 482                         with open(
 483                                 "{}/pod-{}-{}-{}.log".format(
 484                                     self.res_dir, pod.name,
 485                                     container.name, log_file_slug),
 486                                 'w') as log_result:
 487                             log_result.write(log_files[log_file])
 488             except client.rest.ApiException as exc:
 489                 self.__logger.warning("%scontainer %s of pod %s has an exception: %s",
 490                                       prefix, container.name, pod.name, exc.reason)
 491             self.jinja_env.get_template('container_log.html.j2').stream(
 492                 container=container,
 493                 pod_name=pod.name,
 494                 logs=logs,
 495                 old_logs=old_logs,
 496                 log_files=log_files).dump('{}/pod-{}-{}-logs.html'.format(
 497                     self.res_dir, pod.name, container.name))
 498         if any(waiver_elt in container.name for waiver_elt in settings.WAIVER_LIST):
 499             self.__logger.warn(
 500                 "Waiver pattern found in container, exclude %s", container.name)
 501         else:
 502             containers_list.append(container)
 503             if k8s_container.ready:
 504                 return 1
 505         return 0
 506
 507
 508 class CheckK8sServicesStep(CheckK8sResourcesUsingPodsStep):
 509
 510     def __init__(self, pods):
 511         """Init CheckK8sServicesStep."""
 512         super().__init__("service", pods)
 513
 514     def _init_resources(self):
 515         super()._init_resources()
 516         self.k8s_resources = self.core.list_namespaced_service(NAMESPACE).items
 517
 518     def _parse_resources(self):
 519         """Parse the services."""
 520         super()._parse_resources()
 521         for k8s in self.k8s_resources:
 522             service = Service(k8s=k8s)
 523
 524             (service.pods,
 525              service.failed_pods) = self._find_child_pods(k8s.spec.selector)
 526
 527             self.jinja_env.get_template('service.html.j2').stream(
 528                 service=service).dump('{}/service-{}.html'.format(
 529                     self.res_dir, service.name))
 530             self.all_resources.append(service)
 531
 532
 533 class CheckK8sDeploymentsStep(CheckK8sResourcesUsingPodsStep):
 534
 535     def __init__(self, pods):
 536         """Init CheckK8sDeploymentsStep."""
 537         super().__init__("deployment", pods)
 538
 539     def _init_resources(self):
 540         super()._init_resources()
 541         self.k8s_resources = self.app.list_namespaced_deployment(NAMESPACE).items
 542
 543     def _parse_resources(self):
 544         """Parse the deployments."""
 545         super()._parse_resources()
 546         for k8s in self.k8s_resources:
 547             deployment = Deployment(k8s=k8s)
 548
 549             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 550                 continue
 551             if k8s.spec.selector and k8s.spec.selector.match_labels:
 552                 (deployment.pods,
 553                  deployment.failed_pods) = self._find_child_pods(
 554                      k8s.spec.selector.match_labels)
 555             field_selector = "involvedObject.name={}".format(deployment.name)
 556             field_selector += ",involvedObject.kind=Deployment"
 557             deployment.events = self.core.list_namespaced_event(
 558                 NAMESPACE,
 559                 field_selector=field_selector).items
 560
 561             self.jinja_env.get_template('deployment.html.j2').stream(
 562                 deployment=deployment).dump('{}/deployment-{}.html'.format(
 563                     self.res_dir, deployment.name))
 564
 565             if k8s.status.unavailable_replicas:
 566                 self._add_failing_resource(deployment)
 567
 568             self.all_resources.append(deployment)
 569
 570
 571 class CheckK8sReplicaSetsStep(CheckK8sResourcesUsingPodsStep):
 572
 573     def __init__(self, pods):
 574         """Init CheckK8sReplicaSetsStep."""
 575         super().__init__("replicaset", pods)
 576
 577     def _init_resources(self):
 578         super()._init_resources()
 579         self.k8s_resources = self.app.list_namespaced_replica_set(NAMESPACE).items
 580
 581     def _parse_resources(self):
 582         """Parse the replicasets."""
 583         super()._parse_resources()
 584         for k8s in self.k8s_resources:
 585             replicaset = ReplicaSet(k8s=k8s)
 586
 587             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 588                 continue
 589
 590             if k8s.spec.selector and k8s.spec.selector.match_labels:
 591                 (replicaset.pods,
 592                  replicaset.failed_pods) = self._find_child_pods(
 593                      k8s.spec.selector.match_labels)
 594             field_selector = "involvedObject.name={}".format(replicaset.name)
 595             field_selector += ",involvedObject.kind=ReplicaSet"
 596             replicaset.events = self.core.list_namespaced_event(
 597                 NAMESPACE,
 598                 field_selector=field_selector).items
 599
 600             self.jinja_env.get_template('replicaset.html.j2').stream(
 601                 replicaset=replicaset).dump('{}/replicaset-{}.html'.format(
 602                     self.res_dir, replicaset.name))
 603
 604             if (not k8s.status.ready_replicas or
 605                     (k8s.status.ready_replicas < k8s.status.replicas)):
 606                 self._add_failing_resource(replicaset)
 607
 608             self.all_resources.append(replicaset)
 609
 610
 611 class CheckK8sStatefulSetsStep(CheckK8sResourcesUsingPodsStep):
 612
 613     def __init__(self, pods):
 614         """Init CheckK8sStatefulSetsStep."""
 615         super().__init__("statefulset", pods)
 616
 617     def _init_resources(self):
 618         super()._init_resources()
 619         self.k8s_resources = self.app.list_namespaced_stateful_set(NAMESPACE).items
 620
 621     def _parse_resources(self):
 622         """Parse the statefulsets."""
 623         super()._parse_resources()
 624         for k8s in self.k8s_resources:
 625             statefulset = StatefulSet(k8s=k8s)
 626
 627             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 628                 continue
 629
 630             if k8s.spec.selector and k8s.spec.selector.match_labels:
 631                 (statefulset.pods,
 632                  statefulset.failed_pods) = self._find_child_pods(
 633                      k8s.spec.selector.match_labels)
 634             field_selector = "involvedObject.name={}".format(statefulset.name)
 635             field_selector += ",involvedObject.kind=StatefulSet"
 636             statefulset.events = self.core.list_namespaced_event(
 637                 NAMESPACE,
 638                 field_selector=field_selector).items
 639
 640             self.jinja_env.get_template('statefulset.html.j2').stream(
 641                 statefulset=statefulset).dump('{}/statefulset-{}.html'.format(
 642                     self.res_dir, statefulset.name))
 643
 644             if ((not k8s.status.ready_replicas)
 645                     or (k8s.status.ready_replicas < k8s.status.replicas)):
 646                 self._add_failing_resource(statefulset)
 647
 648             self.all_resources.append(statefulset)
 649
 650
 651 class CheckK8sDaemonSetsStep(CheckK8sResourcesUsingPodsStep):
 652
 653     def __init__(self, pods):
 654         """Init CheckK8sDaemonSetsStep."""
 655         super().__init__("daemonset", pods)
 656
 657     def _init_resources(self):
 658         super()._init_resources()
 659         self.k8s_resources = self.app.list_namespaced_daemon_set(NAMESPACE).items
 660
 661     def _parse_resources(self):
 662         """Parse the daemonsets."""
 663         super()._parse_resources()
 664         for k8s in self.k8s_resources:
 665             daemonset = DaemonSet(k8s=k8s)
 666
 667             if settings.IGNORE_EMPTY_REPLICAS and k8s.spec.replicas == 0:
 668                 continue
 669
 670             if k8s.spec.selector and k8s.spec.selector.match_labels:
 671                 (daemonset.pods,
 672                  daemonset.failed_pods) = self._find_child_pods(
 673                      k8s.spec.selector.match_labels)
 674             field_selector = "involvedObject.name={}".format(daemonset.name)
 675             field_selector += ",involvedObject.kind=DaemonSet"
 676             daemonset.events = self.core.list_namespaced_event(
 677                 NAMESPACE,
 678                 field_selector=field_selector).items
 679
 680             self.jinja_env.get_template('daemonset.html.j2').stream(
 681                 daemonset=daemonset).dump('{}/daemonset-{}.html'.format(
 682                     self.res_dir, daemonset.name))
 683
 684             if (k8s.status.number_ready < k8s.status.desired_number_scheduled):
 685                 self._add_failing_resource(daemonset)
 686
 687             self.all_resources.append(daemonset)
 688
 689
 690 class CheckNamespaceStatusStep(CheckK8sResourcesStep):
 691     """Check status of all k8s resources in the selected namespace."""
 692
 693     __logger = logging.getLogger(__name__)
 694
 695     def __init__(self):
 696         """Init CheckNamespaceStatusStep."""
 697         super().__init__(resource_type="")
 698         self.__logger.debug("%s namespace status test init started", NAMESPACE)
 699         if settings.IN_CLUSTER:
 700             config.load_incluster_config()
 701         else:
 702             config.load_kube_config(config_file=settings.K8S_CONFIG)
 703
 704         self.job_list_step = CheckK8sJobsStep()
 705         self.pod_list_step = CheckK8sPodsStep(self.job_list_step)
 706         self.service_list_step = CheckK8sServicesStep(self.pod_list_step)
 707         self.deployment_list_step = CheckK8sDeploymentsStep(self.pod_list_step)
 708         self.replicaset_list_step = CheckK8sReplicaSetsStep(self.pod_list_step)
 709         self.statefulset_list_step = CheckK8sStatefulSetsStep(self.pod_list_step)
 710         self.daemonset_list_step = CheckK8sDaemonSetsStep(self.pod_list_step)
 711         self.configmap_list_step = CheckK8sConfigMapsStep()
 712         self.secret_list_step = CheckK8sSecretsStep()
 713         self.ingress_list_step = CheckK8sIngressesStep()
 714         self.pvc_list_step = CheckK8sPvcsStep()
 715         self.add_step(self.job_list_step)
 716         self.add_step(self.pod_list_step)
 717         self.add_step(self.service_list_step)
 718         self.add_step(self.deployment_list_step)
 719         self.add_step(self.replicaset_list_step)
 720         self.add_step(self.statefulset_list_step)
 721         self.add_step(self.daemonset_list_step)
 722         self.add_step(self.configmap_list_step)
 723         self.add_step(self.secret_list_step)
 724         self.add_step(self.ingress_list_step)
 725         self.add_step(self.pvc_list_step)
 726
 727     @property
 728     def description(self) -> str:
 729         """Step description."""
 730         return "Check status of all k8s resources in the selected namespace."
 731
 732     @property
 733     def component(self) -> str:
 734         """Component name."""
 735         return "ALL"
 736
 737     @BaseStep.store_state
 738     def execute(self):
 739         """Check status of all k8s resources in the selected namespace.
 740
 741         Use settings values:
 742          - K8S_ONAP_NAMESPACE
 743          - STATUS_RESULTS_DIRECTORY
 744          - STORE_ARTIFACTS
 745          - CHECK_POD_VERSIONS
 746         """
 747         super().execute()
 748
 749         self.pods = self.pod_list_step.all_resources
 750         self.services = self.service_list_step.all_resources
 751         self.jobs = self.job_list_step.all_resources
 752         self.deployments = self.deployment_list_step.all_resources
 753         self.replicasets = self.replicaset_list_step.all_resources
 754         self.statefulsets = self.statefulset_list_step.all_resources
 755         self.daemonsets = self.daemonset_list_step.all_resources
 756         self.pvcs = self.pvc_list_step.all_resources
 757         self.configmaps = self.configmap_list_step.all_resources
 758         self.secrets = self.secret_list_step.all_resources
 759         self.ingresses = self.ingress_list_step.all_resources
 760
 761         self.failing_statefulsets = self.statefulset_list_step.failing_resources
 762         self.failing_jobs = self.job_list_step.failing_resources
 763         self.failing_deployments = self.deployment_list_step.failing_resources
 764         self.failing_replicasets = self.replicaset_list_step.failing_resources
 765         self.failing_daemonsets = self.daemonset_list_step.failing_resources
 766         self.failing_pvcs = self.pvc_list_step.failing_resources
 767
 768         self.jinja_env.get_template('index.html.j2').stream(
 769             ns=self,
 770             delta=delta).dump('{}/index.html'.format(self.res_dir))
 771         self.jinja_env.get_template('raw_output.txt.j2').stream(
 772             ns=self, namespace=NAMESPACE).dump('{}/onap-k8s.log'.format(
 773                 self.res_dir))
 774
 775         details = {}
 776         for step in self._steps:
 777             if step.failing:
 778                 self.failing = True
 779                 self.__logger.info("%s failing: %s",
 780                                    step.resource_type,
 781                                    len(step.failing_resources))
 782             details[step.resource_type] = {
 783                 'number_failing': len(step.failing_resources),
 784                 'failing': self.map_by_name(step.failing_resources)
 785             }
 786             if settings.INCLUDE_ALL_RES_IN_DETAILS:
 787                 details[step.resource_type]['all'] = self.map_by_name(step.all_resources)
 788                 details[step.resource_type]['number_all'] = len(step.all_resources)
 789
 790         with (Path(self.res_dir).joinpath(settings.STATUS_DETAILS_JSON)).open('w') as file:
 791             json.dump(details, file, indent=4)
 792         if self.failing:
 793             raise StatusCheckException
 794
 795     def map_by_name(self, resources):
 796         return list(map(lambda resource: resource.name, resources))