src/onaptests/steps/cloud/check_status.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # SPDX-License-Identifier: Apache-2.0
   4 import json
   5 import os
   6 import logging
   7 import re
   8 from onapsdk.configuration import settings
   9 from natural.date import delta
  10 from xtesting.core import testcase
  11 from kubernetes import client, config
  12 from kubernetes.stream import stream
  13 from urllib3.exceptions import MaxRetryError, NewConnectionError
  14 from jinja2 import Environment, PackageLoader, select_autoescape
  15
  16 from ..base import BaseStep
  17 from .resources import Pod, Container, Service, Job
  18 from .resources import Deployment, StatefulSet, DaemonSet, Pvc, ReplicaSet
  19 from .resources import ConfigMap, Secret, Ingress
  20 from onaptests.utils.exceptions import StatusCheckException
  21
  22 NAMESPACE = settings.K8S_ONAP_NAMESPACE
  23
  24 FULL_LOGS_CONTAINERS = [
  25     'dcae-bootstrap', 'dcae-cloudify-manager', 'aai-resources',
  26     'aai-traversal', 'aai-modelloader', 'sdnc', 'so', 'so-bpmn-infra',
  27     'so-openstack-adapter', 'so-sdc-controller', 'mariadb-galera', 'sdc-be',
  28     'sdc-fe'
  29 ]
  30
  31 # patterns to be excluded from the check
  32 WAIVER_LIST = ['integration']
  33
  34 SPECIFIC_LOGS_CONTAINERS = {
  35     'sdc-be': ['/var/log/onap/sdc/sdc-be/error.log'],
  36     'sdc-onboarding-be': ['/var/log/onap/sdc/sdc-onboarding-be/error.log'],
  37     'aaf-cm': [
  38         '/opt/app/osaaf/logs/cm/cm-service.log',
  39         '/opt/app/osaaf/logs/cm/cm-init.log'
  40     ],
  41     'aaf-fs': [
  42         '/opt/app/osaaf/logs/fs/fs-service.log',
  43         '/opt/app/osaaf/logs/fs/fs-init.log'
  44     ],
  45     'aaf-locate': [
  46         '/opt/app/osaaf/logs/locate/locate-service.log',
  47         '/opt/app/osaaf/logs/locate/locate-init.log'
  48     ],
  49     'aaf-service': [
  50         '/opt/app/osaaf/logs/service/authz-service.log',
  51         '/opt/app/osaaf/logs/service/authz-init.log'
  52     ],
  53     'sdc-be': [
  54         '/var/log/onap/sdc/sdc-be/debug.log',
  55         '/var/log/onap/sdc/sdc-be/error.log'
  56     ],
  57     'sdc-fe': [
  58         '/var/log/onap/sdc/sdc-fe/debug.log',
  59         '/var/log/onap/sdc/sdc-fe/error.log'
  60     ],
  61     'vid': [
  62         '/var/log/onap/vid/audit.log',
  63         '/var/log/onap/vid/application.log',
  64         '/var/log/onap/vid/debug.log',
  65         '/var/log/onap/vid/error.log'
  66     ],
  67 }
  68
  69 DOCKER_REPOSITORIES = [
  70     'nexus3.onap.org:10001', 'docker.elastic.co', 'docker.io', 'library',
  71     'registry.gitlab.com', 'registry.hub.docker.com', 'k8s.gcr.io', 'gcr.io'
  72 ]
  73 DOCKER_REPOSITORIES_NICKNAMES = {
  74     'nexus3.onap.org:10001': 'onap',
  75     'docker.elastic.co': 'elastic',
  76     'docker.io': 'dockerHub (docker.io)',
  77     'registry.hub.docker.com': 'dockerHub (registry)',
  78     'registry.gitlab.com': 'gitlab',
  79     'library': 'dockerHub (library)',
  80     'default': 'dockerHub',
  81     'k8s.gcr.io': 'google (k8s.gcr)',
  82     'gcr.io': 'google (gcr)'
  83 }
  84
  85 GENERIC_NAMES = {
  86     'postgreSQL': ['crunchydata/crunchy-postgres', 'postgres'],
  87     'mariadb': ['adfinissygroup/k8s-mariadb-galera-centos', 'mariadb'],
  88     'elasticsearch': [
  89         'bitnami/elasticsearch', 'elasticsearch/elasticsearch',
  90         'onap/clamp-dashboard-elasticsearch'
  91     ],
  92     'nginx': ['bitnami/nginx', 'nginx'],
  93     'cassandra': [
  94         'cassandra', 'onap/music/cassandra_3_11', 'onap/music/cassandra_music',
  95         'onap/aaf/aaf_cass'
  96     ],
  97     'zookeeper': ['google_samples/k8szk', 'onap/dmaap/zookeeper', 'zookeeper'],
  98     'redis': [
  99         'onap/vfc/db',
 100         'onap/org.onap.dcaegen2.deployments.redis-cluster-container'
 101     ],
 102     'consul': ['consul', 'oomk8s/consul'],
 103     'rabbitmq': ['ansible/awx_rabbitmq', 'rabbitmq']
 104 }
 105
 106 MAX_LOG_BYTES = 512000
 107
 108
 109 class CheckNamespaceStatusStep(BaseStep):
 110     """Check status of all k8s resources in the selected namespace."""
 111
 112     __logger = logging.getLogger(__name__)
 113
 114     def __init__(self, cleanup: bool = False,**kwargs):
 115         """Init CheckNamespaceStatusStep."""
 116         super().__init__(cleanup=cleanup)
 117
 118         if settings.STATUS_RESULTS_DIRECTORY:
 119             self.res_dir = f"{settings.STATUS_RESULTS_DIRECTORY}"
 120         else:
 121             self.res_dir = f"{testcase.TestCase.dir_results}/kubernetes-status"
 122
 123         if settings.IN_CLUSTER:
 124             config.load_incluster_config()
 125         else:
 126             config.load_kube_config(config_file=settings.K8S_CONFIG)
 127
 128         self.core = client.CoreV1Api()
 129         self.batch = client.BatchV1Api()
 130         self.app = client.AppsV1Api()
 131         self.networking = client.NetworkingV1Api()
 132
 133         self.__logger.debug("namespace status init started")
 134         self.pods = []
 135         self.services = []
 136         self.jobs = []
 137         self.deployments = []
 138         self.replicasets =[]
 139         self.statefulsets = []
 140         self.daemonsets = []
 141         self.pvcs = []
 142         self.configmaps = []
 143         self.secrets = []
 144         self.ingresses = []
 145         self.details = {}
 146
 147     @property
 148     def description(self) -> str:
 149         """Step description."""
 150         return "Check status of all k8s resources in the selected namespace."
 151
 152     @property
 153     def component(self) -> str:
 154         """Component name."""
 155         return "ALL"
 156
 157     @BaseStep.store_state
 158     def execute(self):
 159         """Check status of all k8s resources in the selected namespace.
 160
 161         Use settings values:
 162          - K8S_ONAP_NAMESPACE.
 163
 164         """
 165         super().execute()
 166         if settings.STORE_ARTIFACTS:
 167             os.makedirs(self.res_dir, exist_ok=True)
 168         self.__logger.debug("start test")
 169         try:
 170             self.k8s_pods = self.core.list_namespaced_pod(NAMESPACE).items
 171             self.__logger.info("%4s Pods in the namespace", len(self.k8s_pods))
 172
 173             self.k8s_jobs = self.batch.list_namespaced_job(NAMESPACE).items
 174             self.__logger.info("%4s Jobs in the namespace", len(self.k8s_jobs))
 175
 176             self.k8s_deployments = self.app.list_namespaced_deployment(
 177                 NAMESPACE).items
 178             self.__logger.info("%4s Deployments in the namespace",
 179                                len(self.k8s_deployments))
 180
 181             self.k8s_replicasets = self.app.list_namespaced_replica_set(
 182                 NAMESPACE).items
 183             self.__logger.info("%4s Replicasets in the namespace",
 184                                len(self.k8s_replicasets))
 185
 186             self.k8s_statefulsets = self.app.list_namespaced_stateful_set(
 187                 NAMESPACE).items
 188             self.__logger.info("%4s StatefulSets in the namespace",
 189                                len(self.k8s_statefulsets))
 190
 191             self.k8s_daemonsets = self.app.list_namespaced_daemon_set(
 192                 NAMESPACE).items
 193             self.__logger.info("%4s DaemonSets in the namespace",
 194                                len(self.k8s_daemonsets))
 195
 196             self.k8s_services = self.core.list_namespaced_service(
 197                 NAMESPACE).items
 198             self.__logger.info("%4s Services in the namespace",
 199                                len(self.k8s_services))
 200
 201             self.k8s_pvcs = self.core.list_namespaced_persistent_volume_claim(
 202                 NAMESPACE).items
 203             self.__logger.info("%4s PVCs in the namespace", len(self.pvcs))
 204
 205             self.k8s_configmaps = self.core.list_namespaced_config_map(
 206                 NAMESPACE).items
 207             self.__logger.info("%4s ConfigMaps in the namespace",
 208                                len(self.configmaps))
 209
 210             self.k8s_secrets = self.core.list_namespaced_secret(
 211                 NAMESPACE).items
 212             self.__logger.info("%4s Secrets in the namespace",
 213                                len(self.secrets))
 214
 215             self.k8s_ingresses = self.networking.list_namespaced_ingress(
 216                 NAMESPACE).items
 217             self.__logger.info("%4s Ingresses in the namespace",
 218                                len(self.ingresses))
 219         except (ConnectionRefusedError, MaxRetryError, NewConnectionError):
 220             self.__logger.error("namespace status test failed.")
 221             self.__logger.error("cannot connect to Kubernetes.")
 222             return testcase.TestCase.EX_TESTCASE_FAILED
 223
 224         self.failing_statefulsets = []
 225         self.failing_jobs = []
 226         self.failing_deployments = []
 227         self.failing_replicasets = []
 228         self.failing_daemonsets = []
 229         self.failing_pvcs = []
 230         self.failing = False
 231
 232         self.jinja_env = Environment(autoescape=select_autoescape(['html']),
 233                                      loader=PackageLoader('onaptests.templates','status'))
 234         self.parse_services()
 235         jobs_pods = self.parse_jobs()
 236         self.parse_pods(excluded_pods=jobs_pods)
 237         self.parse_deployments()
 238         self.parse_replicasets()
 239         self.parse_statefulsets()
 240         self.parse_daemonsets()
 241         self.parse_pvcs()
 242         self.parse_configmaps()
 243         self.parse_secrets()
 244         self.parse_ingresses()
 245         self.parse_versions()
 246         if settings.STORE_ARTIFACTS:
 247             self.jinja_env.get_template('index.html.j2').stream(
 248                 ns=self,
 249                 delta=delta).dump('{}/index.html'.format(self.res_dir))
 250             self.jinja_env.get_template('raw_output.txt.j2').stream(
 251                 ns=self, namespace=NAMESPACE).dump('{}/onap-k8s.log'.format(
 252                     self.res_dir))
 253
 254         if len(self.jobs) > 0:
 255             self.details['jobs'] = {
 256                 'number': len(self.jobs),
 257                 'number_failing': len(self.failing_jobs),
 258                 'failing': self.map_by_name(self.failing_jobs)
 259             }
 260         if len(self.deployments) > 0:
 261             self.details['deployments'] = {
 262                 'number': len(self.deployments),
 263                 'number_failing': len(self.failing_deployments),
 264                 'failing': self.map_by_name(self.failing_deployments)
 265             }
 266         if len(self.replicasets) > 0:
 267             self.details['replicasets'] = {
 268                 'number': len(self.replicasets),
 269                 'number_failing': len(self.failing_replicasets),
 270                 'failing': self.map_by_name(self.failing_replicasets)
 271             }
 272         if len(self.statefulsets) > 0:
 273             self.details['statefulsets'] = {
 274                 'number': len(self.statefulsets),
 275                 'number_failing': len(self.failing_statefulsets),
 276                 'failing': self.map_by_name(self.failing_statefulsets)
 277             }
 278         if len(self.daemonsets) > 0:
 279             self.details['daemonsets'] = {
 280                 'number': len(self.daemonsets),
 281                 'number_failing': len(self.failing_daemonsets),
 282                 'failing': self.map_by_name(self.failing_daemonsets)
 283             }
 284         if len(self.pvcs) > 0:
 285             self.details['pvcs'] = {
 286                 'number': len(self.pvcs),
 287                 'number_failing': len(self.failing_pvcs),
 288                 'failing': self.map_by_name(self.failing_pvcs)
 289             }
 290         if self.failing:
 291             self.__logger.error("namespace status test failed.")
 292             self.__logger.error("number of errored Jobs: %s",
 293                                 len(self.failing_jobs))
 294             self.__logger.error("number of errored Deployments: %s",
 295                                 len(self.failing_deployments))
 296             self.__logger.error("number of errored Replicasets: %s",
 297                                 len(self.failing_replicasets))
 298             self.__logger.error("number of errored StatefulSets: %s",
 299                                 len(self.failing_statefulsets))
 300             self.__logger.error("number of errored DaemonSets: %s",
 301                                 len(self.failing_daemonsets))
 302             self.__logger.error("number of errored PVCs: %s",
 303                                 len(self.failing_pvcs))
 304             raise StatusCheckException
 305
 306     def parse_pods(self, excluded_pods=None):
 307         """Parse the pods status."""
 308         self.__logger.info("%4s pods to parse", len(self.k8s_pods))
 309         for k8s in self.k8s_pods:
 310             pod = Pod(k8s=k8s)
 311
 312             if excluded_pods and pod in excluded_pods:
 313                 continue
 314
 315             if k8s.status.init_container_statuses:
 316                 for k8s_container in k8s.status.init_container_statuses:
 317                     pod.runned_init_containers += self.parse_container(
 318                         pod, k8s_container, init=True)
 319             if k8s.status.container_statuses:
 320                 for k8s_container in k8s.status.container_statuses:
 321                     pod.running_containers += self.parse_container(
 322                         pod, k8s_container)
 323             pod.events = self.core.list_namespaced_event(
 324                 NAMESPACE,
 325                 field_selector="involvedObject.name={}".format(pod.name)).items
 326             if settings.STORE_ARTIFACTS:
 327                 self.jinja_env.get_template('pod.html.j2').stream(pod=pod).dump(
 328                     '{}/pod-{}.html'.format(self.res_dir, pod.name))
 329                 if any(waiver_elt in pod.name for waiver_elt in WAIVER_LIST):
 330                     self.__logger.warn("Waiver pattern found in pod, exclude %s", pod.name)
 331                 else:
 332                     self.pods.append(pod)
 333
 334     def parse_container(self, pod, k8s_container, init=False):
 335         """Get the logs of a container."""
 336         logs = ""
 337         old_logs = ""
 338         prefix = ""
 339         containers_list = pod.containers
 340         container = Container(name=k8s_container.name)
 341         container.restart_count = k8s_container.restart_count
 342         container.set_status(k8s_container.state)
 343         container.ready = k8s_container.ready
 344         container.image = k8s_container.image
 345         if init:
 346             prefix = "init "
 347             containers_list = pod.init_containers
 348             if container.restart_count > pod.init_restart_count:
 349                 pod.init_restart_count = container.restart_count
 350             if not container.ready:
 351                 pod.init_done = False
 352         else:
 353             if container.restart_count > pod.restart_count:
 354                 pod.restart_count = container.restart_count
 355
 356         try:
 357             log_files = {}
 358             logs = ""
 359             try:
 360                 logs = self.core.read_namespaced_pod_log(
 361                     pod.name,
 362                     NAMESPACE,
 363                     container=container.name,
 364                     limit_bytes=MAX_LOG_BYTES,
 365                 )
 366             except UnicodeDecodeError:
 367                 logs= "{0} has an unicode decode error...".format(pod.name)
 368                 self.__logger.error(
 369                     "{0} has an unicode decode error in the logs...", pod.name,
 370                 )
 371             if settings.STORE_ARTIFACTS:
 372                 with open(
 373                         "{}/pod-{}-{}.log".format(self.res_dir,
 374                                                 pod.name, container.name),
 375                         'w') as log_result:
 376                     log_result.write(logs)
 377             if (not container.ready) and container.restart_count > 0:
 378                 old_logs = self.core.read_namespaced_pod_log(
 379                     pod.name,
 380                     NAMESPACE,
 381                     container=container.name,
 382                     previous=True)
 383                 if settings.STORE_ARTIFACTS:
 384                     with open(
 385                             "{}/pod-{}-{}.old.log".format(self.res_dir,
 386                                                         pod.name,
 387                                                         container.name),
 388                             'w') as log_result:
 389                         log_result.write(old_logs)
 390             if (container.name in FULL_LOGS_CONTAINERS):
 391                 logs = self.core.read_namespaced_pod_log(
 392                     pod.name, NAMESPACE, container=container.name)
 393                 if settings.STORE_ARTIFACTS:
 394                     with open(
 395                             "{}/pod-{}-{}.log".format(self.res_dir,
 396                                                     pod.name, container.name),
 397                             'w') as log_result:
 398                         log_result.write(logs)
 399             if (container.name in SPECIFIC_LOGS_CONTAINERS):
 400                 for log_file in SPECIFIC_LOGS_CONTAINERS[container.name]:
 401                     exec_command = ['/bin/sh', '-c', "cat {}".format(log_file)]
 402                     log_files[log_file] = stream(
 403                         self.core.connect_get_namespaced_pod_exec,
 404                         pod.name,
 405                         NAMESPACE,
 406                         container=container.name,
 407                         command=exec_command,
 408                         stderr=True,
 409                         stdin=False,
 410                         stdout=True,
 411                         tty=False)
 412                     log_file_slug = log_file.split('.')[0].split('/')[-1]
 413                     with open(
 414                             "{}/pod-{}-{}-{}.log".format(
 415                                 self.res_dir, pod.name,
 416                                 container.name, log_file_slug),
 417                             'w') as log_result:
 418                         log_result.write(log_files[log_file])
 419         except client.rest.ApiException as exc:
 420             self.__logger.warning("%scontainer %s of pod %s has an exception: %s",
 421                                prefix, container.name, pod.name, exc.reason)
 422         if settings.STORE_ARTIFACTS:
 423             self.jinja_env.get_template('container_log.html.j2').stream(
 424                 container=container,
 425                 pod_name=pod.name,
 426                 logs=logs,
 427                 old_logs=old_logs,
 428                 log_files=log_files).dump('{}/pod-{}-{}-logs.html'.format(
 429                     self.res_dir, pod.name, container.name))
 430         if any(waiver_elt in container.name for waiver_elt in WAIVER_LIST):
 431             self.__logger.warn(
 432                 "Waiver pattern found in container, exclude %s", container.name)
 433         else:
 434             containers_list.append(container)
 435             if k8s_container.ready:
 436                 return 1
 437         return 0
 438
 439     def parse_services(self):
 440         """Parse the services."""
 441         self.__logger.info("%4s services to parse", len(self.k8s_services))
 442         for k8s in self.k8s_services:
 443             service = Service(k8s=k8s)
 444
 445             (service.pods,
 446              service.failed_pods) = self._find_child_pods(k8s.spec.selector)
 447
 448             if settings.STORE_ARTIFACTS:
 449                 self.jinja_env.get_template('service.html.j2').stream(
 450                     service=service).dump('{}/service-{}.html'.format(
 451                         self.res_dir, service.name))
 452                 self.services.append(service)
 453
 454     def parse_jobs(self):
 455         """Parse the jobs.
 456         Return a list of Pods that were created to perform jobs.
 457         """
 458         self.__logger.info("%4s jobs to parse", len(self.k8s_jobs))
 459         jobs_pods = []
 460         for i in range(len(self.k8s_jobs)):
 461             k8s = self.k8s_jobs[i]
 462             job = Job(k8s=k8s)
 463             job_pods = []
 464
 465             if k8s.spec.selector and k8s.spec.selector.match_labels:
 466                 (job.pods, job.failed_pods) = self._find_child_pods(
 467                     k8s.spec.selector.match_labels)
 468                 job_pods += job.pods
 469             field_selector = "involvedObject.name={}".format(job.name)
 470             field_selector += ",involvedObject.kind=Job"
 471             job.events = self.core.list_namespaced_event(
 472                 NAMESPACE,
 473                 field_selector=field_selector).items
 474
 475             if settings.STORE_ARTIFACTS:
 476                 self.jinja_env.get_template('job.html.j2').stream(job=job).dump(
 477                     '{}/job-{}.html'.format(self.res_dir, job.name))
 478
 479             # timemout job
 480             if not k8s.status.completion_time:
 481                 self.__logger.warning("a Job is in error: {}".format(job.name))
 482                 if any(
 483                     waiver_elt not in job.name for waiver_elt in WAIVER_LIST):
 484                     self.failing_jobs.append(job)
 485                     self.failing = True
 486             # completed job
 487             if any(waiver_elt not in job.name for waiver_elt in WAIVER_LIST):
 488                 self.jobs.append(job)
 489             jobs_pods += job_pods
 490         return jobs_pods
 491
 492     def parse_deployments(self):
 493         """Parse the deployments."""
 494         self.__logger.info("%4s deployments to parse",
 495                            len(self.k8s_deployments))
 496         for i in range(len(self.k8s_deployments)):
 497             k8s = self.k8s_deployments[i]
 498             deployment = Deployment(k8s=k8s)
 499
 500             if k8s.spec.selector and k8s.spec.selector.match_labels:
 501                 (deployment.pods,
 502                  deployment.failed_pods) = self._find_child_pods(
 503                      k8s.spec.selector.match_labels)
 504             field_selector = "involvedObject.name={}".format(deployment.name)
 505             field_selector += ",involvedObject.kind=Deployment"
 506             deployment.events = self.core.list_namespaced_event(
 507                 NAMESPACE,
 508                 field_selector=field_selector).items
 509
 510             if settings.STORE_ARTIFACTS:
 511                 self.jinja_env.get_template('deployment.html.j2').stream(
 512                     deployment=deployment).dump('{}/deployment-{}.html'.format(
 513                         self.res_dir, deployment.name))
 514
 515             if k8s.status.unavailable_replicas:
 516                 self.__logger.warning("a Deployment is in error: {}".format(deployment.name))
 517                 self.failing_deployments.append(deployment)
 518                 self.failing = True
 519
 520             self.deployments.append(deployment)
 521
 522     def parse_replicasets(self):
 523         """Parse the replicasets."""
 524         self.__logger.info("%4s replicasets to parse",
 525                            len(self.k8s_replicasets))
 526         for i in range(len(self.k8s_replicasets)):
 527             k8s = self.k8s_replicasets[i]
 528             replicaset = ReplicaSet(k8s=k8s)
 529
 530             if k8s.spec.selector and k8s.spec.selector.match_labels:
 531                 (replicaset.pods,
 532                  replicaset.failed_pods) = self._find_child_pods(
 533                      k8s.spec.selector.match_labels)
 534             field_selector = "involvedObject.name={}".format(replicaset.name)
 535             field_selector += ",involvedObject.kind=ReplicaSet"
 536             replicaset.events = self.core.list_namespaced_event(
 537                 NAMESPACE,
 538                 field_selector=field_selector).items
 539
 540             if settings.STORE_ARTIFACTS:
 541                 self.jinja_env.get_template('replicaset.html.j2').stream(
 542                     replicaset=replicaset).dump('{}/replicaset-{}.html'.format(
 543                         self.res_dir, replicaset.name))
 544
 545             if (not k8s.status.ready_replicas
 546                 or (k8s.status.ready_replicas < k8s.status.replicas)):
 547                 self.__logger.warning("a ReplicaSet is in error: {}".format(replicaset.name))
 548                 self.failing_replicasets.append(replicaset)
 549                 self.failing = True
 550
 551             self.replicasets.append(replicaset)
 552
 553     def parse_statefulsets(self):
 554         """Parse the statefulsets."""
 555         self.__logger.info("%4s statefulsets to parse",
 556                            len(self.k8s_statefulsets))
 557         for i in range(len(self.k8s_statefulsets)):
 558             k8s = self.k8s_statefulsets[i]
 559             statefulset = StatefulSet(k8s=k8s)
 560
 561             if k8s.spec.selector and k8s.spec.selector.match_labels:
 562                 (statefulset.pods,
 563                  statefulset.failed_pods) = self._find_child_pods(
 564                      k8s.spec.selector.match_labels)
 565             field_selector = "involvedObject.name={}".format(statefulset.name)
 566             field_selector += ",involvedObject.kind=StatefulSet"
 567             statefulset.events = self.core.list_namespaced_event(
 568                 NAMESPACE,
 569                 field_selector=field_selector).items
 570
 571             if settings.STORE_ARTIFACTS:
 572                 self.jinja_env.get_template('statefulset.html.j2').stream(
 573                     statefulset=statefulset).dump('{}/statefulset-{}.html'.format(
 574                         self.res_dir, statefulset.name))
 575
 576             if ((not k8s.status.ready_replicas)
 577                     or (k8s.status.ready_replicas < k8s.status.replicas)):
 578                 self.__logger.warning("a StatefulSet is in error: {}".format(statefulset.name))
 579                 self.failing_statefulsets.append(statefulset)
 580                 self.failing = True
 581
 582             self.statefulsets.append(statefulset)
 583
 584     def parse_daemonsets(self):
 585         """Parse the daemonsets."""
 586         self.__logger.info("%4s daemonsets to parse", len(self.k8s_daemonsets))
 587         for i in range(len(self.k8s_daemonsets)):
 588             k8s = self.k8s_daemonsets[i]
 589             daemonset = DaemonSet(k8s=k8s)
 590
 591             if k8s.spec.selector and k8s.spec.selector.match_labels:
 592                 (daemonset.pods,
 593                  daemonset.failed_pods) = self._find_child_pods(
 594                      k8s.spec.selector.match_labels)
 595             field_selector = "involvedObject.name={}".format(daemonset.name)
 596             field_selector += ",involvedObject.kind=DaemonSet"
 597             daemonset.events = self.core.list_namespaced_event(
 598                 NAMESPACE,
 599                 field_selector=field_selector).items
 600
 601             if settings.STORE_ARTIFACTS:
 602                 self.jinja_env.get_template('daemonset.html.j2').stream(
 603                     daemonset=daemonset).dump('{}/daemonset-{}.html'.format(
 604                         self.res_dir, daemonset.name))
 605
 606             if (k8s.status.number_ready < k8s.status.desired_number_scheduled):
 607                 self.__logger.warning("a DaemonSet is in error: {}".format(daemonset.name))
 608                 self.failing_daemonsets.append(daemonset)
 609                 self.failing = True
 610
 611             self.daemonsets.append(daemonset)
 612
 613     def parse_pvcs(self):
 614         """Parse the persistent volume claims."""
 615         self.__logger.info("%4s pvcs to parse", len(self.k8s_pvcs))
 616         for k8s in self.k8s_pvcs:
 617             pvc = Pvc(k8s=k8s)
 618             field_selector = f"involvedObject.name={pvc.name},involvedObject.kind=PersistentVolumeClaim"
 619             pvc.events = self.core.list_namespaced_event(
 620                 NAMESPACE,
 621                 field_selector=field_selector).items
 622
 623             if k8s.status.phase != "Bound":
 624                 self.__logger.warning("a PVC is in error: {}".format(pvc.name))
 625                 self.failing_pvcs.append(pvc)
 626                 self.failing = True
 627
 628             self.pvcs.append(pvc)
 629
 630     def parse_configmaps(self):
 631         """Parse the config maps."""
 632         self.__logger.info("%4s config maps to parse",
 633                            len(self.k8s_configmaps))
 634         for k8s in self.k8s_configmaps:
 635             configmap = ConfigMap(k8s=k8s)
 636             self.configmaps.append(configmap)
 637
 638     def parse_secrets(self):
 639         """Parse the secrets."""
 640         self.__logger.info("%4s secrets to parse", len(self.k8s_secrets))
 641         for k8s in self.k8s_secrets:
 642             secret = Secret(k8s=k8s)
 643             self.secrets.append(secret)
 644
 645     def parse_ingresses(self):
 646         """Parse the ingresses."""
 647         self.__logger.info("%4s ingresses to parse", len(self.k8s_ingresses))
 648         for k8s in self.k8s_secrets:
 649             ingress = Ingress(k8s=k8s)
 650             self.ingresses.append(ingress)
 651
 652     def parse_versions(self):
 653         """Parse the versions of the pods."""
 654         self.__logger.info("%4s pods to parse", len(self.k8s_pods))
 655         pod_versions = []
 656         containers = {}
 657         for pod in self.k8s_pods:
 658             pod_component = pod.metadata.name
 659             if 'app' in pod.metadata.labels:
 660                 pod_component = pod.metadata.labels['app']
 661             else:
 662                 if 'app.kubernetes.io/name' in pod.metadata.labels:
 663                     pod_component = pod.metadata.labels[
 664                         'app.kubernetes.io/name']
 665                 else:
 666                     self.__logger.error("pod %s has no 'app' or 'app.kubernetes.io/name' in metadata: %s", pod_component, pod.metadata.labels)
 667
 668             # looks for docker version
 669             for container in pod.spec.containers:
 670                 pod_version = {}
 671                 pod_container_version = container.image.rsplit(":", 1)
 672                 pod_container_image = pod_container_version[0]
 673                 pod_container_tag = "latest"
 674                 if len(pod_container_version) > 1:
 675                     pod_container_tag = pod_container_version[1]
 676
 677                 pod_version.update({
 678                     'container': container.name,
 679                     'component': pod_component,
 680                     'image': pod_container_image,
 681                     'version': pod_container_tag
 682                 })
 683                 pod_versions.append(pod_version)
 684
 685                 search_rule = "^(?P<source>[^/]*)/*(?P<container>[^:]*):*(?P<version>.*)$"
 686                 search = re.search(search_rule, container.image)
 687                 name = "{}/{}".format(search.group('source'),
 688                                       search.group('container'))
 689                 version = search.group('version')
 690                 if name[-1] == '/':
 691                     name = name[0:-1]
 692                 source = "default"
 693                 if search.group('source') in DOCKER_REPOSITORIES:
 694                     source = search.group('source')
 695                     name = search.group('container')
 696                 container_search_rule = "^library/(?P<real_container>[^:]*)$"
 697                 container_search = re.search(container_search_rule, name)
 698                 if container_search:
 699                     name = container_search.group('real_container')
 700                 for common_component in GENERIC_NAMES.keys():
 701                     if name in GENERIC_NAMES[common_component]:
 702                         version = "{}:{}".format(name, version)
 703                         name = common_component
 704                         break
 705
 706                 repository = DOCKER_REPOSITORIES_NICKNAMES[source]
 707                 if name in containers:
 708                     if version in containers[name]['versions']:
 709                         if not (pod_component in containers[name]['versions']
 710                                 [version]['components']):
 711                             containers[name]['versions'][version][
 712                                 'components'].append(pod_component)
 713                             containers[name]['number_components'] += 1
 714                         if not (repository in containers[name]['versions']
 715                                 [version]['repositories']):
 716                             containers[name]['versions'][version][
 717                                 'repositories'].append(repository)
 718                     else:
 719                         containers[name]['versions'][version] = {
 720                             'repositories': [repository],
 721                             'components': [pod_component]
 722                         }
 723                         containers[name]['number_components'] += 1
 724                 else:
 725                     containers[name] = {
 726                         'versions': {
 727                             version: {
 728                                 'repositories': [repository],
 729                                 'components': [pod_component]
 730                             }
 731                         },
 732                         'number_components': 1
 733                     }
 734
 735         if settings.STORE_ARTIFACTS:
 736             self.jinja_env.get_template('version.html.j2').stream(
 737                 pod_versions=pod_versions).dump('{}/versions.html'.format(
 738                     self.res_dir))
 739             self.jinja_env.get_template('container_versions.html.j2').stream(
 740                 containers=containers).dump('{}/container_versions.html'.format(
 741                     self.res_dir))
 742         # create a json file for version tracking
 743         with open(self.res_dir + "/onap_versions.json", "w") as write_file:
 744             json.dump(pod_versions, write_file)
 745
 746     def _find_child_pods(self, selector):
 747         pods_list = []
 748         failed_pods = 0
 749         if selector:
 750             raw_selector = ''
 751             for key, value in selector.items():
 752                 raw_selector += key + '=' + value + ','
 753             raw_selector = raw_selector[:-1]
 754             pods = self.core.list_namespaced_pod(
 755                 NAMESPACE, label_selector=raw_selector).items
 756             for pod in pods:
 757                 for known_pod in self.pods:
 758                     if known_pod.name == pod.metadata.name:
 759                         pods_list.append(known_pod)
 760                         if not known_pod.ready():
 761                             failed_pods += 1
 762         return (pods_list, failed_pods)
 763
 764     def map_by_name(self, resources):
 765         return list(map(lambda resource: resource.name, resources))