From 6a8304c51e129a7860d60ac562feac1f6056b5c3 Mon Sep 17 00:00:00 2001
From: Mahendra Raghuwanshi <mahendra.raghuwanshi@amdocs.com>
Date: Mon, 28 Jan 2019 09:49:30 +0000
Subject: [PATCH] Readiness check for the deployment/statefulsets

Change-Id: I8b9d2bd2720060cab215107bc0031c243155e9f6
Issue-ID: OOM-1611
Signed-off-by: Mahendra Raghuwanshi <mahendra.raghuwanshi@amdocs.com>
---
 kubernetes/readiness/docker/init/ready.py | 84 ++++++++++++++++++++++++-------
 1 file changed, 67 insertions(+), 17 deletions(-)

diff --git a/kubernetes/readiness/docker/init/ready.py b/kubernetes/readiness/docker/init/ready.py
index f4a5e5da8f..87c09a444c 100644
--- a/kubernetes/readiness/docker/init/ready.py
+++ b/kubernetes/readiness/docker/init/ready.py
@@ -30,6 +30,62 @@ configuration.ssl_ca_cert = cert
 configuration.api_key['authorization'] = token
 configuration.api_key_prefix['authorization'] = 'Bearer'
 coreV1Api = client.CoreV1Api(client.ApiClient(configuration))
+api_instance=client.ExtensionsV1beta1Api(client.ApiClient(configuration))
+api = client.AppsV1beta1Api(client.ApiClient(configuration))
+batchV1Api = client.BatchV1Api(client.ApiClient(configuration))
+
+def is_job_complete(job_name):
+    complete = False
+    log.info("Checking if " + job_name + "  is complete")
+    response = ""
+    try:
+        response = batchV1Api.read_namespaced_job_status(job_name, namespace)
+        if response.status.succeeded == 1:
+            job_status_type = response.status.conditions[0].type
+            if job_status_type == "Complete":
+                complete = True
+                log.info(job_name + " is complete")
+            else:
+                log.info(job_name + " is not complete")
+        else:
+            log.info(job_name + " has not succeeded yet")
+        return complete
+    except Exception as e:
+        log.error("Exception when calling read_namespaced_job_status: %s\n" % e)
+
+def wait_for_statefulset_complete(statefulset_name):
+    try:
+        response = api.read_namespaced_stateful_set(statefulset_name, namespace)
+        s = response.status
+        if ( s.updated_replicas == response.spec.replicas and
+                s.replicas == response.spec.replicas and
+                s.ready_replicas == response.spec.replicas and
+                s.current_replicas == response.spec.replicas and
+                s.observed_generation == response.metadata.generation):
+            log.info("Statefulset " + statefulset_name + "  is ready")
+            return True
+        else:
+            log.info("Statefulset " + statefulset_name + "  is not ready")
+        return False
+    except Exception as e:
+        log.error("Exception when waiting for Statefulset status: %s\n" % e)
+
+def wait_for_deployment_complete(deployment_name):
+    try:
+        response = api.read_namespaced_deployment(deployment_name, namespace)
+        s = response.status
+        if ( s.unavailable_replicas == None and
+                s.updated_replicas == response.spec.replicas and
+                s.replicas == response.spec.replicas and
+                s.ready_replicas == response.spec.replicas and
+                s.observed_generation == response.metadata.generation):
+            log.info("Deployment " + deployment_name + "  is ready")
+            return True
+        else:
+            log.info("Deployment " + deployment_name + "  is not ready")
+        return False
+    except Exception as e:
+        log.error("Exception when waiting for deployment status: %s\n" % e)
 
 def is_ready(container_name):
     ready = False
@@ -41,28 +97,23 @@ def is_ready(container_name):
             if i.status.container_statuses is None:
                 continue
             for s in i.status.container_statuses:
-                if i.metadata.owner_references[0].kind  == "StatefulSet":
-                    if i.metadata.name == container_name:
-                        ready = s.ready
-                        if not ready:
-                            log.info(container_name + " is not ready.")
-                        else:
-                            log.info(container_name + " is ready!")
-                    else:
-                        continue
-                elif s.name == container_name:
-                    ready = s.ready
-                    if not ready:
-                        log.info(container_name + " is not ready.")
-                    else:
-                        log.info(container_name + " is ready!")
+                if s.name == container_name:
+                    if i.metadata.owner_references[0].kind  == "StatefulSet":
+                        ready = wait_for_statefulset_complete(i.metadata.owner_references[0].name)
+                    elif i.metadata.owner_references[0].kind == "ReplicaSet":
+                        api_response = api_instance.read_namespaced_replica_set_status(i.metadata.owner_references[0].name, namespace)
+                        ready = wait_for_deployment_complete(api_response.metadata.owner_references[0].name)
+                    elif i.metadata.owner_references[0].kind == "Job":
+                        ready = is_job_complete(i.metadata.owner_references[0].name)
+
+                    return ready
+
                 else:
                     continue
         return ready
     except Exception as e:
         log.error("Exception when calling list_namespaced_pod: %s\n" % e)
 
-
 DEF_TIMEOUT = 10
 DESCRIPTION = "Kubernetes container readiness check utility"
 USAGE = "Usage: ready.py [-t <timeout>] -c <container_name> [-c <container_name> ...]\n" \
@@ -105,7 +156,6 @@ def main(argv):
             else:
                 time.sleep(5)
 
-
 if __name__ == "__main__":
     main(sys.argv[1:])
 
-- 
2.16.6