Add healthcheck container for OOM 41/43941/1
authorJack Lucas <jflucas@research.att.com>
Fri, 20 Apr 2018 13:22:05 +0000 (13:22 +0000)
committerJack Lucas <jflucas@research.att.com>
Fri, 20 Apr 2018 13:22:58 +0000 (13:22 +0000)
Change-Id: Ie4719a0e4705901fd9d0fa3504696fbefc6c704a
Issue-ID: DCAEGEN2-461
Signed-off-by: Jack Lucas <jflucas@research.att.com>
healthcheck-container/Dockerfile [new file with mode: 0644]
healthcheck-container/get-status.js [new file with mode: 0644]
healthcheck-container/healthcheck.js [new file with mode: 0644]
healthcheck-container/package.json [new file with mode: 0644]
healthcheck-container/pom.xml [new file with mode: 0644]
mvn-phase-script.sh
pom.xml

diff --git a/healthcheck-container/Dockerfile b/healthcheck-container/Dockerfile
new file mode 100644 (file)
index 0000000..d1b4231
--- /dev/null
@@ -0,0 +1,8 @@
+FROM node:8.11.1
+RUN mkdir -p /opt/app
+COPY *.js /opt/app/
+COPY package.json /opt/app/
+WORKDIR /opt/app
+RUN npm install --only=production
+EXPOSE 80
+ENTRYPOINT ["/usr/local/bin/node", "healthcheck.js"]
diff --git a/healthcheck-container/get-status.js b/healthcheck-container/get-status.js
new file mode 100644 (file)
index 0000000..2ed1d3d
--- /dev/null
@@ -0,0 +1,109 @@
+/*
+Copyright(c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+/*
+ * Query Kubernetes for status of deployments and extract readiness information
+ */
+
+const fs = require('fs');
+const request = require('request');
+
+const K8S_CREDS = '/var/run/secrets/kubernetes.io/serviceaccount';
+const K8S_API = 'https://kubernetes.default.svc.cluster.local/';       // Full name to match cert for TLS
+const K8S_PATH = 'apis/apps/v1beta2/namespaces/';
+
+//Get token and CA cert
+const ca = fs.readFileSync(K8S_CREDS + '/ca.crt');
+const token = fs.readFileSync(K8S_CREDS + '/token');
+
+const summarizeDeploymentList = function(list) {
+       // list is a DeploymentList object returned by k8s
+       // Individual deployments are in the array 'items'
+       
+       let ret = 
+       {
+               type: "summary",
+               count: 0,
+               ready: 0,
+               items: []
+       };
+       
+       // Extract readiness information
+       for (let deployment of list.items) {
+               ret.items.push(
+                       {
+                               name: deployment.metadata.name,
+                               ready: deployment.status.readyReplicas || 0,
+                               unavailable: deployment.status.unavailableReplicas || 0
+                       }
+               );
+               ret.count ++;
+               ret.ready = ret.ready + (deployment.status.readyReplicas || 0);
+       }
+       
+       return ret;
+};
+
+const summarizeDeployment = function(deployment) {
+       // deployment is a Deployment object returned by k8s
+       // we make it look enough like a DeploymentList object to
+       // satisfy summarizeDeploymentList
+       return summarizeDeploymentList({items: [deployment]});
+};
+
+const queryKubernetes = function(path, callback) {
+       // Make request to Kubernetes
+       
+       const options = {
+               url: K8S_API + path,
+               ca : ca,
+               headers: {
+                       Authorization: 'bearer ' + token
+               },
+               json: true
+       };
+       console.log ("request url: " + options.url);
+       request(options, function(error, res, body) {
+               console.log ("status: " + (res && res.statusCode) ? res.statusCode : "NONE");
+               if (error) {
+                       console.log("error: " + error);
+               }
+               callback(error, res, body);
+       });
+};
+
+const getStatus = function(path, extract, callback) {
+       // Get info from k8s and extract readiness info
+       queryKubernetes(path, function(error, res, body) {
+               let ret = body;
+               if (!error && res && res.statusCode === 200) {
+                       ret = extract(body);
+               }
+               callback (error, res, ret);
+       });
+};
+
+exports.getStatusNamespace = function (namespace, callback) {
+       // Get readiness information for all deployments in namespace
+       const path = K8S_PATH + namespace + '/deployments';
+       getStatus(path, summarizeDeploymentList, callback);
+};
+
+exports.getStatusSingle = function (namespace, deployment, callback) {
+       // Get readiness information for a single deployment
+       const path = K8S_PATH + namespace + '/deployments/' + deployment;
+       getStatus(path, summarizeDeployment, callback);
+};
\ No newline at end of file
diff --git a/healthcheck-container/healthcheck.js b/healthcheck-container/healthcheck.js
new file mode 100644 (file)
index 0000000..7555032
--- /dev/null
@@ -0,0 +1,80 @@
+/*
+Copyright(c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+//Expect ONAP and DCAE namespaces and Helm "release" name to be passed via environment variables
+// 
+const ONAP_NS = process.env.ONAP_NAMESPACE || 'default';
+const DCAE_NS = process.env.DCAE_NAMESPACE || 'default';
+const HELM_REL = process.env.HELM_RELEASE || '';
+
+const HEALTHY = 200;
+const UNHEALTHY = 500;
+const UNKNOWN = 503;
+
+const status = require('./get-status');
+const http = require('http');
+
+const isHealthy = function(summary) {
+       // Current healthiness criterion is simple--all deployments are ready
+       return summary.count && summary.ready && summary.count === summary.ready;
+};
+
+const checkHealth = function (callback) {
+       // Makes queries to Kubernetes and checks results
+       // If we encounter some kind of error contacting k8s (or other), health status is UNKNOWN (500)
+       // If we get responses from k8s but don't find all deployments ready, health status is UNHEALTHY (503)
+       // If we get responses from k8s and all deployments are ready, health status is HEALTHY (200)
+       // This could be a lot more nuanced, but what's here should be sufficient for R2 OOM healthchecking
+       status.getStatusNamespace(DCAE_NS, function(err, res, body) {
+               let ret = {status : UNKNOWN, body: [body]};
+               if (err) {
+                       callback(ret);
+               }
+               else if (body.type && body.type === 'summary') {
+                       if (isHealthy(body)) {
+                               // All the DCAE components report healthy -- check Cloudify Manager
+                               let cmDeployment = 'dcae-cloudify-manager';
+                               if (HELM_REL.length > 0) {
+                                       cmDeployment = HELM_REL + '-' + cmDeployment;
+                               }
+                               status.getStatusSingle(ONAP_NS, cmDeployment, function (err, res, body){
+                                       ret.body.push(body);
+                                       if (err) {
+                                               callback(ret);
+                                       }
+                                       if (body.type && body.type === 'summary') {
+                                               ret.status = isHealthy(body) ? HEALTHY : UNHEALTHY;
+                                       }
+                                       callback(ret);
+                               });
+                       }
+                       else {
+                               callback(ret);
+                       }
+               }
+       });
+};
+
+// Simple HTTP server--any incoming request triggers a health check
+const server = http.createServer(function(req, res) {
+       checkHealth(function(ret) {
+               console.log ((new Date()).toISOString() + ": " + JSON.stringify(ret));
+               res.statusCode = ret.status;
+               res.setHeader('Content-Type', 'application/json');
+               res.end(JSON.stringify(ret.body || {}), 'utf8');
+       });
+});
+server.listen(80);
diff --git a/healthcheck-container/package.json b/healthcheck-container/package.json
new file mode 100644 (file)
index 0000000..2a08bdd
--- /dev/null
@@ -0,0 +1,11 @@
+{
+  "name": "k8s-healthcheck",
+  "description": "DCAE healthcheck server",
+  "version": "1.0.0",
+  "main": "healthcheck.js",
+  "dependencies": {
+    "request": "2.85.0"
+  },
+  "author": "author",
+  "license": "(Apache-2.0)"
+}
diff --git a/healthcheck-container/pom.xml b/healthcheck-container/pom.xml
new file mode 100644 (file)
index 0000000..dea3c48
--- /dev/null
@@ -0,0 +1,172 @@
+<?xml version="1.0"?>
+<!--
+================================================================================
+Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+================================================================================
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+============LICENSE_END=========================================================
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.onap.dcaegen2.deployments</groupId>
+    <artifactId>deployments</artifactId>
+    <version>1.2.0-SNAPSHOT</version>
+  </parent>
+  <groupId>org.onap.dcaegen2.deployments</groupId>
+  <artifactId>healthcheck-container</artifactId>
+  <name>dcaegen2-deployments-healthcheck-container</name>
+  <version>1.0.0</version>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <sonar.skip>true</sonar.skip>
+    <sonar.sources>.</sonar.sources>
+    <!-- customize the SONARQUBE URL -->
+    <!-- sonar.host.url>http://localhost:9000</sonar.host.url -->
+    <!-- below are language dependent -->
+    <!-- for Python -->
+    <sonar.language>py</sonar.language>
+    <sonar.pluginName>Python</sonar.pluginName>
+    <sonar.inclusions>**/*.py</sonar.inclusions>
+    <!-- for JavaScaript -->
+    <!--
+    <sonar.language>js</sonar.language>
+    <sonar.pluginName>JS</sonar.pluginName>
+    <sonar.inclusions>**/*.js</sonar.inclusions>
+    -->
+  </properties>
+  <build>
+    <finalName>${project.artifactId}-${project.version}</finalName>
+    <plugins>
+      <!-- plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.4.1</version>
+        <configuration>
+          <descriptors>
+            <descriptor>assembly/dep.xml</descriptor>
+          </descriptors>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin -->
+      <!-- now we configure custom action (calling a script) at various lifecycle phases -->
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <executions>
+          <execution>
+            <id>clean phase script</id>
+            <phase>clean</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>clean</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>generate-sources script</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>generate-sources</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>compile script</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>compile</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>package script</id>
+            <phase>package</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>package</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>test script</id>
+            <phase>test</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>test</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>install script</id>
+            <phase>install</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>install</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>deploy script</id>
+            <phase>deploy</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>deploy</argument>
+              </arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
index 9897d5e..acada60 100755 (executable)
@@ -83,7 +83,7 @@ deploy)
     upload_files_of_extension sh
     build_and_push_docker
     ;;
-  k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container)
+  k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container|healthcheck-container)
     build_and_push_docker
     ;;
   scripts|cloud_init|heat)
diff --git a/pom.xml b/pom.xml
index ce585ba..ae1f1b9 100644 (file)
--- a/pom.xml
+++ b/pom.xml
@@ -44,6 +44,7 @@ limitations under the License.
      <module>cm-container</module>
      <module>k8s-bootstrap-container</module>
      <module>tca-cdap-container</module>
+     <module>healthcheck-container</module>
   </modules>
 
   <properties>