From 646446fdd7a37124e4d5d31cf855296cf2ec8024 Mon Sep 17 00:00:00 2001 From: Rajamohan Raj Date: Tue, 18 Jun 2019 22:29:41 +0000 Subject: [PATCH] Helmize the sample promql_query_app Release a helm chart and dockerfile for the sample promql_query_app Issue-ID: ONAPARC-508 Signed-off-by: Rajamohan Raj Change-Id: I8fdddd3fe0094c132a485c1d40b524b8a8cb2062 --- vnfs/DAaaS/lib/promql_api/README.md | 2 +- vnfs/DAaaS/sample-apps/m3db_promql/.helmignore | 24 +++++++ vnfs/DAaaS/sample-apps/m3db_promql/Chart.yaml | 5 ++ .../sample-apps/m3db_promql/build/.dockerignore | 12 ++++ .../DAaaS/sample-apps/m3db_promql/build/Dockerfile | 78 ++++++++++++++++++++ .../sample-apps/m3db_promql/build/entrypoint.sh | 45 ++++++++++++ .../sample-apps/m3db_promql/build/requirements.txt | 1 + .../m3db_promql/{ => build}/sample_promql_query.py | 0 .../sample-apps/m3db_promql/templates/_helpers.tpl | 32 +++++++++ .../templates/configmap_promql_app_env.yaml | 6 ++ .../m3db_promql/templates/deployment.yaml | 84 ++++++++++++++++++++++ .../sample-apps/m3db_promql/templates/secrets.yaml | 31 ++++++++ .../m3db_promql/templates/serviceaccount.yaml | 25 +++++++ vnfs/DAaaS/sample-apps/m3db_promql/values.yaml | 60 ++++++++++++++++ .../sample-apps/sample-minio-load-model/Chart.yaml | 2 +- 15 files changed, 405 insertions(+), 2 deletions(-) create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/.helmignore create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/Chart.yaml create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/build/.dockerignore create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/build/Dockerfile create mode 100755 vnfs/DAaaS/sample-apps/m3db_promql/build/entrypoint.sh create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/build/requirements.txt rename vnfs/DAaaS/sample-apps/m3db_promql/{ => build}/sample_promql_query.py (100%) create mode 100755 vnfs/DAaaS/sample-apps/m3db_promql/templates/_helpers.tpl create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/templates/configmap_promql_app_env.yaml create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/templates/deployment.yaml create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/templates/secrets.yaml create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/templates/serviceaccount.yaml create mode 100644 vnfs/DAaaS/sample-apps/m3db_promql/values.yaml diff --git a/vnfs/DAaaS/lib/promql_api/README.md b/vnfs/DAaaS/lib/promql_api/README.md index f2d45951..89f996af 100644 --- a/vnfs/DAaaS/lib/promql_api/README.md +++ b/vnfs/DAaaS/lib/promql_api/README.md @@ -22,7 +22,7 @@ The return is: 'service': 'collectd'}, 'value': [1559177169.415, '119727200']}] ``` -i + ## How to use this API ? ``` diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/.helmignore b/vnfs/DAaaS/sample-apps/m3db_promql/.helmignore new file mode 100644 index 00000000..33aaef25 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +build \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/Chart.yaml b/vnfs/DAaaS/sample-apps/m3db_promql/Chart.yaml new file mode 100644 index 00000000..5bfedbf8 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: Helm chart for a sample promql query app +name: promql_query_app +version: 0.1.0 diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/build/.dockerignore b/vnfs/DAaaS/sample-apps/m3db_promql/build/.dockerignore new file mode 100644 index 00000000..98bc4363 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/build/.dockerignore @@ -0,0 +1,12 @@ +.git +.ipynb_checkpoints/* +Dockerfile +.DS_Store +.gitignore +README.md +env.* +/devops/* + +# To prevent storing dev/temporary container data +*.csv +/tmp/* \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/build/Dockerfile b/vnfs/DAaaS/sample-apps/m3db_promql/build/Dockerfile new file mode 100644 index 00000000..61b15fee --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/build/Dockerfile @@ -0,0 +1,78 @@ +# Copyright (c) 2019 Intel Corporation +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +FROM ubuntu:18.04 + +RUN apt update -yqq + +# Install all the essentials +RUN apt-get update --fix-missing && \ + apt-get install -y --no-install-recommends wget curl ca-certificates libglib2.0-0 libxext6 libsm6 libxrender1 \ + git build-essential openssh-server openssh-client && \ + mkdir -p /var/run/sshd && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PATH /opt/conda/bin:$PATH + +# Install miniconda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + +# Install python libraries using conda in the virtual_environment:ml_env +SHELL ["/bin/bash", "-c"] +RUN conda update -n base -c defaults conda && \ + conda create -n ml_env +RUN conda install -n ml_env -y -c anaconda pip +RUN pwd +RUN conda install -n ml_env -y -c anaconda h5py + +RUN conda install -n ml_env -y -c pytorch pytorch-cpu +RUN conda install -n ml_env -y -c conda-forge nlopt + +RUN echo "conda activate ml_env" >> ~/.bashrc +RUN source ~/.bashrc +RUN /opt/conda/envs/ml_env/bin/pip install --no-cache-dir minio + +# Install tini +RUN apt-get install -y --no-install-recommends curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && echo ${TINI_VERSION} && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt clean + +# This is needed to match the original entrypoint.sh file. +RUN cp /usr/bin/tini /sbin +RUN echo "export PATH=/opt/conda/envs/ml_env/bin:$PATH" >> ~/.bashrc + +COPY . /app +WORKDIR /app + +RUN mkdir promql_api +RUN wget https://raw.githubusercontent.com/onap/demo/master/vnfs/DAaaS/lib/promql_api/prom_ql_api.py +RUN mv prom_ql_api.py ./promql_api +RUN touch ./promql_api/__init__.py +RUN source ~/.bashrc +RUN conda install -n ml_env -y --file requirements.txt + +ENTRYPOINT [ "/app/entrypoint.sh" ] diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/build/entrypoint.sh b/vnfs/DAaaS/sample-apps/m3db_promql/build/entrypoint.sh new file mode 100755 index 00000000..571e03ca --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/build/entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# echo commands to the terminal output +set -ex + +# Check whether there is a passwd entry for the container UID +myuid=$(id -u) +mygid=$(id -g) +# turn off -e for getent because it will return error code in anonymous uid case +set +e +uidentry=$(getent passwd $myuid) +set -e + +# If there is no passwd entry for the container UID, attempt to create one +if [ -z "$uidentry" ] ; then + if [ -w /etc/passwd ] ; then + echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd + else + echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" + fi +fi + +# CMD=( +# /bin/bash -c "source activate ml_env && exec python sample_promql_query.py") + +CMD=(/bin/bash -c "source activate ml_env && exec python sample_promql_query.py") + +# Execute the container CMD under tini for better hygiene +exec /sbin/tini -s -- "${CMD[@]}" diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/build/requirements.txt b/vnfs/DAaaS/sample-apps/m3db_promql/build/requirements.txt new file mode 100644 index 00000000..663bd1f6 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/build/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/sample_promql_query.py b/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py similarity index 100% rename from vnfs/DAaaS/sample-apps/m3db_promql/sample_promql_query.py rename to vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/templates/_helpers.tpl b/vnfs/DAaaS/sample-apps/m3db_promql/templates/_helpers.tpl new file mode 100755 index 00000000..f3ee5322 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "promql_query_app.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "promql_query_app.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "promql_query_app.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/templates/configmap_promql_app_env.yaml b/vnfs/DAaaS/sample-apps/m3db_promql/templates/configmap_promql_app_env.yaml new file mode 100644 index 00000000..1f54f341 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/templates/configmap_promql_app_env.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: configmap_promql_app_env +data: + DATA_ENDPOINT: {{ .Values.prometheus.serverURL }} \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/templates/deployment.yaml b/vnfs/DAaaS/sample-apps/m3db_promql/templates/deployment.yaml new file mode 100644 index 00000000..99a4e2ad --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/templates/deployment.yaml @@ -0,0 +1,84 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ include "promql_query_app.fullname" . }} + labels: + app: {{ include "promql_query_app.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service | quote }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "promql_query_app.name" . }} + release: {{ .Release.Name }} + modelName: {{ .Values.modelName }} + template: + metadata: + labels: + app: {{ include "promql_query_app.name" . }} + release: {{ .Release.Name }} + modelName: {{ .Values.modelName }} + spec: + serviceAccountName: {{ template "promql_query_app.serviceAccountName" . }} + containers: + - name: promql_query_app + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: DATA_ENDPOINT + valueFrom: + configMapKeyRef: + name: configmap_promql_app_env + key: DATA_ENDPOINT + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "promql_query_app.fullname" . }}{{ end }} + key: accesskey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "promql_query_app.fullname" . }}{{ end }} + key: secretkey + {{- range $key, $val := .Values.minio.environment }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end}} + resources: + {{ toYaml .Values.resources | indent 12 }} + volumeMounts: + - mountPath: /models + name: models + volumes: + - name: models + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/templates/secrets.yaml b/vnfs/DAaaS/sample-apps/m3db_promql/templates/secrets.yaml new file mode 100644 index 00000000..60717a65 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/templates/secrets.yaml @@ -0,0 +1,31 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{- if not .Values.minio.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "model-loader.fullname" . }} + labels: + app: {{ include "model-loader.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service | quote }} +type: Opaque +data: + accesskey: {{ .Values.minio.accessKey | b64enc }} + secretkey: {{ .Values.minio.secretKey | b64enc }} +{{- end }} diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/templates/serviceaccount.yaml b/vnfs/DAaaS/sample-apps/m3db_promql/templates/serviceaccount.yaml new file mode 100644 index 00000000..1573ae94 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/templates/serviceaccount.yaml @@ -0,0 +1,25 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "model-loader.fullname" . }} + labels: + app: {{ include "model-loader.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service | quote }} diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/values.yaml b/vnfs/DAaaS/sample-apps/m3db_promql/values.yaml new file mode 100644 index 00000000..7a669847 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/m3db_promql/values.yaml @@ -0,0 +1,60 @@ +# Default values for promql_query_app. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: dcr.cluster.local:32644/promql_query_app + tag: v3 + pullPolicy: Always + +nameOverride: "" +fullnameOverride: "" + + +prometheus: + serverURL: http://172.25.103.1:30090 + +## Model repository information (Minio) +## Model repository information (Minio) +minio: + existingSecret: "" + accessKey: "onapdaas" + secretKey: "onapsecretdaas" + environment: + AWS_REGION: "us-west-1" + S3_REGION: "us-west-1" + S3_ENDPOINT: "minio.edge1.svc.cluster.local:9000" + AWS_ENDPOINT_URL: "http://minio.edge1.svc.cluster.local:9000" + S3_USE_HTTPS: 0 + S3_VERIFY_SSL: 0 + AWS_LOG_LEVEL: 3 + TF_CPP_MIN_LOG_LEVEL: 3 + MODEL_BUCKET: "numpy" + # List of object files to be loaded given in a , separated list format + MODEL_OBJECTS: '["model1.data", "model2.data"]' + +## Service account for model-loader to use. +serviceAccount: + create: true + name: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml b/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml index 084b7d80..6c7c120e 100644 --- a/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 appVersion: "1.0" -description: A Helm chart for Kubernetes +description: A Helm chart for loading a model from minio name: model-loader version: 0.1.0 -- 2.16.6