From 091f88e5953c02b622a20c7a7fa1e49df4666116 Mon Sep 17 00:00:00 2001 From: Alexander Dehn Date: Thu, 27 Feb 2020 11:37:46 +0000 Subject: [PATCH] Enhancements for common templates _labels.tpl: - support of additional customized labels in common.labels, common.matchLabels, common.selectors common.templateMetadata - support of name suffix in common.resourceMetadata _name.tpl: - support of name suffix in common.name, common.fullname, common.fullnameExplicit _service.tpl - support of additional customized labels in common.serviceMetadata, common.*service - support of sessionAffinity in common.service New common template: _aafconfig - new common template to enable charts for AAF includes templates for init container, volumemounts, pvc and pv Issue-ID: SDNC-1088 Change-Id: Icbaa806608f9e1f36f0e47686668ae3632d3f2b0 Signed-off-by: Alexander Dehn Signed-off-by: Sylvain Desbureaux --- kubernetes/common/common/templates/_aafconfig.tpl | 226 ++++++++++++++++++++++ kubernetes/common/common/templates/_labels.tpl | 64 ++++-- kubernetes/common/common/templates/_name.tpl | 22 ++- kubernetes/common/common/templates/_service.tpl | 54 ++++-- 4 files changed, 329 insertions(+), 37 deletions(-) create mode 100644 kubernetes/common/common/templates/_aafconfig.tpl diff --git a/kubernetes/common/common/templates/_aafconfig.tpl b/kubernetes/common/common/templates/_aafconfig.tpl new file mode 100644 index 0000000000..db7cbe8d8d --- /dev/null +++ b/kubernetes/common/common/templates/_aafconfig.tpl @@ -0,0 +1,226 @@ +{{/* +# Copyright © 2020 Amdocs, Bell Canada, highstreet technologies GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{/* + common templates to enable aaf configs for applictaions + + Parameter for aafConfig to be defined in values.yaml + aafConfig: --> if a different key is used, call templates with argument (dict "aafRoot" "" "dot" .) + # additional scripts can be defined to handle certs + addconfig: true|false + fqdn: "sdnc" + image: onap/aaf/aaf_agent:2.1.15 + app_ns: "org.osaaf.aaf" + fqi: "sdnc@sdnc.onap.org" + fqi_namespace: org.onap.sdnc + public_fqdn: "aaf.osaaf.org" + aafDeployFqi: "deployer@people.osaaf.org" + aafDeployPass: demo123456! + cadi_latitude: "38.0" + cadi_longitude: "-72.0" + persistence: + enabled: true + config.volumeReclaimPolicy: Delete + config.accessMode: ReadWriteMany + config.size: 40M + config.storageClass: "manual" + config.mountPath: "/dockerdata-nfs" + config.mountSubPath: "sdnc/aaf" + # secrets configuration, Note: create a secrets template + secrets: + - uid: aaf-deploy-creds + type: basicAuth + externalSecret: '{{ ternary (tpl (default "" .Values.aafConfig.aafDeployCredsExternalSecret) .) "aafIsDiabled" .Values.global.aafEnabled }}' + login: '{{ .Values.aafConfig.aafDeployFqi }}' + password: '{{ .Values.aafConfig.aafDeployPass }}' + passwordPolicy: required + + In deployments/jobs/stateful include: + initContainers: + {{ include "common.aaf-config" . | nindent XX}} + + containers: + volumeMounts: + {{- if .Values.global.aafEnabled }} + - mountPath: "/opt/app/osaaf" + name: {{ include "common.fullname" . }}-aaf-config-vol + {{- end }} + volumes: + {{- include "common.aaf-config-volumes" . | nindent XX}} + + If persistence.enabled = true + Create pvc: + {{ include "common.aaf-config-pvc" . }} + Create pv + {{ include "common.aaf-config-pv" . }} + +*/}} +{{- define "common.aaf-config" -}} +{{- $dot := default . .dot -}} +{{- $aafRoot := default "aafConfig" .aafRoot -}} +{{ if .Values.global.aafEnabled }} +- name: {{ include "common.name" . }}-aaf-readiness + image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + command: + - /root/ready.py + args: + - --container-name + - aaf-locate + - --container-name + - aaf-cm + - --container-name + - aaf-service + + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace +- name: {{ include "common.name" . }}-aaf-config + image: {{ .Values.global.repository }}/{{index .Values $aafRoot "image" }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + volumeMounts: + - mountPath: "/opt/app/osaaf" + name: {{ include "common.fullname" . }}-aaf-config-vol + {{- if (index .Values $aafRoot "addconfig") }} + - name: aaf-add-config + mountPath: /opt/app/aaf_config/bin/aaf-add-config.sh + subPath: aaf-add-config.sh + {{- end }} + command: + - sh + - -c + - | + #!/usr/bin/env bash + /opt/app/aaf_config/bin/agent.sh + {{- if (index .Values $aafRoot "addconfig") }} + /opt/app/aaf_config/bin/aaf-add-config.sh + {{- end }} + env: + - name: APP_FQI + value: "{{ index .Values $aafRoot "fqi" }}" + - name: aaf_locate_url + value: "https://aaf-locate.{{ .Release.Namespace}}:8095" + - name: aaf_locator_container + value: "oom" + - name: aaf_locator_container_ns + value: "{{ .Release.Namespace }}" + - name: aaf_locator_fqdn + value: "{{ index .Values $aafRoot "fqdn" }}" + - name: aaf_locator_app_ns + value: "{{ index .Values $aafRoot "app_ns" }}" + - name: DEPLOY_FQI + {{- include "common.secret.envFromSecret" (dict "global" . "uid" "aaf-deploy-creds" "key" "login") | indent 6 }} + - name: DEPLOY_PASSWORD + {{- include "common.secret.envFromSecret" (dict "global" . "uid" "aaf-deploy-creds" "key" "password") | indent 6 }} + #Note: want to put this on Nodes, evenutally + - name: cadi_longitude + value: "{{ default "52.3" (index .Values $aafRoot "cadi_longitude") }}" + - name: cadi_latitude + value: "{{ default "13.2" (index .Values $aafRoot "cadi_latitude") }}" + #Hello specific. Clients don't don't need this, unless Registering with AAF Locator + - name: aaf_locator_public_fqdn + value: "{{ (index .Values $aafRoot "public_fqdn") | default "" }}" +{{- end -}} +{{- end -}} + + +{{- define "common.aaf-config-volume-mountpath" -}} +{{ if .Values.global.aafEnabled }} +- mountPath: "/opt/app/osaaf" + name: {{ include "common.fullname" . }}-aaf-config-vol +{{- end -}} +{{- end -}} + +{{- define "common.aaf-config-volumes" -}} +{{ if .Values.global.aafEnabled }} +{{- $dot := default . .dot -}} +{{- $aafRoot := default "aafConfig" .aafRoot -}} +- name: {{ include "common.fullname" . }}-aaf-config-vol + persistentVolumeClaim: + claimName: {{ include "common.fullname" . }}-aaf-config-pvc +{{- if (index .Values $aafRoot "addconfig") }} +- name: aaf-add-config + configMap: + name: {{ include "common.fullname" . }}-aaf-add-config + defaultMode: 0700 +{{- end }} +{{- end -}} +{{- end }} + +{{- define "common.aaf-config-pv" -}} +{{- $dot := default . .dot -}} +{{- $aafRoot := default "aafConfig" .aafRoot -}} +metadata: + name: {{ include "common.fullname" . }}-aaf-config-pv + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }}-aaf-config-pv + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + name: {{ include "common.fullname" . }} +spec: + capacity: + storage: {{ index .Values $aafRoot "persistence" "config" "size"}} + accessModes: + - {{ index .Values $aafRoot "persistence" "config" "accessMode" }} + persistentVolumeReclaimPolicy: {{ index .Values $aafRoot "persistence" "config" "volumeReclaimPolicy" }} + hostPath: + path: {{ index .Values $aafRoot "persistence" "config" "mountPath" }}/{{ .Release.Name }}/{{ index .Values $aafRoot "persistence" "config" "mountSubPath" }} +{{- if (index .Values $aafRoot "persistence" "config" "storageClass") }} +{{- if (eq "-" (index .Values $aafRoot "persistence" "config" "storageClass")) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ index .Values $aafRoot "persistence" "config" "storageClass" }}" +{{- end }} +{{- end }} +{{- end -}} + +{{- define "common.aaf-config-pvc" -}} +{{- $dot := default . .dot -}} +{{- $aafRoot := default "aafConfig" .aafRoot -}} +metadata: + name: {{ include "common.fullname" . }}-aaf-config-pvc + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +{{- if (index .Values $aafRoot "persistence" "annotations") }} + annotations: +{{ toYaml (index .Values $aafRoot "persistence" "annotations" ) | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app: {{ include "common.name" . }}-aaf-config-pv + accessModes: + - {{ index .Values $aafRoot "persistence" "config" "accessMode" }} + resources: + requests: + storage: {{ index .Values $aafRoot "persistence" "config" "size" }} +{{- if (index .Values $aafRoot "persistence" "config" "storageClass") }} +{{- if (eq "-" (index .Values $aafRoot "persistence" "config" "storageClass")) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ index .Values $aafRoot "persistence" "config" "storageClass" }}" +{{- end }} +{{- end }} +{{- end -}} diff --git a/kubernetes/common/common/templates/_labels.tpl b/kubernetes/common/common/templates/_labels.tpl index 95d51e17b7..854019c197 100644 --- a/kubernetes/common/common/templates/_labels.tpl +++ b/kubernetes/common/common/templates/_labels.tpl @@ -18,45 +18,81 @@ {{/* Common labels +The function takes several arguments (inside a dictionary): + - .dot : environment (.) + - .labels : labels to add (dict) */}} {{- define "common.labels" -}} -app.kubernetes.io/name: {{ include "common.name" . }} -helm.sh/chart: {{ include "common.chart" . }} -app.kubernetes.io/instance: {{ include "common.release" . }} -app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- $dot := default . .dot -}} +app.kubernetes.io/name: {{ include "common.name" $dot }} +helm.sh/chart: {{ include "common.chart" $dot }} +app.kubernetes.io/instance: {{ include "common.release" $dot }} +app.kubernetes.io/managed-by: {{ $dot.Release.Service }} +{{ if .labels }} +{{- include "common.tplValue" (dict "value" .labels "context" $dot) }} +{{- end -}} {{- end -}} {{/* Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +The function takes several arguments (inside a dictionary): + - .dot : environment (.) + - .matchLabels: selectors/matchlLabels to add (dict) */}} {{- define "common.matchLabels" -}} -app.kubernetes.io/name: {{ include "common.name" . }} -app.kubernetes.io/instance: {{ include "common.release" . }} +{{- $dot := default . .dot -}} +{{- if not .matchLabels.nameNoMatch -}} +app.kubernetes.io/name: {{ include "common.name" $dot }} +{{- end }} +app.kubernetes.io/instance: {{ include "common.release" $dot }} +{{ if .matchLabels }} +{{$_ := unset .matchLabels "nameNoMatch"}} +{{- include "common.tplValue" (dict "value" .matchLabels "context" $dot) }} +{{- end -}} {{- end -}} {{/* Generate "top" metadata for Deployment / StatefulSet / ... + The function takes several arguments (inside a dictionary): + - .dot : environment (.) + - .labels: labels to add (dict) + - .suffix: suffix to name + */}} {{- define "common.resourceMetadata" -}} -name: {{ include "common.fullname" . }} -namespace: {{ include "common.namespace" . }} -labels: {{- include "common.labels" . | nindent 2 }} +{{- $dot := default . .dot -}} +{{- $suffix := default "" .suffix -}} +{{- $labels := default (dict) .labels -}} + +name: {{ include "common.fullname" (dict "suffix" $suffix "dot" $dot )}} +namespace: {{ include "common.namespace" $dot }} +labels: {{- include "common.labels" (dict "labels" $labels "dot" $dot ) | nindent 2 }} {{- end -}} {{/* Generate selectors for Deployment / StatefulSet / ... + The function takes several arguments (inside a dictionary): + - .dot : environment (.) + - .matchLabels: labels to add (dict) */}} {{- define "common.selectors" -}} -matchLabels: {{- include "common.matchLabels" . | nindent 2 }} +{{- $dot := default . .dot -}} +{{- $matchLabels := default (dict) .matchLabels -}} +matchLabels: {{- include "common.matchLabels" (dict "matchLabels" $matchLabels "dot" $dot) | nindent 2 }} {{- end -}} {{/* Generate "template" metadata for Deployment / StatefulSet / ... + The function takes several arguments (inside a dictionary) + - .dot : environment (.) + - .labels: labels to add (dict) */}} {{- define "common.templateMetadata" -}} -{{- if .Values.podAnnotations }} -annotations: {{- include "common.tplValue" (dict "value" .Values.podAnnotations "context" $) | nindent 2 }} +{{- $dot := default . .dot -}} +{{- $labels := default (dict) .labels -}} +{{- if $dot.Values.podAnnotations }} +annotations: {{- include "common.tplValue" (dict "value" $dot.Values.podAnnotations "context" $) | nindent 2 }} {{- end }} -labels: {{- include "common.labels" . | nindent 2 }} -name: {{ include "common.name" . }} +labels: {{- include "common.labels" (dict "labels" $labels "dot" $dot) | nindent 2 }} +name: {{ include "common.name" $dot }} {{- end -}} diff --git a/kubernetes/common/common/templates/_name.tpl b/kubernetes/common/common/templates/_name.tpl index 943078ff2f..e918cc1dd8 100644 --- a/kubernetes/common/common/templates/_name.tpl +++ b/kubernetes/common/common/templates/_name.tpl @@ -16,9 +16,14 @@ {{/* Expand the name of a chart. + The function takes from one to two arguments (inside a dictionary): + - .dot : environment (.) + - .suffix : add a suffix to the name */}} {{- define "common.name" -}} - {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} + {{- $dot := default . .dot -}} + {{- $suffix := .suffix -}} + {{- default $dot.Chart.Name $dot.Values.nameOverride | trunc 63 | trimSuffix "-" -}}{{ if $suffix }}{{ print "-" $suffix }}{{ end }} {{- end -}} {{/* @@ -28,16 +33,25 @@ {{- define "common.fullnameExplicit" -}} {{- $dot := .dot }} {{- $name := .chartName }} - {{- printf "%s-%s" (include "common.release" $dot) $name | trunc 63 | trimSuffix "-" -}} + {{- $suffix := default "" .suffix -}} + {{- printf "%s-%s-%s" (include "common.release" $dot) $name $suffix | trunc 63 | trimSuffix "-" | trimSuffix "-" -}} {{- end -}} {{/* Create a default fully qualified application name. Truncated at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). + Usage: + include "common.fullname" . + include "common.fullname" (dict "suffix" "mySuffix" "dot" .) + The function takes from one to two arguments: + - .dot : environment (.) + - .suffix : add a suffix to the fullname */}} {{- define "common.fullname" -}} - {{- $name := default .Chart.Name .Values.nameOverride -}} - {{- include "common.fullnameExplicit" (dict "dot" . "chartName" $name) }} +{{- $dot := default . .dot -}} +{{- $suffix := default "" .suffix -}} + {{- $name := default $dot.Chart.Name $dot.Values.nameOverride -}} + {{- include "common.fullnameExplicit" (dict "dot" $dot "chartName" $name "suffix" $suffix) }} {{- end -}} {{/* diff --git a/kubernetes/common/common/templates/_service.tpl b/kubernetes/common/common/templates/_service.tpl index cd1595b0ca..8b430ef2bd 100644 --- a/kubernetes/common/common/templates/_service.tpl +++ b/kubernetes/common/common/templates/_service.tpl @@ -31,11 +31,12 @@ {{- end -}} {{/* Define the metadata of Service - The function takes from one to three arguments (inside a dictionary): + The function takes from one to four arguments (inside a dictionary): - .dot : environment (.) - .suffix : a string which will be added at the end of the name (with a '-'). - .annotations: the annotations to add - .msb_informations: msb information in order to create msb annotation + - .labels : labels to add Usage example: {{ include "common.serviceMetadata" ( dict "suffix" "myService" "dot" .) }} {{ include "common.serviceMetadata" ( dict "annotations" .Values.service.annotation "dot" .) }} @@ -45,6 +46,7 @@ {{- $suffix := default "" .suffix -}} {{- $annotations := default "" .annotations -}} {{- $msb_informations := default "" .msb_informations -}} + {{- $labels := default (dict) .labels -}} {{- if or $annotations $msb_informations -}} annotations: {{- if $annotations }} @@ -65,7 +67,7 @@ annotations: {{- end }} name: {{ include "common.servicename" $dot }}{{ if $suffix }}{{ print "-" $suffix }}{{ end }} namespace: {{ include "common.namespace" $dot }} -labels: {{- include "common.labels" $dot | nindent 2 -}} +labels: {{- include "common.labels" (dict "labels" $labels "dot" $dot) | nindent 2 -}} {{- end -}} {{/* Define the ports of Service @@ -125,6 +127,9 @@ labels: {{- include "common.labels" $dot | nindent 2 -}} - .publishNotReadyAddresses: if we publish not ready address - .headless: if the service is headless - .add_plain_port: add tls port AND plain port + - .labels : labels to add (dict) + - .matchLabels: selectors/machLabels to add (dict) + - .sessionAffinity: ClientIP - enables sticky sessions based on client IP, default: None */}} {{- define "common.genericService" -}} {{- $dot := default . .dot -}} @@ -136,9 +141,12 @@ labels: {{- include "common.labels" $dot | nindent 2 -}} {{- $ports := .ports -}} {{- $headless := default false .headless -}} {{- $add_plain_port := default false .add_plain_port }} +{{- $labels := default (dict) .labels -}} +{{- $matchLabels := default (dict) .matchLabels -}} +{{- $sessionAffinity := default "None" $dot.Values.service.sessionAffinity -}} apiVersion: v1 kind: Service -metadata: {{ include "common.serviceMetadata" (dict "suffix" $suffix "annotations" $annotations "msb_informations" $msb_informations "dot" $dot) | nindent 2 }} +metadata: {{ include "common.serviceMetadata" (dict "suffix" $suffix "annotations" $annotations "msb_informations" $msb_informations "labels" $labels "dot" $dot) | nindent 2 }} spec: {{- if $headless }} clusterIP: None @@ -148,7 +156,8 @@ spec: publishNotReadyAddresses: true {{- end }} type: {{ $serviceType }} - selector: {{- include "common.matchLabels" $dot | nindent 4 }} + selector: {{- include "common.matchLabels" (dict "matchLabels" $matchLabels "dot" $dot) | nindent 4 }} + sessionAffinity: {{ $sessionAffinity }} {{- end -}} {{/* @@ -166,15 +175,19 @@ spec: ports and the other one is NodePort (or LoadBalancer) with crypted port only. */}} {{- define "common.service" -}} -{{- $suffix := default "" .Values.service.suffix -}} -{{- $annotations := default "" .Values.service.annotations -}} -{{- $publishNotReadyAddresses := default false .Values.service.publishNotReadyAddresses -}} -{{- $msb_informations := default "" .Values.service.msb -}} -{{- $serviceType := .Values.service.type -}} -{{- $ports := .Values.service.ports -}} -{{- $both_tls_and_plain:= default false .Values.service.both_tls_and_plain }} +{{- $dot := default . .dot -}} +{{- $suffix := default "" $dot.Values.service.suffix -}} +{{- $annotations := default "" $dot.Values.service.annotations -}} +{{- $publishNotReadyAddresses := default false $dot.Values.service.publishNotReadyAddresses -}} +{{- $msb_informations := default "" $dot.Values.service.msb -}} +{{- $serviceType := $dot.Values.service.type -}} +{{- $ports := $dot.Values.service.ports -}} +{{- $both_tls_and_plain:= default false $dot.Values.service.both_tls_and_plain }} +{{- $labels := default (dict) .labels -}} +{{- $matchLabels := default (dict) .matchLabels -}} + {{- if (and (include "common.needTLS" .) $both_tls_and_plain) }} -{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "msb_informations" $msb_informations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" "ClusterIP" "add_plain_port" true) }} +{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "msb_informations" $msb_informations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" "ClusterIP" "add_plain_port" true $labels "matchLabels" $matchLabels) }} {{- if (ne $serviceType "ClusterIP") }} --- {{- if $suffix }} @@ -182,20 +195,23 @@ spec: {{- else }} {{- $suffix = "external" }} {{- end }} -{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" $serviceType) }} +{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" $serviceType $labels "matchLabels" $matchLabels) }} {{- end }} {{- else }} -{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" $serviceType) }} +{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" $serviceType $labels "matchLabels" $matchLabels) }} {{- end }} {{- end -}} {{/* Create headless service template */}} {{- define "common.headlessService" -}} -{{- $suffix := include "common._makeHeadlessSuffix" . -}} -{{- $annotations := default "" .Values.service.headless.annotations -}} -{{- $publishNotReadyAddresses := default false .Values.service.headless.publishNotReadyAddresses -}} -{{- $ports := .Values.service.headlessPorts -}} -{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "dot" . "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" "ClusterIP" "headless" true ) }} +{{- $dot := default . .dot -}} +{{- $suffix := include "common._makeHeadlessSuffix" $dot -}} +{{- $annotations := default "" $dot.Values.service.headless.annotations -}} +{{- $publishNotReadyAddresses := default false $dot.Values.service.headless.publishNotReadyAddresses -}} +{{- $ports := $dot.Values.service.headlessPorts -}} +{{- $labels := default (dict) .labels -}} +{{- $matchLabels := default (dict) .matchLabels -}} +{{ include "common.genericService" (dict "suffix" $suffix "annotations" $annotations "dot" $dot "publishNotReadyAddresses" $publishNotReadyAddresses "ports" $ports "serviceType" "ClusterIP" "headless" true "labels" $labels "matchLabels" $matchLabels) }} {{- end -}} {{/* -- 2.16.6