From 46fb5809d4aa57c10c57ee6b828b72b9a80ccd9b Mon Sep 17 00:00:00 2001 From: leila Date: Tue, 15 Nov 2022 11:33:21 -0500 Subject: [PATCH] [AAI] Service monitor for scraping prometheus metrics Changes are done for aai resources graphadmin and traversal Session affinity for aai resources traversal and aai haproxy Issue-ID: AAI-3599 Signed-off-by: leila Change-Id: Ib3853c45051ca8e9c025bd0d6e700a08a54cb1a9 --- .../resources/config/application.properties | 5 ++ .../aai-graphadmin/templates/deployment.yaml | 3 ++ .../aai-graphadmin/templates/service.yaml | 5 ++ .../aai-graphadmin/templates/servicemonitor.yaml | 3 ++ .../aai/components/aai-graphadmin/values.yaml | 20 ++++++++ .../resources/config/application.properties | 14 +++++- .../aai-resources/templates/deployment.yaml | 3 ++ .../aai-resources/templates/service.yaml | 7 +++ .../aai-resources/templates/servicemonitor.yaml | 3 ++ .../aai/components/aai-resources/values.yaml | 55 ++++++++++++++++++++++ .../resources/config/application.properties | 16 +++++++ .../aai-traversal/templates/deployment.yaml | 3 ++ .../aai-traversal/templates/service.yaml | 7 +++ .../aai-traversal/templates/servicemonitor.yaml | 3 ++ .../aai/components/aai-traversal/values.yaml | 55 ++++++++++++++++++++++ kubernetes/aai/templates/service.yaml | 2 + kubernetes/aai/values.yaml | 2 + 17 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml create mode 100644 kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml create mode 100644 kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml diff --git a/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties b/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties index 27606021ef..6e64fd8400 100644 --- a/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties +++ b/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties @@ -122,3 +122,8 @@ aperture.service.ssl.key-store-password=password(${KEYSTORE_JKS_PASSWORD}) aperture.service.ssl.trust-store-password=password(${TRUSTSTORE_PASSWORD}) {{ end }} aperture.service.timeout-in-milliseconds=300000 + +#To Expose the Prometheus scraping endpoint +management.port=8448 +endpoints.enabled=false +management.security.enabled=false \ No newline at end of file diff --git a/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml b/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml index 5241c6edf2..3e1479e335 100644 --- a/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml +++ b/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml @@ -143,6 +143,8 @@ spec: value: {{ .Values.service.internalPort | quote }} - name: INTERNAL_PORT_2 value: {{ .Values.service.internalPort2 | quote }} + - name: INTERNAL_PORT_3 + value: {{ .Values.service.internalPort3 | quote }} volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }} - mountPath: /etc/localtime name: localtime @@ -173,6 +175,7 @@ spec: ports: - containerPort: {{ .Values.service.internalPort }} - containerPort: {{ .Values.service.internalPort2 }} + - containerPort: {{ .Values.service.internalPort3 }} lifecycle: # wait for active requests (long-running tasks) to be finished # Before the SIGTERM is invoked, Kubernetes exposes a preStop hook in the Pod. diff --git a/kubernetes/aai/components/aai-graphadmin/templates/service.yaml b/kubernetes/aai/components/aai-graphadmin/templates/service.yaml index cf4655361d..6350f858f1 100644 --- a/kubernetes/aai/components/aai-graphadmin/templates/service.yaml +++ b/kubernetes/aai/components/aai-graphadmin/templates/service.yaml @@ -40,11 +40,16 @@ spec: - port: {{ .Values.service.internalPort2 }} nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} name: {{ .Values.service.portName2 }} + - port: {{ .Values.service.internalPort3 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort3 }} + name: {{ .Values.service.portName3 }} {{- else -}} - port: {{ .Values.service.internalPort }} name: {{ .Values.service.portName }}{{ (eq "true" (include "common.needTLS" .)) | ternary "s" "" }} - port: {{ .Values.service.internalPort2 }} name: {{ .Values.service.portName2 }} + - port: {{ .Values.service.internalPort3 }} + name: {{ .Values.service.portName3 }} {{- end}} selector: app: {{ include "common.name" . }} diff --git a/kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml b/kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml new file mode 100644 index 0000000000..c0d9f212b4 --- /dev/null +++ b/kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml @@ -0,0 +1,3 @@ +{{- if .Values.metrics.serviceMonitor.enabled }} +{{ include "common.serviceMonitor" . }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/aai/components/aai-graphadmin/values.yaml b/kubernetes/aai/components/aai-graphadmin/values.yaml index deaf26a5ab..5a5daa6c92 100644 --- a/kubernetes/aai/components/aai-graphadmin/values.yaml +++ b/kubernetes/aai/components/aai-graphadmin/values.yaml @@ -240,6 +240,8 @@ service: internalPort: 8449 portName2: tcp-5005 internalPort2: 5005 + portName3: aai-graphadmin-8448 + internalPort3: 8448 terminationGracePeriodSeconds: 120 ingress: @@ -296,6 +298,24 @@ resources: memory: 2Gi unlimited: {} +metrics: + serviceMonitor: + enabled: false + targetPort: 8448 + path: /prometheus + basicAuth: + enabled: false + + selector: + app: '{{ include "common.name" . }}' + chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}' + release: '{{ include "common.release" . }}' + heritage: '{{ .Release.Service }}' + + relabelings: [] + + metricRelabelings: [] + # Not fully used for now securityContext: user_id: *user_id diff --git a/kubernetes/aai/components/aai-resources/resources/config/application.properties b/kubernetes/aai/components/aai-resources/resources/config/application.properties index 7258b7bef8..b5b64bec4b 100644 --- a/kubernetes/aai/components/aai-resources/resources/config/application.properties +++ b/kubernetes/aai/components/aai-resources/resources/config/application.properties @@ -106,5 +106,17 @@ schema.service.ssl.trust-store-password=${TRUSTSTORE_PASSWORD} {{ end }} #to expose the Prometheus scraping endpoint +management.port=8448 +management.endpoints.enabled-by-default=false +management.security.enabled=false +endpoints.enabled=false +endpoints.info.enabled=false +endpoints.prometheus.enabled=false +endpoints.health.enabled=false +management.metrics.web.server.auto-time-requests=false management.metrics.distribution.percentiles-histogram[http.server.requests]=true -management.metrics.distribution.sla[http.server.requests]=20ms, 30ms, 40ms, 50ms, 60ms, 70ms, 80ms, 90ms, 100ms, 500ms, 1000ms, 5000ms, 7000ms \ No newline at end of file +management.metrics.distribution.sla[http.server.requests]=20ms, 30ms, 40ms, 50ms, 60ms, 70ms, 80ms, 90ms, 100ms, 500ms, 1000ms, 5000ms, 7000ms +#Add common tag for grouping all aai related metrics +management.metrics.tags.group_id=aai +#It is not advisable to use labels to store dimensions with high cardinality. Enable this option only for debug purposes. For more information: https://github.com/micrometer-metrics/micrometer/issues/1584 +scrape.uri.metrics=false \ No newline at end of file diff --git a/kubernetes/aai/components/aai-resources/templates/deployment.yaml b/kubernetes/aai/components/aai-resources/templates/deployment.yaml index 74c58ff2d3..33aa97179e 100644 --- a/kubernetes/aai/components/aai-resources/templates/deployment.yaml +++ b/kubernetes/aai/components/aai-resources/templates/deployment.yaml @@ -136,6 +136,8 @@ spec: value: {{ .Values.service.internalPort | quote }} - name: INTERNAL_PORT_2 value: {{ .Values.service.internalPort2 | quote }} + - name: INTERNAL_PORT_3 + value: {{ .Values.service.internalPort3 | quote }} volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }} - mountPath: /etc/localtime name: localtime @@ -184,6 +186,7 @@ spec: ports: - containerPort: {{ .Values.service.internalPort }} - containerPort: {{ .Values.service.internalPort2 }} + - containerPort: {{ .Values.service.internalPort3 }} lifecycle: # wait for active requests (long-running tasks) to be finished # Before the SIGTERM is invoked, Kubernetes exposes a preStop hook in the Pod. diff --git a/kubernetes/aai/components/aai-resources/templates/service.yaml b/kubernetes/aai/components/aai-resources/templates/service.yaml index 46118461d1..8feb071bc6 100644 --- a/kubernetes/aai/components/aai-resources/templates/service.yaml +++ b/kubernetes/aai/components/aai-resources/templates/service.yaml @@ -34,12 +34,19 @@ spec: - port: {{ .Values.service.internalPort2 }} nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} name: {{ .Values.service.portName2 }} + - port: {{ .Values.service.internalPort3 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort3 }} + name: {{ .Values.service.portName3 }} {{- else -}} - port: {{ .Values.service.internalPort }} name: {{ .Values.service.portName }}{{ (eq "true" (include "common.needTLS" .)) | ternary "s" "" }} - port: {{ .Values.service.internalPort2 }} name: {{ .Values.service.portName2 }} + - port: {{ .Values.service.internalPort3 }} + name: {{ .Values.service.portName3 }} {{- end }} selector: app: {{ include "common.name" . }} release: {{ include "common.release" . }} + clusterIP: None + sessionAffinity: {{ .Values.service.sessionAffinity }} diff --git a/kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml b/kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml new file mode 100644 index 0000000000..c0d9f212b4 --- /dev/null +++ b/kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml @@ -0,0 +1,3 @@ +{{- if .Values.metrics.serviceMonitor.enabled }} +{{ include "common.serviceMonitor" . }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/aai/components/aai-resources/values.yaml b/kubernetes/aai/components/aai-resources/values.yaml index 2dfbfeebe5..de7bf2dd84 100644 --- a/kubernetes/aai/components/aai-resources/values.yaml +++ b/kubernetes/aai/components/aai-resources/values.yaml @@ -231,7 +231,10 @@ service: internalPort: 8447 portName2: tcp-5005 internalPort2: 5005 + portName3: aai-resources-8448 + internalPort3: 8448 terminationGracePeriodSeconds: 120 + sessionAffinity: None ingress: enabled: false @@ -270,6 +273,58 @@ resources: memory: 4Gi unlimited: {} +metrics: + serviceMonitor: + enabled: false + targetPort: 8448 + path: /prometheus + basicAuth: + enabled: false + externalSecretName: mysecretname + externalSecretUserKey: login + externalSecretPasswordKey: password + + ## Namespace in which Prometheus is running + ## + # namespace: monitoring + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + #interval: 30s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + selector: + app: '{{ include "common.name" . }}' + chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}' + release: '{{ include "common.release" . }}' + heritage: '{{ .Release.Service }}' + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## Value is evalued as a template + ## + relabelings: [] + + ## MetricRelabelConfigs to apply to samples before ingestion + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## Value is evalued as a template + ## + metricRelabelings: [] + # - sourceLabels: + # - "__name__" + # targetLabel: "__name__" + # action: replace + # regex: '(.*)' + # replacement: 'example_prefix_$1' + #Pods Service Account serviceAccount: nameOverride: aai-resources diff --git a/kubernetes/aai/components/aai-traversal/resources/config/application.properties b/kubernetes/aai/components/aai-traversal/resources/config/application.properties index 24a5241fe2..3022b17f97 100644 --- a/kubernetes/aai/components/aai-traversal/resources/config/application.properties +++ b/kubernetes/aai/components/aai-traversal/resources/config/application.properties @@ -106,3 +106,19 @@ schema.service.ssl.trust-store={{ .Values.certInitializer.credsPath }}/{{ .Value schema.service.ssl.key-store-password=${KEYSTORE_PASSWORD} schema.service.ssl.trust-store-password=${TRUSTSTORE_PASSWORD} {{ end }} + +#to expose the Prometheus scraping endpoint +management.port=8448 +management.endpoints.enabled-by-default=false +management.security.enabled=false +endpoints.enabled=false +endpoints.info.enabled=false +endpoints.prometheus.enabled=false +endpoints.health.enabled=false +management.metrics.web.server.auto-time-requests=false +management.metrics.distribution.percentiles-histogram[http.server.requests]=true +management.metrics.distribution.sla[http.server.requests]=20ms, 30ms, 40ms, 50ms, 60ms, 70ms, 80ms, 90ms, 100ms, 500ms, 1000ms, 5000ms, 7000ms +#Add common tag for grouping all aai related metrics +management.metrics.tags.group_id=aai +#It is not advisable to use labels to store dimensions with high cardinality. Enable this option only for debug purposes. For more information: https://github.com/micrometer-metrics/micrometer/issues/1584 +scrape.uri.metrics=false \ No newline at end of file diff --git a/kubernetes/aai/components/aai-traversal/templates/deployment.yaml b/kubernetes/aai/components/aai-traversal/templates/deployment.yaml index b9f2a802f2..093277169a 100644 --- a/kubernetes/aai/components/aai-traversal/templates/deployment.yaml +++ b/kubernetes/aai/components/aai-traversal/templates/deployment.yaml @@ -155,6 +155,8 @@ spec: value: {{ .Values.service.internalPort | quote }} - name: INTERNAL_PORT_2 value: {{ .Values.service.internalPort2 | quote }} + - name: INTERNAL_PORT_3 + value: {{ .Values.service.internalPort3 | quote }} volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }} - mountPath: /etc/localtime name: localtime @@ -205,6 +207,7 @@ spec: ports: - containerPort: {{ .Values.service.internalPort }} - containerPort: {{ .Values.service.internalPort2 }} + - containerPort: {{ .Values.service.internalPort3 }} lifecycle: # wait for active requests (long-running tasks) to be finished # Before the SIGTERM is invoked, Kubernetes exposes a preStop hook in the Pod. diff --git a/kubernetes/aai/components/aai-traversal/templates/service.yaml b/kubernetes/aai/components/aai-traversal/templates/service.yaml index b54b6be644..daf1758890 100644 --- a/kubernetes/aai/components/aai-traversal/templates/service.yaml +++ b/kubernetes/aai/components/aai-traversal/templates/service.yaml @@ -34,12 +34,19 @@ spec: - port: {{ .Values.service.internalPort2 }} nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} name: {{ .Values.service.portName2 }} + - port: {{ .Values.service.internalPort3 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort3 }} + name: {{ .Values.service.portName3 }} {{- else -}} - port: {{ .Values.service.internalPort }} name: {{ .Values.service.portName }}{{ (eq "true" (include "common.needTLS" .)) | ternary "s" "" }} - port: {{ .Values.service.internalPort2 }} name: {{ .Values.service.portName2 }} + - port: {{ .Values.service.internalPort3 }} + name: {{ .Values.service.portName3 }} {{- end }} selector: app: {{ include "common.name" . }} release: {{ include "common.release" . }} + clusterIP: None + sessionAffinity: {{ .Values.service.sessionAffinity }} diff --git a/kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml b/kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml new file mode 100644 index 0000000000..c0d9f212b4 --- /dev/null +++ b/kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml @@ -0,0 +1,3 @@ +{{- if .Values.metrics.serviceMonitor.enabled }} +{{ include "common.serviceMonitor" . }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/aai/components/aai-traversal/values.yaml b/kubernetes/aai/components/aai-traversal/values.yaml index 105b962c64..e532384ccc 100644 --- a/kubernetes/aai/components/aai-traversal/values.yaml +++ b/kubernetes/aai/components/aai-traversal/values.yaml @@ -253,7 +253,10 @@ service: internalPort: 8446 portName2: tcp-5005 internalPort2: 5005 + portName3: aai-traversal-8448 + internalPort3: 8448 terminationGracePeriodSeconds: 120 + sessionAffinity: None ingress: enabled: false @@ -289,6 +292,58 @@ resources: memory: 4Gi unlimited: {} +metrics: + serviceMonitor: + enabled: false + targetPort: 8448 + path: /prometheus + basicAuth: + enabled: false + externalSecretName: mysecretname + externalSecretUserKey: login + externalSecretPasswordKey: password + + ## Namespace in which Prometheus is running + ## + # namespace: monitoring + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + #interval: 30s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + selector: + app: '{{ include "common.name" . }}' + chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}' + release: '{{ include "common.release" . }}' + heritage: '{{ .Release.Service }}' + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## Value is evalued as a template + ## + relabelings: [] + + ## MetricRelabelConfigs to apply to samples before ingestion + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## Value is evalued as a template + ## + metricRelabelings: [] + # - sourceLabels: + # - "__name__" + # targetLabel: "__name__" + # action: replace + # regex: '(.*)' + # replacement: 'example_prefix_$1' + #Pods Service Account serviceAccount: nameOverride: aai-traversal diff --git a/kubernetes/aai/templates/service.yaml b/kubernetes/aai/templates/service.yaml index 4a6dc8e497..c8a756179b 100644 --- a/kubernetes/aai/templates/service.yaml +++ b/kubernetes/aai/templates/service.yaml @@ -35,6 +35,8 @@ spec: type: {{ .Values.service.type }} selector: app: {{ include "common.name" . }} + clusterIP: {{ .Values.service.aaiServiceClusterIp }} + sessionAffinity: {{ .Values.service.sessionAffinity }} --- apiVersion: v1 kind: Service diff --git a/kubernetes/aai/values.yaml b/kubernetes/aai/values.yaml index 247c58be2b..f4061be4c0 100644 --- a/kubernetes/aai/values.yaml +++ b/kubernetes/aai/values.yaml @@ -392,6 +392,8 @@ service: externalPlainPort: 80 internalPlainPort: 8080 nodeport: 33 + aaiServiceClusterIp: + sessionAffinity: None ingress: enabled: false -- 2.16.6