Use health from Spring Boot Actuator for healtchecks 02/142302/2
authoradheli.tavares <adheli.tavares@est.tech>
Tue, 21 Oct 2025 12:22:17 +0000 (13:22 +0100)
committerAdheli Tavares <adheli.tavares@est.tech>
Wed, 22 Oct 2025 10:31:39 +0000 (10:31 +0000)
Issue-ID: POLICY-5444
Change-Id: I07f15d582f21bf8492a80e11f2b559eb1106ebe2
Signed-off-by: adheli.tavares <adheli.tavares@est.tech>
14 files changed:
compose/compose.acm.scale.yml
compose/compose.yaml
compose/config/clamp/A1pmsParticipantParameters.yaml
compose/config/clamp/AcRuntimeParameters.yaml
compose/config/clamp/HttpParticipantParameters.yaml
compose/config/clamp/KserveParticipantParameters.yaml
compose/config/clamp/KubernetesParticipantParameters.yaml
compose/config/clamp/PolicyParticipantParameters.yaml
compose/config/clamp/SimulatorParticipantParameters.yaml
csit/resources/scripts/cluster_setup.sh
csit/resources/scripts/robot_setup.sh
csit/run-k8s-csit.sh
csit/run-project-csit.sh
helm/policy/components/policy-clamp-runtime-acm/resources/config/acRuntimeParameters.yaml

index 5019335..7cea86f 100644 (file)
@@ -37,6 +37,8 @@ services:
     deploy:
       mode: replicated
       replicas: ${REPLICAS:-2}
+    restart:
+      on-failure:3
     environment:
       OTEL_SERVICE_NAME: acm-r
       OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
@@ -59,16 +61,14 @@ services:
     volumes:
       - ./config/clamp/AcRuntimeParameters.yaml:/opt/app/policy/clamp/etc/AcRuntimeParameters.yaml
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'apex-pdp', '6969',
-      'kafka', '9092',
-      'policy-clamp-ac-pf-ppnt', '6969',
-      'policy-clamp-ac-sim-ppnt-1', '6969',
-      'policy-clamp-ac-sim-ppnt-2', '6969',
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+    entrypoint: java -jar /app/app.jar
+
   policy-clamp-ac-http-ppnt-replica:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-http-ppnt:${POLICY_CLAMP_PPNT_VERSION}
     hostname: policy-clamp-ac-http-ppnt
@@ -93,12 +93,14 @@ services:
     volumes:
       - ./config/clamp/HttpParticipantParameters.yaml:/opt/app/policy/clamp/etc/HttpParticipantParameters.yaml
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/httpparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+    entrypoint: java -jar /app/app.jar
+
   policy-clamp-ac-k8s-ppnt-replica:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-k8s-ppnt:${POLICY_CLAMP_PPNT_VERSION}
     hostname: policy-clamp-ac-k8s-ppnt
@@ -123,12 +125,14 @@ services:
     volumes:
       - ./config/clamp/KubernetesParticipantParameters.yaml:/opt/app/policy/clamp/etc/KubernetesParticipantParameters.yaml
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/k8sparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 20s
+    entrypoint: java -jar /app/app.jar
+
   policy-clamp-ac-pf-ppnt-replica:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-pf-ppnt:${POLICY_CLAMP_PPNT_VERSION}
     hostname: policy-clamp-ac-pf-ppnt
@@ -153,12 +157,14 @@ services:
     volumes:
       - ./config/clamp/PolicyParticipantParameters.yaml:/opt/app/policy/clamp/etc/PolicyParticipantParameters.yaml:ro
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/policyparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    entrypoint: java -jar /app/app.jar
+
   nginx:
     image: ${CONTAINER_LOCATION}library/nginx:latest
     volumes:
index 0e6227e..f97be8f 100644 (file)
@@ -223,6 +223,8 @@ services:
       - policy-clamp-ac-sim-ppnt-2
     ports:
       - ${ACM_PORT}:6969
+    restart:
+      on-failure:3
     environment:
       OTEL_SERVICE_NAME: acm-r
       OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
@@ -246,16 +248,13 @@ services:
     volumes:
       - ./config/clamp/AcRuntimeParameters.yaml:/opt/app/policy/clamp/etc/application.yaml
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'apex-pdp', '6969',
-      'kafka', '9092',
-      'policy-clamp-ac-pf-ppnt', '6969',
-      'policy-clamp-ac-sim-ppnt-1', '6969',
-      'policy-clamp-ac-sim-ppnt-2', '6969'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+    entrypoint: java -jar /app/app.jar
 
   policy-clamp-ac-http-ppnt:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-http-ppnt:${POLICY_CLAMP_PPNT_VERSION}
@@ -279,12 +278,13 @@ services:
     volumes:
       - ./config/clamp/HttpParticipantParameters.yaml:/opt/app/policy/clamp/etc/HttpParticipantParameters.yaml
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/httpparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+    entrypoint: java -jar /app/app.jar
 
   policy-clamp-ac-k8s-ppnt:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-k8s-ppnt:${POLICY_CLAMP_PPNT_VERSION}
@@ -308,12 +308,13 @@ services:
     volumes:
       - ./config/clamp/KubernetesParticipantParameters.yaml:/opt/app/policy/clamp/etc/KubernetesParticipantParameters.yaml
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/k8sparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 20s
+    entrypoint: java -jar /app/app.jar
 
   policy-clamp-ac-pf-ppnt:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-pf-ppnt:${POLICY_CLAMP_PPNT_VERSION}
@@ -338,13 +339,13 @@ services:
     volumes:
       - ./config/clamp/PolicyParticipantParameters.yaml:/opt/app/policy/clamp/etc/PolicyParticipantParameters.yaml:ro
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092',
-      'api', '6969'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/policyparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    entrypoint: java -jar /app/app.jar
 
   policy-clamp-ac-a1pms-ppnt:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-a1pms-ppnt:${POLICY_CLAMP_PPNT_VERSION}
@@ -367,12 +368,13 @@ services:
     volumes:
       - ./config/clamp/A1pmsParticipantParameters.yaml:/opt/app/policy/clamp/etc/A1pmsParticipantParameters.yaml:ro
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/a1pmsparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 20s
+    entrypoint: java -jar /app/app.jar
 
   policy-clamp-ac-kserve-ppnt:
     image: ${CONTAINER_LOCATION}onap/policy-clamp-ac-kserve-ppnt:${POLICY_CLAMP_PPNT_VERSION}
@@ -395,9 +397,10 @@ services:
     volumes:
       - ./config/clamp/KserveParticipantParameters.yaml:/opt/app/policy/clamp/etc/KserveParticipantParameters.yaml:ro
       - ./config/clamp/logback-ppnt.xml:/opt/app/policy/clamp/etc/logback.xml:ro
-      - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
-    entrypoint: /opt/app/policy/bin/wait_for_port.sh
-    command: [
-      '-c', 'java -jar /app/app.jar',
-      'kafka', '9092'
-    ]
+    healthcheck:
+      test: "curl --fail --silent http://localhost:6969/onap/policy/clamp/acm/kserveparticipant/health/readiness | grep UP || exit 1"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 20s
+    entrypoint: java -jar /app/app.jar
index 7139553..bdf77df 100755 (executable)
@@ -68,6 +68,14 @@ management:
       base-path: /
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
+
 server:
   port: 6969
   ssl:
index 5387aaf..5a2844b 100644 (file)
@@ -109,14 +109,19 @@ management:
       base-path: /
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
 
 metrics:
   security:
     disabled: false
 
-
 # Tracing can be configured for grpc or http - default is http
-
 tracing:
   enabled: true
   exporter:
index 9f9e2f0..1a48b27 100644 (file)
@@ -58,6 +58,14 @@ management:
     web:
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
+
 server:
   port: 6969
   ssl:
index efc2918..3c23565 100755 (executable)
@@ -73,6 +73,14 @@ management:
       base-path: /\r
       exposure:\r
         include: health, metrics, prometheus\r
+  endpoint:\r
+    health:\r
+      access: unrestricted\r
+      show-details: always\r
+      probes:\r
+        enabled: true\r
+      show-components: always\r
+\r
 server:\r
   port: 6969\r
   servlet:\r
index ab96f51..6204ea7 100644 (file)
@@ -66,6 +66,14 @@ management:
     web:
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
+
 server:
   port: 6969
   ssl:
index c7af9dd..bcc76f7 100644 (file)
@@ -78,6 +78,14 @@ management:
     web:
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
+
 server:
   port: 6969
   ssl:
index c0d05ed..86e0431 100644 (file)
@@ -63,6 +63,14 @@ management:
       base-path: /
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
+
 server:
   port: 6969
   ssl:
index 6d72bac..21b3660 100755 (executable)
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-#  Copyright (C) 2025 Nordix Foundation. All rights reserved.
+#  Copyright (C) 2025 OpenInfra Foundation Europe.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -115,11 +115,12 @@ function uninstall_policy() {
     helm uninstall csit-robot
     kubectl delete deploy $ZK_CONTAINER $KAFKA_CONTAINER
     rm -rf ${WORKSPACE}/helm/policy/Chart.lock
+
     if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
       helm uninstall policy-chartmuseum
       helm repo remove chartmuseum-git policy-chartmuseum
     fi
-    sudo rm -rf /dockerdata-nfs/mariadb-galera/
+
     kubectl delete pvc --all
     echo "Policy deployment deleted"
     echo "Clean up docker"
@@ -190,7 +191,7 @@ function wait_for_pods_running() {
       IFS=' ' read -r -a pod_array <<< "$pod_names"
       if [ "${#pod_array[@]}" -eq 0 ]; then
              echo "*** Error: No pods found for the deployment $pod_name_prefix . Exiting ***"
-             return -1
+             return 1
       fi
       for pod in "${pod_array[@]}"; do
          local pod_status
index f5ef2f3..9b1332f 100755 (executable)
@@ -1,6 +1,6 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
-#  Copyright (C) 2025 Nordix Foundation. All rights reserved.
+#  Copyright (C) 2025 OpenInfra Foundation Europe.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +36,8 @@ DIST_TEMP_FOLDER=/tmp/distribution
 function clone_models() {
     local retry_count=3
     local success=false
-    cd tests
+    cd tests || exit
+    sudo rm -rf models/
     for ((i = 1; i <= retry_count; i++)); do
         git clone "https://gerrit.onap.org/r/policy/models" && success=true && break
         echo "Retrying git clone ($i/$retry_count)..."
@@ -61,17 +62,17 @@ function clone_models() {
 }
 
 function copy_csar_file() {
-    zip -F ${DISTRIBUTION_CSAR}/sample_csar_with_apex_policy.csar \
-        --out ${DISTRIBUTION_CSAR}/csar_temp.csar -q
+    zip -F "${DISTRIBUTION_CSAR}"/sample_csar_with_apex_policy.csar \
+        --out "${DISTRIBUTION_CSAR}"/csar_temp.csar -q
     sudo rm -rf "${DIST_TEMP_FOLDER}"
     sudo mkdir "${DIST_TEMP_FOLDER}"
-    sudo cp ${DISTRIBUTION_CSAR}/csar_temp.csar ${DISTRIBUTION_CSAR}/temp.csar
-    sudo mv ${DISTRIBUTION_CSAR}/temp.csar ${DIST_TEMP_FOLDER}/sample_csar_with_apex_policy.csar
+    sudo cp "${DISTRIBUTION_CSAR}"/csar_temp.csar "${DISTRIBUTION_CSAR}"/temp.csar
+    sudo mv "${DISTRIBUTION_CSAR}"/temp.csar ${DIST_TEMP_FOLDER}/sample_csar_with_apex_policy.csar
 }
 
 function build_robot_image() {
     echo "Build docker image for robot framework"
-    cd ${WORKSPACE}/csit/resources || exit
+    cd "${WORKSPACE}"/csit/resources || exit
     clone_models
     if [ "${PROJECT}" == "distribution" ] || [ "${PROJECT}" == "policy-distribution" ]; then
         copy_csar_file
@@ -100,9 +101,9 @@ function push_acelement_chart() {
     done
 
     ACELEMENT_CHART=${WORKSPACE}/csit/resources/tests/clamp/examples/src/main/resources/clamp/acm/acelement-helm/acelement
-    helm cm-push $ACELEMENT_CHART policy-chartmuseum
+    helm cm-push "$ACELEMENT_CHART" policy-chartmuseum
     helm repo update
-    rm -rf ${WORKSPACE}/csit/resources/tests/clamp/
+    rm -rf "${WORKSPACE}"/csit/resources/tests/clamp/
     echo "-------------------------------------------"
 }
 
@@ -130,8 +131,8 @@ function start_csit() {
         echo "Importing robot image into microk8s registry"
         docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest
         sudo microk8s ctr image import policy-csit-robot.tar
-        rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar
-        rm -rf ${WORKSPACE}/csit/resources/tests/models/
+        rm -rf "${WORKSPACE}"/csit/resources/policy-csit-robot.tar
+        rm -rf "${WORKSPACE}"/csit/resources/tests/models/
         echo "---------------------------------------------"
         if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then
           POD_READY_STATUS="0/1"
@@ -143,9 +144,9 @@ function start_csit() {
           push_acelement_chart
         fi
         echo "Installing Robot framework pod for running CSIT"
-        cd ${WORKSPACE}/helm || exit
-        mkdir -p ${ROBOT_LOG_DIR}
-        helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={$(echo ${READINESS_CONTAINERS} | sed 's/[{}]//g' | sed 's/,$//')}" --set robotLogDir=$ROBOT_LOG_DIR
+        cd "${WORKSPACE}"/helm || exit
+        mkdir -p "${ROBOT_LOG_DIR}"
+        helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={$(echo "${READINESS_CONTAINERS}" | sed 's/[{}]//g' | sed 's/,$//')}" --set robotLogDir=$ROBOT_LOG_DIR
         print_robot_log
     fi
 }
index 2d5dfed..7eb0f58 100755 (executable)
@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # ============LICENSE_START====================================================
-#  Copyright (C) 2022-2025 Nordix Foundation.
+#  Copyright (C) 2022-2025 OpenInfra Foundation Europe.
 #  Modifications Copyright © 2024 Deutsche Telekom
 # =============================================================================
 #
@@ -86,16 +86,16 @@ fi
 # Execute the appropriate script based on the operation
 case $OPERATION in
     install)
-        ${WORKSPACE}/csit/resources/scripts/cluster_setup.sh install $PROJECT $LOCAL_IMAGE_ARG
+        "${WORKSPACE}"/csit/resources/scripts/cluster_setup.sh install "$PROJECT" $LOCAL_IMAGE_ARG
         if [ "$CLUSTER_ONLY" = false ]; then
-            ${WORKSPACE}/csit/resources/scripts/robot_setup.sh $PROJECT
+            "${WORKSPACE}"/csit/resources/scripts/robot_setup.sh "$PROJECT"
         fi
         ;;
     uninstall)
-        ${WORKSPACE}/csit/resources/scripts/cluster_setup.sh uninstall
+        "${WORKSPACE}"/csit/resources/scripts/cluster_setup.sh uninstall
         ;;
     clean)
-        ${WORKSPACE}/csit/resources/scripts/cluster_setup.sh clean
+        "${WORKSPACE}"/csit/resources/scripts/cluster_setup.sh clean
         ;;
     *)
         echo "Error: Invalid operation"
index 48904a0..2435ff2 100755 (executable)
@@ -357,10 +357,6 @@ cd "${WORKSPACE}" || exit
 sudo rm -rf "${ROBOT_LOG_DIR}"
 mkdir -p "${ROBOT_LOG_DIR}"
 
-if [ "${TEARDOWN}" == "true" ]; then
-    on_exit
-fi
-
 # log into nexus docker
 docker login -u docker -p docker nexus3.onap.org:10001
 
@@ -376,6 +372,10 @@ else
     sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
 fi
 
+if [ "${TEARDOWN}" == "true" ]; then
+    on_exit
+fi
+
 set_project_config
 
 unset http_proxy https_proxy
index bae68b0..841eb41 100644 (file)
@@ -102,8 +102,6 @@ runtime:
     toscaCompositionName: {{ .Values.customNaming.toscaCompositionName }}
     enableEncryption: true
 
-
-
 management:
   tracing:
     propagation:
@@ -115,6 +113,13 @@ management:
       base-path: /
       exposure:
         include: health, metrics, prometheus
+  endpoint:
+    health:
+      access: unrestricted
+      show-details: always
+      probes:
+        enabled: true
+      show-components: always
 
 tracing:
   enabled: {{ .Values.jaeger.enabled }}