Add CSITs for Tracing 23/138323/4
authorsaul.gill <saul.gill@est.tech>
Tue, 25 Jun 2024 16:42:22 +0000 (17:42 +0100)
committersaul.gill <saul.gill@est.tech>
Wed, 26 Jun 2024 14:48:04 +0000 (15:48 +0100)
Added custom start scripts for acm and participants
Added config for tracing to acm and participants
Added CSITs for Jaeger tracing of acm and participants

Issue-ID: POLICY-5062
Change-Id: I58c4dcc85d119d5dc3a74c39b786491f080bafdc
Signed-off-by: saul.gill <saul.gill@est.tech>
18 files changed:
compose/config/clamp/A1pmsParticipantParameters.yaml
compose/config/clamp/AcRuntimeParameters.yaml
compose/config/clamp/HttpParticipantParameters.yaml
compose/config/clamp/KserveParticipantParameters.yaml
compose/config/clamp/KubernetesParticipantParameters.yaml
compose/config/clamp/PolicyParticipantParameters.yaml
compose/config/clamp/SimulatorParticipantParameters.yaml
compose/config/clamp/a1pms-participant.sh [new file with mode: 0644]
compose/config/clamp/acm-runtime.sh [new file with mode: 0755]
compose/config/clamp/http-participant.sh [new file with mode: 0755]
compose/config/clamp/kserve-participant.sh [new file with mode: 0644]
compose/config/clamp/kubernetes-participant.sh [new file with mode: 0755]
compose/config/clamp/policy-participant.sh [new file with mode: 0755]
compose/config/clamp/sim-participant.sh [new file with mode: 0755]
compose/docker-compose.yml
csit/resources/scripts/run-test.sh
csit/resources/tests/common-library.robot
csit/resources/tests/policy-clamp-test.robot

index 1b81a67..7139553 100755 (executable)
@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: a1pms-ppnt
   security:
     user:
       name: participantUser
@@ -36,6 +38,7 @@ participant:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
+          allowTracing: true
           additionalProps:
             group.id: policy-clamp-ac-a1pms-ppnt
 
@@ -45,14 +48,21 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
       topicSinks:
         - topic: ${participant.intermediaryParameters.topics.operationTopic}
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
+          allowTracing: true
 
 
 management:
+  tracing:
+    propagation:
+      produce: B3
+    sampling:
+      probability: 1.0
   endpoints:
     web:
       base-path: /
@@ -64,3 +74,12 @@ server:
     enabled: false
   servlet:
     context-path: /onap/policy/clamp/acm/a1pmsparticipant
+
+tracing:
+  enabled: true
+  exporter:
+    endpoint: http://jaeger:4318/v1/traces
+    protocol: http
+  sampler:
+    jaeger-remote:
+      endpoint: http://jaeger:14250
index abb5c48..dbcbc56 100644 (file)
@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: acm-r
   security:
     user:
       name: runtimeUser
@@ -56,12 +58,14 @@ runtime:
         useHttps: false
         additionalProps:
             group.id: policy-clamp-runtime-acm
+        allowTracing: true
     topicSinks:
       -
         topic: ${runtime.topics.operationTopic}
         servers:
           - ${topicServer:kafka:9092}
         topicCommInfrastructure: kafka
+        allowTracing: true
         useHttps: false
       -
         topic: ${runtime.topics.syncTopic}
@@ -69,11 +73,17 @@ runtime:
           - ${topicServer:kafka:9092}
         topicCommInfrastructure: kafka
         useHttps: false
+        allowTracing: true
   acmParameters:
     toscaElementName: org.onap.policy.clamp.acm.AutomationCompositionElement
     toscaCompositionName: org.onap.policy.clamp.acm.AutomationComposition
 
 management:
+  tracing:
+    propagation:
+      produce: B3
+    sampling:
+      probability: 1.0
   endpoints:
     web:
       base-path: /
@@ -83,3 +93,23 @@ management:
 metrics:
   security:
     disabled: false
+
+
+# Tracing can be configured for grpc or http - default is http
+#tracing:
+#  enabled: true
+#  exporter:
+#    endpoint: http://jaeger:4317
+#    protocol: grpc
+#  sampler:
+#    jaeger-remote:
+#      endpoint: http://jaeger:14250
+
+tracing:
+  enabled: true
+  exporter:
+    endpoint: http://jaeger:4318/v1/traces
+    protocol: http
+  sampler:
+    jaeger-remote:
+      endpoint: http://jaeger:14250
index 6761f61..62e309c 100644 (file)
@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: http-ppnt
   security:
     user:
       name: participantUser
@@ -26,6 +28,7 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
           additionalProps:
             group.id: policy-clamp-ac-http-ppnt
         - topic: ${participant.intermediaryParameters.topics.syncTopic}
@@ -34,16 +37,23 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
       topicSinks:
         - topic: ${participant.intermediaryParameters.topics.operationTopic}
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
           useHttps: false
+          allowTracing: true
     participantSupportedElementTypes:
       - typeName: org.onap.policy.clamp.acm.HttpAutomationCompositionElement
         typeVersion: 1.0.0
 management:
+  tracing:
+    propagation:
+      produce: B3
+    sampling:
+      probability: 1.0
   endpoints:
     web:
       exposure:
@@ -54,3 +64,12 @@ server:
     enabled: false
   servlet:
     context-path: /onap/policy/clamp/acm/httpparticipant
+
+tracing:
+  enabled: true
+  exporter:
+    endpoint: http://jaeger:4318/v1/traces
+    protocol: http
+  sampler:
+    jaeger-remote:
+      endpoint: http://jaeger:14250
index 1865d0e..efc2918 100755 (executable)
@@ -1,4 +1,6 @@
 spring:\r
+  application:\r
+    name: kserve-ppnt\r
   cloud:\r
     kubernetes:\r
       enabled: false\r
@@ -32,6 +34,7 @@ participant:
             - ${topicServer:kafka:9092}\r
           topicCommInfrastructure: kafka\r
           fetchTimeout: 15000\r
+          allowTracing: true\r
           additionalProps:\r
             group.id: policy-clamp-ac-kserve-ppnt\r
 \r
@@ -41,11 +44,13 @@ participant:
           topicCommInfrastructure: kafka\r
           fetchTimeout: 15000\r
           useHttps: false\r
+          allowTracing: true\r
       topicSinks:\r
         - topic: ${participant.intermediaryParameters.topics.operationTopic}\r
           servers:\r
             - ${topicServer:kafka:9092}\r
           topicCommInfrastructure: kafka\r
+          allowTracing: true\r
     participantSupportedElementTypes:\r
       -\r
         typeName: org.onap.policy.clamp.acm.KserveAutomationCompositionElement\r
@@ -58,6 +63,11 @@ customresourcedefinition:
   grace-period: 10\r
 \r
 management:\r
+  tracing:\r
+    propagation:\r
+      produce: B3\r
+    sampling:\r
+      probability: 1.0\r
   endpoints:\r
     web:\r
       base-path: /\r
@@ -69,3 +79,12 @@ server:
     context-path: /onap/policy/clamp/acm/kserveparticipant\r
   ssl:\r
     enabled: false\r
+\r
+tracing:\r
+  enabled: true\r
+  exporter:\r
+    endpoint: http://jaeger:4318/v1/traces\r
+    protocol: http\r
+  sampler:\r
+    jaeger-remote:\r
+      endpoint: http://jaeger:14250
\ No newline at end of file
index c85c7e5..76ae969 100644 (file)
@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: k8s-ppnt
   security:
     user:
       name: participantUser
@@ -30,6 +32,7 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
           additionalProps:
             group.id: policy-clamp-ac-k8s-ppnt
         -
@@ -37,15 +40,23 @@ participant:
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
+          additionalProps:
+            group.id: k8s-ppnt
+            security.protocol: PLAINTEXT
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
       topicSinks:
         -
           topic: ${participant.intermediaryParameters.topics.operationTopic}
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
+          additionalProps:
+            group.id: k8s-ppnt
+            security.protocol: PLAINTEXT
           useHttps: false
+          allowTracing: true
     participantSupportedElementTypes:
       - typeName: org.onap.policy.clamp.acm.K8SMicroserviceAutomationCompositionElement
         typeVersion: 1.0.0
index e510874..a65eeb0 100644 (file)
@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: policy-ppnt
   security:
     user:
       name: participantUser
@@ -43,6 +45,7 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
           additionalProps:
             group.id: policy-clamp-ac-pf-ppnt
         -
@@ -50,15 +53,23 @@ participant:
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
+          additionalProps:
+            group.id: policy-ppnt
+            security.protocol: PLAINTEXT
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
       topicSinks:
         -
           topic: ${participant.intermediaryParameters.topics.operationTopic}
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
+          additionalProps:
+            group.id: policy-ppnt
+            security.protocol: PLAINTEXT
           useHttps: false
+          allowTracing: true
     participantSupportedElementTypes:
       - typeName: org.onap.policy.clamp.acm.PolicyAutomationCompositionElement
         typeVersion: 1.0.0
index ae0b0f9..6f6a513 100644 (file)
@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: sim-ppnt
   security:
     user:
       name: participantUser
@@ -28,6 +30,7 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
           additionalProps:
             group.id: policy-clamp-ac-sim-ppnt
         - topic: ${participant.intermediaryParameters.topics.syncTopic}
@@ -36,18 +39,25 @@ participant:
           topicCommInfrastructure: kafka
           fetchTimeout: 15000
           useHttps: false
+          allowTracing: true
       topicSinks:
         - topic: ${participant.intermediaryParameters.topics.operationTopic}
           servers:
             - ${topicServer:kafka:9092}
           topicCommInfrastructure: kafka
           useHttps: false
+          allowTracing: true
     participantSupportedElementTypes:
       -
         typeName: ${supportedElementTypeName:org.onap.policy.clamp.acm.SimAutomationCompositionElement}
         typeVersion: ${supportedElementTypeVersion:1.0.0}
 
 management:
+  tracing:
+    propagation:
+      produce: B3
+    sampling:
+      probability: 1.0
   endpoints:
     web:
       base-path: /
@@ -59,3 +69,12 @@ server:
     enabled: false
   servlet:
     context-path: /onap/policy/simparticipant
+
+tracing:
+  enabled: true
+  exporter:
+    endpoint: http://jaeger:4318/v1/traces
+    protocol: http
+  sampler:
+    jaeger-remote:
+      endpoint: http://jaeger:14250
\ No newline at end of file
diff --git a/compose/config/clamp/a1pms-participant.sh b/compose/config/clamp/a1pms-participant.sh
new file mode 100644 (file)
index 0000000..88ff3b8
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/A1pmsParticipantParameters.yaml"
+fi
+
+echo "Policy clamp A1 Pms participant config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
diff --git a/compose/config/clamp/acm-runtime.sh b/compose/config/clamp/acm-runtime.sh
new file mode 100755 (executable)
index 0000000..695c08f
--- /dev/null
@@ -0,0 +1,66 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/AcRuntimeParameters.yaml"
+fi
+
+echo "Policy clamp runtime acm config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dcom.sun.management.jmxremote.rmi.port=9090 \
+    -Dcom.sun.management.jmxremote=true \
+    -Dcom.sun.management.jmxremote.port=9090 \
+    -Dcom.sun.management.jmxremote.ssl=false \
+    -Dcom.sun.management.jmxremote.authenticate=false \
+    -Dcom.sun.management.jmxremote.local.only=false \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
diff --git a/compose/config/clamp/http-participant.sh b/compose/config/clamp/http-participant.sh
new file mode 100755 (executable)
index 0000000..e35fb86
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/HttpParticipantParameters.yaml"
+fi
+
+echo "Policy clamp HTTP participant config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
diff --git a/compose/config/clamp/kserve-participant.sh b/compose/config/clamp/kserve-participant.sh
new file mode 100644 (file)
index 0000000..7da0b0f
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/KserveParticipantParameters.yaml"
+fi
+
+echo "Policy clamp Kserve participant config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
diff --git a/compose/config/clamp/kubernetes-participant.sh b/compose/config/clamp/kubernetes-participant.sh
new file mode 100755 (executable)
index 0000000..ce08684
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/KubernetesParticipantParameters.yaml"
+fi
+
+echo "Policy clamp Kubernetes participant config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
diff --git a/compose/config/clamp/policy-participant.sh b/compose/config/clamp/policy-participant.sh
new file mode 100755 (executable)
index 0000000..f36c2b3
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/PolicyParticipantParameters.yaml"
+fi
+
+echo "Policy clamp policy participant config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
diff --git a/compose/config/clamp/sim-participant.sh b/compose/config/clamp/sim-participant.sh
new file mode 100755 (executable)
index 0000000..556f4ef
--- /dev/null
@@ -0,0 +1,60 @@
+#!/usr/bin/env sh
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2024 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+
+KEYSTORE="${KEYSTORE:-$POLICY_HOME/etc/ssl/policy-keystore}"
+TRUSTSTORE="${TRUSTSTORE:-$POLICY_HOME/etc/ssl/policy-truststore}"
+KEYSTORE_PASSWD="${KEYSTORE_PASSWD:-Pol1cy_0nap}"
+TRUSTSTORE_PASSWD="${TRUSTSTORE_PASSWD:-Pol1cy_0nap}"
+
+if [ "$#" -eq 1 ]; then
+    CONFIG_FILE=$1
+fi
+
+if [ -z "$CONFIG_FILE" ]; then
+    CONFIG_FILE="${POLICY_HOME}/etc/SimulatorParticipantParameters.yaml"
+fi
+
+echo "Policy clamp Simulator participant config file: $CONFIG_FILE"
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-truststore" ]; then
+    echo "overriding policy-truststore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-truststore "${TRUSTSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/policy-keystore" ]; then
+    echo "overriding policy-keystore"
+    cp -f "${POLICY_HOME}"/etc/mounted/policy-keystore "${KEYSTORE}"
+fi
+
+if [ -f "${POLICY_HOME}/etc/mounted/logback.xml" ]; then
+    echo "overriding logback xml file"
+    cp -f "${POLICY_HOME}"/etc/mounted/logback.xml "${POLICY_HOME}"/etc/
+fi
+
+$JAVA_HOME/bin/java \
+    -Dlogging.config="${POLICY_HOME}/etc/logback.xml" \
+    -Dserver.ssl.keyStore="${KEYSTORE}" \
+    -Dserver.ssl.keyStorePassword="${KEYSTORE_PASSWD}" \
+    -Djavax.net.ssl.trustStore="${TRUSTSTORE}" \
+    -Djavax.net.ssl.trustStorePassword="${TRUSTSTORE_PASSWD}" \
+    -Dotel.java.global-autoconfigure.enabled=true \
+    -jar /app/app.jar \
+    --spring.config.location="${CONFIG_FILE}"
index 85631e4..d5a2cc7 100644 (file)
@@ -264,6 +264,7 @@ services:
     depends_on:
       - policy-db-migrator-clamp
       - kafka
+      - jaeger
       - policy-clamp-ac-http-ppnt
       - policy-clamp-ac-k8s-ppnt
       - policy-clamp-ac-pf-ppnt
@@ -271,9 +272,19 @@ services:
       - apex-pdp
     ports:
       - ${ACM_PORT}:6969
+    environment:
+      OTEL_SERVICE_NAME: acm-r
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
+      # Tracing can be configured for grpc or http - default is http
+      #OTEL_EXPORTER_OTLP_PROTOCOL: grpc
+      #OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: grpc
+      #OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4317
     volumes:
       - ./config/clamp/AcRuntimeParameters.yaml:/opt/app/policy/clamp/etc/AcRuntimeParameters.yaml
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
+      - ./config/clamp/acm-runtime.sh:/opt/app/policy/clamp/bin/acm-runtime.sh:rw
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
     command: [
@@ -292,10 +303,17 @@ services:
     hostname: policy-clamp-ac-http-ppnt
     depends_on:
       - kafka
+      - jaeger
     ports:
       - "30290:6969"
+    environment:
+      OTEL_SERVICE_NAME: http-ppnt
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
     volumes:
       - ./config/clamp/HttpParticipantParameters.yaml:/opt/app/policy/clamp/etc/HttpParticipantParameters.yaml
+      - ./config/clamp/http-participant.sh:/opt/app/policy/clamp/bin/http-participant.sh:rw
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
@@ -309,10 +327,17 @@ services:
     hostname: policy-clamp-ac-k8s-ppnt
     depends_on:
       - kafka
+      - jaeger
+    environment:
+      OTEL_SERVICE_NAME: k8s-ppnt
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
     ports:
       - "30295:6969"
     volumes:
       - ./config/clamp/KubernetesParticipantParameters.yaml:/opt/app/policy/clamp/etc/KubernetesParticipantParameters.yaml
+      - ./config/clamp/kubernetes-participant.sh:/opt/app/policy/clamp/bin/kubernetes-participant.sh:rw
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
@@ -327,10 +352,17 @@ services:
     depends_on:
       - kafka
       - api
+      - jaeger
     ports:
       - ${POLICY_PARTICIPANT_PORT}:6969
+    environment:
+      OTEL_SERVICE_NAME: policy-ppnt
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
     volumes:
       - ./config/clamp/PolicyParticipantParameters.yaml:/opt/app/policy/clamp/etc/PolicyParticipantParameters.yaml:ro
+      - ./config/clamp/policy-participant.sh:/opt/app/policy/clamp/bin/policy-participant.sh:rw
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
@@ -345,10 +377,17 @@ services:
     hostname: policy-clamp-ac-a1pms-ppnt
     depends_on:
       - kafka
+      - jaeger
+    environment:
+      OTEL_SERVICE_NAME: a1pms-ppnt
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
     ports:
       - "30296:6969"
     volumes:
       - ./config/clamp/A1pmsParticipantParameters.yaml:/opt/app/policy/clamp/etc/A1pmsParticipantParameters.yaml:ro
+      - ./config/clamp/a1pms-participant.sh:/opt/app/policy/clamp/bin/a1pms-participant.sh:rw
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
@@ -362,10 +401,17 @@ services:
     hostname: policy-clamp-ac-kserve-ppnt
     depends_on:
       - kafka
+      - jaeger
+    environment:
+      OTEL_SERVICE_NAME: kserve-ppnt
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
     ports:
       - "30297:6969"
     volumes:
       - ./config/clamp/KserveParticipantParameters.yaml:/opt/app/policy/clamp/etc/KserveParticipantParameters.yaml:ro
+      - ./config/clamp/kserve-participant.sh:/opt/app/policy/clamp/bin/kserve-participant.sh:rw
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
@@ -379,10 +425,17 @@ services:
     hostname: policy-clamp-ac-sim-ppnt
     depends_on:
       - kafka
+      - jaeger
+    environment:
+      OTEL_SERVICE_NAME: sim-ppnt
+      OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318
     ports:
       - ${SIM_PARTICIPANT_PORT}:6969
     volumes:
       - ./config/clamp/SimulatorParticipantParameters.yaml:/opt/app/policy/clamp/etc/SimulatorParticipantParameters.yaml:ro
+      - ./config/clamp/sim-participant.sh:/opt/app/policy/clamp/bin/sim-participant.sh:rw
       - ./config/clamp/logback.xml:/opt/app/policy/clamp/etc/logback.xml:ro
       - ./wait_for_port.sh:/opt/app/policy/bin/wait_for_port.sh:ro
     entrypoint: /opt/app/policy/bin/wait_for_port.sh
@@ -445,3 +498,14 @@ services:
     environment:
       ROBOT_FILE: ${ROBOT_FILES}
       TEST_ENV: ${TEST_ENV}
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    container_name: jaeger
+    ports:
+      - 4318:4318
+      - 4317:4317
+      - 14250:14250
+      - 16686:16686
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
index fb7d30b..1e756f6 100755 (executable)
@@ -41,6 +41,7 @@ DISTRIBUTION_IP=policy-distribution:${DEFAULT_PORT}
 
 POLICY_RUNTIME_ACM_IP=policy-clamp-runtime-acm:${DEFAULT_PORT}
 POLICY_PARTICIPANT_SIM_IP=policy-clamp-ac-sim-ppnt:${DEFAULT_PORT}
+JAEGER_IP=jaeger:16686
 
 KAFKA_IP=kafka:9092
 PROMETHEUS_IP=prometheus:9090
@@ -63,7 +64,8 @@ ROBOT_VARIABLES="-v DATA:${DATA}
 -v DROOLS_IP_2:${DROOLS_IP_2}
 -v TEMP_FOLDER:${DIST_TEMP_FOLDER}
 -v DISTRIBUTION_IP:${DISTRIBUTION_IP}
--v TEST_ENV:${TEST_ENV}"
+-v TEST_ENV:${TEST_ENV}
+-v JAEGER_IP:${JAEGER_IP}"
 
 export ROBOT_VARIABLES
 
index bee80a9..a585bc9 100644 (file)
@@ -115,6 +115,36 @@ GetMetrics
     Log  Received response from policy ${resp.text}
     RETURN  ${resp}
 
+VerifyTracingWorks
+    [Arguments]  ${domain}    ${service}
+    Log  Creating session http://${domain}
+    ${session}=  Create Session  jaeger  http://${domain}
+    ${resp}=  GET On Session  jaeger  /api/traces  params=service=${service}    expected_status=200
+    Log  Received response from jaeger ${resp.text}
+    RETURN  ${resp}
+
+VerifyKafkaInTraces
+    [Arguments]  ${domain}    ${service}
+    Log  Creating session http://${domain}
+    ${session}=  Create Session  jaeger  http://${domain}
+    ${tags}=    Create Dictionary    otel.library.name=io.opentelemetry.kafka-clients-2.6    messaging.system=kafka
+    ${tags_json}=    evaluate    json.dumps(${tags})    json
+    ${params}=    Create Dictionary    service=${service}    tags=${tags_json}    operation=policy-acruntime-participant send    lookback=1h    limit=10
+    ${resp}=  GET On Session  jaeger  /api/traces  params=${params}    expected_status=200
+    Log  Received response from jaeger ${resp.text}
+    RETURN  ${resp}
+
+VerifyHttpInTraces
+    [Arguments]  ${domain}    ${service}
+    Log  Creating session http://${domain}
+    ${session}=  Create Session  jaeger  http://${domain}
+    ${tags}=    Create Dictionary    uri=/v2/compositions/{compositionId}
+    ${tags_json}=    evaluate    json.dumps(${tags})    json
+    ${params}=    Create Dictionary    service=${service}    tags=${tags_json}    operation=http put /v2/compositions/{compositionId}    lookback=1h    limit=10
+    ${resp}=  GET On Session  jaeger  /api/traces  params=${params}    expected_status=200
+    Log  Received response from jaeger ${resp.text}
+    RETURN  ${resp}
+
 QueryPrometheus
     [Arguments]  ${query}
     ${params}=  Create Dictionary  query=${query}
index ca7cf7d..9d69520 100644 (file)
@@ -5,6 +5,7 @@ Library     OperatingSystem
 Library     String
 Library     json
 Library     yaml
+Resource    common-library.robot
 
 *** Test Cases ***
 HealthcheckAcm
@@ -159,6 +160,39 @@ DeployAutomationComposition
      Should Be Equal As Strings    ${resp.status_code}     202
      Wait Until Keyword Succeeds    10 min    5 sec    VerifyDeployStatus  ${compositionId}  ${instanceId}  DEPLOYED
 
+CheckTraces
+     [Documentation]    Verify that traces are being recorded in jaeger
+     Log    Verifying Jaeger traces
+     ${acmResp}=    VerifyTracingWorks    ${JAEGER_IP}    acm-r
+     ${httpResp}=    VerifyTracingWorks    ${JAEGER_IP}    http-ppnt
+     ${policyResp}=    VerifyTracingWorks    ${JAEGER_IP}    policy-ppnt
+     ${k8sResp}=    VerifyTracingWorks    ${JAEGER_IP}    k8s-ppnt
+     Should Not Be Empty    ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Log  Received spanID is ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${httpResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${policyResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${k8sResp.json()["data"][0]["spans"][0]["spanID"]}
+
+CheckKafkaPresentInTraces
+     [Documentation]    Verify that kafka traces are being recorded in jaeger
+     Log    Verifying Kafka Jaeger traces
+     ${acmResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    acm-r
+     ${httpResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    http-ppnt
+     ${policyResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    policy-ppnt
+     ${k8sResp}=    VerifyKafkaInTraces    ${JAEGER_IP}    k8s-ppnt
+     Should Not Be Empty    ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Log  Received spanID is ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${httpResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${policyResp.json()["data"][0]["spans"][0]["spanID"]}
+     Should Not Be Empty    ${k8sResp.json()["data"][0]["spans"][0]["spanID"]}
+
+CheckHttpPresentInAcmTraces
+     [Documentation]    Verify that http traces are being recorded in jaeger
+     Log    Verifying Http Jaeger traces
+     ${acmResp}=    VerifyHttpInTraces    ${JAEGER_IP}    acm-r
+     Should Not Be Empty    ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+     Log  Received spanID is ${acmResp.json()["data"][0]["spans"][0]["spanID"]}
+
 QueryPolicies
      [Documentation]    Verify the new policies deployed
      ${auth}=    Create List    policyadmin    zb!XztG34