Add SLAs validations to docker csit robot tests

Issue-ID: POLICY-4529
Change-Id: I197a51892e9135d20c2e8a49ce7787df3bfbd4fa
Signed-off-by: adheli.tavares <adheli.tavares@est.tech>

diff --git a/.gitignore b/.gitignore
index d4aa5c2..fee07ae 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ env.properties
 .pydevproject
 models
 .vscode/
+**/*.log

diff --git a/csit/common-library.robot b/csit/common-library.robot
index e8a9640..871e7ef 100644 (file)
--- a/csit/common-library.robot
+++ b/csit/common-library.robot
@@ -120,7 +120,18 @@ GetMetrics
     Log  Received response from policy ${resp.text}
     [return]  ${resp}
 
-QueryPrometheus ${query}
-    ${resp}=  GET http://localhost:30259/api/v1/query?query=${query}  expected_status=200
-    Log  Received response from policy ${resp.text}
-    [return]  ${resp}
+QueryPrometheus
+    [Arguments]  ${query}
+    ${params}=  Create Dictionary  query=${query}
+    ${resp}=  GET  http://localhost:30259/api/v1/query  ${params}
+    Status Should Be    OK
+    Log  Received response from Prometheus ${resp.text}
+    [return]  ${resp.json()}
+
+ValidateResponseTime
+    [Arguments]  ${job}  ${uri}  ${method}  ${timeLimit}
+    [Documentation]  Check if uri response is under the required time
+    ${resp}=  QueryPrometheus  http_server_requests_seconds_sum{uri="${uri}",method="${method}",job="${job}"}/http_server_requests_seconds_count{uri="${uri}",method="${method}",job="${job}"}
+    ${rawNumber}=  Evaluate  ${resp['data']['result'][0]['value'][1]}
+    ${actualTime}=   Set Variable  ${rawNumber * ${1000}}
+    Should Be True   ${actualTime} <= ${timeLimit}

diff --git a/csit/pap/plans/setup.sh b/csit/pap/plans/setup.sh
index b656ef5..4c77acc 100755 (executable)
--- a/csit/pap/plans/setup.sh
+++ b/csit/pap/plans/setup.sh
@@ -37,15 +37,8 @@ docker-compose -f "${SCRIPTS}"/docker-compose-all.yml up -d pap apex-pdp grafana
 sleep 10
 unset http_proxy https_proxy
 
-POLICY_PAP_IP=$(get-instance-ip.sh policy-pap)
 POLICY_PAP_PORT=30442
-POLICY_API_IP=$(get-instance-ip.sh policy-api)
 POLICY_API_PORT=30440
-MARIADB_IP=$(get-instance-ip.sh mariadb)
-
-echo PAP IP IS "${POLICY_PAP_IP}"
-echo API IP IS "${POLICY_API_IP}"
-echo MARIADB IP IS "${MARIADB_IP}"
 
 # wait for the app to start up
 "${SCRIPTS}"/wait_for_rest.sh localhost "${POLICY_PAP_PORT}"
@@ -55,8 +48,6 @@ DATA=${WORKSPACE}/models/models-examples/src/main/resources/policies
 NODETEMPLATES=${WORKSPACE}/models/models-examples/src/main/resources/nodetemplates
 
 ROBOT_VARIABLES=""
-ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_PAP_IP:${POLICY_PAP_IP}"
-ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_API_IP:${POLICY_API_IP}"
 ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_PAP_PORT:${POLICY_PAP_PORT}"
 ROBOT_VARIABLES="${ROBOT_VARIABLES} -v POLICY_API_PORT:${POLICY_API_PORT}"
 ROBOT_VARIABLES="${ROBOT_VARIABLES} -v DATA:${DATA}"

diff --git a/csit/pap/plans/testplan.txt b/csit/pap/plans/testplan.txt
index 1e5b1f9..4fac102 100644 (file)
--- a/csit/pap/plans/testplan.txt
+++ b/csit/pap/plans/testplan.txt
@@ -1,3 +1,4 @@
 # Test suites are relative paths under [policy/docker.git]/csit/[project]/tests.
 # Place the suites in run order.
 pap-test.robot
+pap-slas.robot

diff --git a/csit/pap/tests/pap-slas.robot b/csit/pap/tests/pap-slas.robot
new file mode 100644 (file)
index 0000000..045530a
--- /dev/null
+++ b/csit/pap/tests/pap-slas.robot
@@ -0,0 +1,60 @@
+*** Settings ***
+Library    Collections
+Library    RequestsLibrary
+Library    OperatingSystem
+Library    json
+Resource    ${CURDIR}/../../common-library.robot
+
+*** Keywords ***
+ValidateResponseTimeForPap
+    [Arguments]  ${uri}  ${method}
+    [Documentation]  Check if uri response is under the 500ms required time for pap metrics
+    ValidateResponseTime  pap-metrics  ${uri}  ${method}  500
+
+*** Test Cases ***
+WaitForPrometheusServer
+    [Documentation]  Sleep time to wait for Prometheus server to gather all metrics
+    Sleep    1 minute
+
+ValidateResponseTimeForHealthcheck
+    [Documentation]  Validate component healthcheck response time
+    ValidateResponseTimeForPap  /healthcheck  GET
+
+ValidateResponseTimeForSystemHealthcheck
+    [Documentation]  Validate if system healthcheck response time is under 1000ms
+    ValidateResponseTime  pap-metrics  /components/healthcheck  GET  10000
+
+ValidateResponseTimeForStatistics
+    [Documentation]  Validate statistics response time
+    ValidateResponseTimeForPap  /statistics  GET
+
+# TODO: includes notification, so always over 500ms
+# ValidateResponseTimeCreateGroup
+#     [Documentation]  Validate create group response time
+#     ValidateResponseTimeForPap  /pdps/groups/batch  POST
+
+ValidateResponseTimeQueryPolicyAudit
+    [Documentation]  Validate query audits response time
+    ValidateResponseTimeForPap  /policies/audit  GET
+
+ValidateResponseTimeUpdateGroup
+    [Documentation]  Validate pdps/group response time
+    ValidateResponseTimeForPap  /pdps/groups/{name}  PUT
+
+ValidatePolicyDeploymentTime
+    [Documentation]  Check if deployment of policy is under 2000ms
+    ${resp}=  QueryPrometheus  pap_policy_deployments_seconds_sum{operation="deploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="deploy",status="SUCCESS"}
+    ${rawNumber}=  Evaluate  ${resp['data']['result'][0]['value'][1]}
+    ${actualTime}=   Set Variable  ${rawNumber * ${1000}}
+    Should Be True   ${actualTime} <= ${2000}
+
+ValidateResponseTimeDeletePolicy
+    [Documentation]  Check if undeployment of policy is under 2000ms
+    ${resp}=  QueryPrometheus  pap_policy_deployments_seconds_sum{operation="undeploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="undeploy",status="SUCCESS"}
+    ${rawNumber}=  Evaluate  ${resp['data']['result'][0]['value'][1]}
+    ${actualTime}=   Set Variable  ${rawNumber * ${1000}}
+    Should Be True   ${actualTime} <= ${2000}
+
+ValidateResponseTimeDeleteGroup
+    [Documentation]  Validate delete group response time
+    ValidateResponseTimeForPap  /pdps/groups/{name}  DELETE

diff --git a/csit/pap/tests/pap-test.robot b/csit/pap/tests/pap-test.robot
index c417a83..a4f0853 100644 (file)
--- a/csit/pap/tests/pap-test.robot
+++ b/csit/pap/tests/pap-test.robot
@@ -12,6 +12,18 @@ GetReq
     ${resp}=  PerformGetRequest  ${POLICY_PAP_PORT}  ${url}  200  null  ${auth}
     [return]  ${resp}
 
+ValidateResponseTimeForPap
+    [Arguments]  ${uri}  ${method}
+    [Documentation]  Check if uri response is under the required time for pap metrics
+    ValidateResponseTime  pap-metrics  ${uri}  ${method}  500
+
+ValidateDeploymentTime
+    [Documentation]  Check if deployment of policy is under 2000ms
+    ${resp}=  QueryPrometheus  pap_policy_deployments_seconds_sum{operation="deploy",status="SUCCESS"}/pap_policy_deployments_seconds_count{operation="deploy",status="SUCCESS"}
+    ${rawNumber}=  Evaluate  ${resp['data']['result'][0]['value'][1]}
+    ${actualTime}=   Set Variable  ${rawNumber * ${1000}}
+    Should Be True   ${actualTime} <= ${2000}
+
 *** Test Cases ***
 LoadPolicy
     [Documentation]  Create a policy named 'onap.restart.tca' and version '1.0.0' using specific api
@@ -126,3 +138,16 @@ DeletePdpGroups
 QueryPdpGroupsAfterDelete
     [Documentation]    Verify PdpGroups after delete
     QueryPdpGroups  1  defaultGroup  ACTIVE  0  null  null  null
+
+# ValidateSlaForPap
+#     [Documentation]  Run checks against Prometheus server to check response time
+#     Sleep    30s
+#     ValidateDeploymentTime
+#     ValidateResponseTime  pap-metrics  /components/healthcheck  GET  10000
+#     ValidateResponseTimeForPap  /healthcheck  GET
+#     ValidateResponseTimeForPap  /statistics  GET
+#     ValidateResponseTimeForPap  /policies/audit  GET
+#     ValidateResponseTimeForPap  /pdps/groups/{name}  PUT
+#     ValidateResponseTimeForPap  /pdps/policies/{name}  DELETE
+#     ValidateResponseTimeForPap  /pdps/groups/{name}  DELETE
+#     ValidateResponseTimeForPap  /pdps/groups/batch  POST

diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh
index a3837b0..0f15c75 100755 (executable)
--- a/csit/run-project-csit.sh
+++ b/csit/run-project-csit.sh
@@ -27,17 +27,18 @@
 function on_exit(){
     rc=$?
     if [[ ${WORKSPACE} ]]; then
-        if [[ ${WORKDIR} ]]; then
-            rsync -av "${WORKDIR}/" "${WORKSPACE}/csit/archives/${PROJECT}"
-        fi
         # Record list of active docker containers
-        docker ps
+        docker ps --format "table {{ .Names }}\t{{ .Status }}"
 
         # Show the logs from all containers
-        docker-compose -f "${WORKSPACE}/csit/docker-compose-all.yml" logs
+        docker-compose -f "${WORKSPACE}/csit/docker-compose-all.yml" logs > docker_compose.log
 
         # show memory consumption after all docker instances initialized
         docker_stats
+
+        if [[ ${WORKDIR} ]]; then
+            rsync -av "${WORKDIR}/" "${WORKSPACE}/csit/archives/${PROJECT}"
+        fi
     fi
     # Run teardown script plan if it exists
     cd "${TESTPLANDIR}/plans/"
@@ -57,25 +58,20 @@ function docker_stats(){
     # General memory details
     if [ "$(uname -s)" == "Darwin" ]
     then
-        echo "> top -l1 | head -10"
         sh -c "top -l1 | head -10"
         echo
     else
-        echo "> top -bn1 | head -3"
         sh -c "top -bn1 | head -3"
         echo
 
-        echo "> free -h"
         sh -c "free -h"
         echo
     fi
 
     # Memory details per Docker
-    echo "> docker ps"
-    docker ps
+    docker ps --format "table {{ .Names }}\t{{ .Status }}"
     echo
 
-    echo "> docker stats --no-stream"
     docker stats --no-stream
     echo
 }

diff --git a/csit/stop-all.sh b/csit/stop-all.sh
index 4e6b0aa..9adfcf3 100755 (executable)
--- a/csit/stop-all.sh
+++ b/csit/stop-all.sh
@@ -23,4 +23,4 @@ export SCRIPTS="${SCRIPTS}"/csit
 
 source "${SCRIPTS}"/get-versions.sh
 
-docker-compose -f "${SCRIPTS}"/docker-compose-all.yml down
+docker-compose -f "${SCRIPTS}"/docker-compose-all.yml down -v

diff --git a/csit/wait_for_port.sh b/csit/wait_for_port.sh
index 5a8be25..6dcb3ca 100755 (executable)
--- a/csit/wait_for_port.sh
+++ b/csit/wait_for_port.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 # ============LICENSE_START====================================================
 #  Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
-#  Modifications Copyright (C) 2022 Nordix Foundation.
+#  Modifications Copyright (C) 2022-2023 Nordix Foundation.
 # =============================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -64,7 +64,7 @@ do
     do
         if command -v docker > /dev/null 2>&1
         then
-            docker ps
+            docker ps --format "table {{ .Names }}\t{{ .Status }}"
         fi
 
         nc -vz "$host" "$port"

diff --git a/csit/wait_for_rest.sh b/csit/wait_for_rest.sh
index bf47ccf..a64c306 100755 (executable)
--- a/csit/wait_for_rest.sh
+++ b/csit/wait_for_rest.sh
@@ -54,7 +54,7 @@ do
     do
         if command -v docker > /dev/null 2>&1
         then
-            docker ps
+            docker ps --format "table {{ .Names }}\t{{ .Status }}"
         fi
         curl "http://$host:$port" > /dev/null 2>&1
         rc=$?