From: Marcin Przybysz Date: Wed, 31 Mar 2021 11:13:38 +0000 (+0000) Subject: Merge "Add test suit for SDC helm chart validator" X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=ddd487752cf6eddd891d7b8fe5062316a932414d;hp=a700e2f085776f201b9a24726d3d961981089161;p=integration%2Fcsit.git Merge "Add test suit for SDC helm chart validator" --- diff --git a/plans/sdc/sdc-helm-validator/setup.sh b/plans/sdc/sdc-helm-validator/setup.sh new file mode 100644 index 00000000..6d079679 --- /dev/null +++ b/plans/sdc/sdc-helm-validator/setup.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +export SDC_HELM_VALIDATOR="sdc-helm-validator" +export SDC_HELM_VALIDATOR_IMAGE="nexus3.onap.org:10001/onap/org.onap.sdc.sdc-helm-validator:latest" + +unset http_proxy +unset https_proxy + +docker run -p 8080:8080 -d --name ${SDC_HELM_VALIDATOR} ${SDC_HELM_VALIDATOR_IMAGE} + +# Wait container ready +HELM_VALIDATOR_IP='none' +for i in {1..5} +do + HELM_VALIDATOR_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${SDC_HELM_VALIDATOR}) + RESP_CODE=$(curl -s http://${HELM_VALIDATOR_IP}:8080/actuator/health | python2 -c 'import json,sys;obj=json.load(sys.stdin);print obj["status"]') + if [[ "$RESP_CODE" == "UP" ]]; then + echo 'SDC Helm Validator is ready' + break + fi + + echo 'Waiting for SDC Helm Validator to start up...' + sleep 10s +done + +echo HELM_VALIDATOR_IP=${HELM_VALIDATOR_IP} + +# Pass any variables required by Robot test suites in ROBOT_VARIABLES +ROBOT_VARIABLES="-v VALIDATOR:${HELM_VALIDATOR_IP}:8080" diff --git a/plans/sdc/sdc-helm-validator/teardown.sh b/plans/sdc/sdc-helm-validator/teardown.sh new file mode 100644 index 00000000..9916a424 --- /dev/null +++ b/plans/sdc/sdc-helm-validator/teardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash +echo "Starting teardown script" + +running_container=$(docker ps --filter name=sdc-helm-validator -qa) + +docker stop $running_container +docker rm $running_container diff --git a/plans/sdc/sdc-helm-validator/testplan.txt b/plans/sdc/sdc-helm-validator/testplan.txt new file mode 100644 index 00000000..9133005c --- /dev/null +++ b/plans/sdc/sdc-helm-validator/testplan.txt @@ -0,0 +1,3 @@ +# Test suites are relative paths under [integration/csit.git]/tests/. +# Place the suites in run order. +sdc/sdc-helm-validator diff --git a/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz new file mode 100644 index 00000000..a1a99bab Binary files /dev/null and b/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz differ diff --git a/tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz new file mode 100644 index 00000000..874dd3c0 Binary files /dev/null and b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz differ diff --git a/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz new file mode 100644 index 00000000..51f4dd1d Binary files /dev/null and b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz differ diff --git a/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot b/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot new file mode 100644 index 00000000..04b570f9 --- /dev/null +++ b/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot @@ -0,0 +1,274 @@ +*** Settings *** +Documentation Testing SDC Helm Validator +Suite Setup Create Validator Session +Library RequestsLibrary +Library OperatingSystem +Library Collections + +*** Variables *** +${VALIDATOR_URL} http://${VALIDATOR} +${VERSIONS_ENDPOINT} /versions +${VALIDATE_ENDPOINT} /validate + +${CHARTS_PATH} %{WORKSPACE}/tests/sdc/sdc-helm-validator/assets/charts/ + +${CHART_CORRECT_V2} /correct-apiVersion-v2.tgz +${CHART_LINT_WARNING_V2} /one-lint-warning-apiVersion-v2.tgz +${CHART_LINT_RENDER_ERROR_V2} /one-lint-one-render-error-apiVersion-v2.tgz +${CHART_INCORRECT} /incorrect-chart.tgz + +${REQ_KEY_VERSION_DESIRED} versionDesired +${REQ_KEY_IS_LINTED} isLinted +${REQ_KEY_IS_STRICT_LINTED} isStrictLinted +${REQ_KEY_FILE} file + +${RESP_KEY_VERSIONS} versions +${RESP_KEY_VERSION_USED} versionUsed +${RESP_KEY_DEPLOYABLE} deployable +${RESP_KEY_VALID} valid +${RESP_KEY_RENDER_ERRORS} renderErrors +${RESP_KEY_LINT_ERROR} lintError +${RESP_KEY_LINT_WARNING} lintWarning + + +*** Test Cases *** + +Verify That Sdc Helm Validator Correctly Responds With Supported Versions Array + [Tags] SDC_HELM_VALIDATOR_1 + [Documentation] Verify that validator correctly responds with supported helm versions array. + ... Send GET request to ask for supported versions array. + ... Should reply with JSON containing an array of Helm versions that are supported by the validator. + [Timeout] 5 minute + + ${resp}= GET On Session ${VALIDATOR_SESSION} ${VERSIONS_ENDPOINT} + Status Should Be 200 ${resp} + + @{versions}= Get From Dictionary ${resp.json()} ${RESP_KEY_VERSIONS} + Should Not Be Empty ${versions} + + FOR ${version} IN @{versions} + Should Match Regexp ${version} \\d+\.\\d+\.\\d+ + END + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Default Version + [Tags] SDC_HELM_VALIDATOR_2 + [Documentation] Verify that validator correctly responds for correct chart validation request with default version. + ... Send POST request to validate correct chart. Input: Helm chart with api version v2, no additional data. + ... Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = [] + [Timeout] 5 minute + + ${other_data}= Create Dictionary + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + + Status Should Be 200 ${resp} + + Dictionary Should Not Contain Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Dictionary Should Not Contain Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Dictionary Should Not Contain Key ${resp.json()} ${RESP_KEY_VALID} + + ${version}= Get By Key ${resp.json()} ${RESP_KEY_VERSION_USED} + Should Start With ${version} 3. + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${errors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${errors} + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Given V3 Version + [Tags] SDC_HELM_VALIDATOR_3 + [Documentation] Verify that validator correctly responds for correct chart validation request with given v3 version. + ... Send POST request to validate correct chart. Input: Helm chart with api version v2, desired version = v3. + ... Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = []. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_VERSION_DESIRED}=v3 + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + + Status Should Be 200 ${resp} + + ${version}= Get By Key ${resp.json()} ${RESP_KEY_VERSION_USED} + Should Start With ${version} 3. + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${errors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${errors} + +Verify That Sdc Helm Validator Responds With Error For Chart Validation Request With Invalid Version + [Tags] SDC_HELM_VALIDATOR_4 + [Documentation] Verify that validator responds with error and 400 status code for validation request with invalid version. + ... Send POST request with correct chart but not supported Helm version. Input: Correct helm chart, desired version = v10. + ... Should reply with JSON containing error message with information regarding not supported Helm version. Response code should be 400. + [Timeout] 5 minute + + ${chart_path} Catenate SEPARATOR= ${CHARTS_PATH} ${CHART_CORRECT_V2} + ${files}= Create Multi Part ${chart_path} + ${other_data}= Create Dictionary ${REQ_KEY_VERSION_DESIRED}=v10 + ${resp}= Post Request ${VALIDATOR_SESSION} ${VALIDATE_ENDPOINT} files=${files} data=${other_data} + + Should Be Equal As Strings ${resp.status_code} 400 + Should Be Equal As Strings ${resp.text} {"message":"Version: 10 is not supported"} + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Random Supported Version + [Tags] SDC_HELM_VALIDATOR_5 + [Documentation] Verify that validator correctly responds for correct chart validation request with random supported version. + ... Send GET request to ask for supported versions array. + ... Should reply with JSON containing an array of Helm versions that are supported by the validator. + ... Select random version from the returned array. + ... Send POST request with correct chart and randomly chosen supported Helm version. Input: Correct helm chart, desired version = [randomly selected]. + ... Response code should be 200. + [Timeout] 5 minute + + ${resp}= GET On Session ${VALIDATOR_SESSION} ${VERSIONS_ENDPOINT} + ${versions}= Get From Dictionary ${resp.json()} ${RESP_KEY_VERSIONS} + ${list_size}= Get length ${versions} + ${random_index}= Evaluate random.randint(0, ${list_size}-1) + ${version}= Get From List ${versions} ${random_index} + Status Should Be 200 ${resp} + + ${other_data}= Create Dictionary ${REQ_KEY_VERSION_DESIRED}=${version} + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + Status Should Be 200 ${resp} + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Lint + [Tags] SDC_HELM_VALIDATOR_6 + [Documentation] Verify that validator correctly responds for correct chart validation request with lint. + ... Send POST request to validate correct chart and lint. Input: Helm chart with api version v2, linted = true. + ... Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint errors = [], lint warnings = []. + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Be True ${isValid} + + ${renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${renderErrors} + + ${lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Be Empty ${lintErrors} + + ${lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Be Empty ${lintWarnings} + +Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Warnings + [Tags] SDC_HELM_VALIDATOR_7 + [Documentation] Verify that validator correctly responds for chart validation request with lint warnings. + ... Send POST request to validate chart and lint. Input: Helm chart that should cause lint warning, linted = true. + ... Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint warning = [not empty] + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true + ${resp}= Send Post Request ${CHART_LINT_WARNING_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Be True ${isValid} + + ${renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${renderErrors} + + ${lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Be Empty ${lintErrors} + + @{lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Not Be Empty @{lintWarnings} + Should Contain @{lintWarnings} [WARNING] templates/: directory not found + +Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Strict Checking + [Tags] SDC_HELM_VALIDATOR_8 + [Documentation] Verify that validator correctly responds for chart validation request with lint strict checking. + ... Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint warning, linted = true, strict linted = true. + ... Should reply with JSON containing the following information: deployable = true, valid = false, render errors = [], lint warning = [not empty]. + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true ${REQ_KEY_IS_STRICT_LINTED}=true + ${resp}= Send Post Request ${CHART_LINT_WARNING_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Not Be True ${isValid} There should be a lint warning, which in strict mode on should make the chart invalid + + ${renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${renderErrors} + + ${lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Be Empty ${lintErrors} + + ${lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Not Be Empty ${lintWarnings} + Should Contain @{lintWarnings} [WARNING] templates/: directory not found + +Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint And Render Errors + [Tags] SDC_HELM_VALIDATOR_9 + [Documentation] Verify that validator correctly responds for chart validation request with lint and render errors. + ... Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint and render errors, linted = true, strict linted = true. + ... Should reply with JSON containing the following information: deployable = false, valid = false, render errors = [not empty], lint errors = [not empty], lint warnings = []. + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true ${REQ_KEY_IS_STRICT_LINTED}=true + ${resp}= Send Post Request ${CHART_LINT_RENDER_ERROR_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Not Be True ${isDeployable} There should be render errors which should make the chart not deployable + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Not Be True ${isValid} There should be lint errors which should make the chart invalid + + @{renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Not Be Empty @{renderErrors} + Should Contain @{renderErrors} Error: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository + + @{lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Not Be Empty @{lintErrors} + Should Contain @{lintErrors} [ERROR] templates/: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository + + ${lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Be Empty ${lintWarnings} + +*** Keywords *** + +Create Validator Session + Create Session validator_session ${VALIDATOR_URL} + Set Suite Variable ${VALIDATOR_SESSION} validator_session + +Send Post Request + [Arguments] ${chart_name} ${data_dictionary} + ${chart_path} Catenate SEPARATOR= ${CHARTS_PATH} ${chart_name} + ${files}= Create Multi Part ${chart_path} + + ${resp}= POST On Session ${VALIDATOR_SESSION} ${VALIDATE_ENDPOINT} files=${files} data=${data_dictionary} + [Return] ${resp} + +Create Multi Part + [Arguments] ${path} + ${data}= Get Binary File ${path} + ${files}= Create Dictionary + ${fileDir} ${fileName}= Split Path ${path} + ${partData}= Create List ${fileName} ${data} + Set To Dictionary ${files} ${REQ_KEY_FILE}=${partData} + [Return] ${files} + +Get By Key + [Arguments] ${dict} ${key} + Dictionary Should Contain Key ${dict} ${key} + ${value}= Get From Dictionary ${dict} ${key} + [Return] ${value}