From 9979cb6be5037336497c5c17e0ece5e2355836e3 Mon Sep 17 00:00:00 2001 From: Joanna Jeremicz Date: Fri, 26 Mar 2021 11:36:30 +0100 Subject: [PATCH] Add test suit for SDC helm chart validator Issue-ID: SDC-3185 Signed-off-by: Joanna Jeremicz Change-Id: Id53318a912766ff8bd76637614f65533dfd62ed6 --- plans/sdc/sdc-helm-validator/setup.sh | 29 +++ plans/sdc/sdc-helm-validator/teardown.sh | 7 + plans/sdc/sdc-helm-validator/testplan.txt | 3 + .../assets/charts/correct-apiVersion-v2.tgz | Bin 0 -> 651 bytes .../one-lint-one-render-error-apiVersion-v2.tgz | Bin 0 -> 668 bytes .../charts/one-lint-warning-apiVersion-v2.tgz | Bin 0 -> 738 bytes .../sdc-helm-validator/sdc-helm-validator.robot | 274 +++++++++++++++++++++ 7 files changed, 313 insertions(+) create mode 100644 plans/sdc/sdc-helm-validator/setup.sh create mode 100644 plans/sdc/sdc-helm-validator/teardown.sh create mode 100644 plans/sdc/sdc-helm-validator/testplan.txt create mode 100644 tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz create mode 100644 tests/sdc/sdc-helm-validator/assets/charts/one-lint-one-render-error-apiVersion-v2.tgz create mode 100644 tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz create mode 100644 tests/sdc/sdc-helm-validator/sdc-helm-validator.robot diff --git a/plans/sdc/sdc-helm-validator/setup.sh b/plans/sdc/sdc-helm-validator/setup.sh new file mode 100644 index 00000000..6d079679 --- /dev/null +++ b/plans/sdc/sdc-helm-validator/setup.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +export SDC_HELM_VALIDATOR="sdc-helm-validator" +export SDC_HELM_VALIDATOR_IMAGE="nexus3.onap.org:10001/onap/org.onap.sdc.sdc-helm-validator:latest" + +unset http_proxy +unset https_proxy + +docker run -p 8080:8080 -d --name ${SDC_HELM_VALIDATOR} ${SDC_HELM_VALIDATOR_IMAGE} + +# Wait container ready +HELM_VALIDATOR_IP='none' +for i in {1..5} +do + HELM_VALIDATOR_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${SDC_HELM_VALIDATOR}) + RESP_CODE=$(curl -s http://${HELM_VALIDATOR_IP}:8080/actuator/health | python2 -c 'import json,sys;obj=json.load(sys.stdin);print obj["status"]') + if [[ "$RESP_CODE" == "UP" ]]; then + echo 'SDC Helm Validator is ready' + break + fi + + echo 'Waiting for SDC Helm Validator to start up...' + sleep 10s +done + +echo HELM_VALIDATOR_IP=${HELM_VALIDATOR_IP} + +# Pass any variables required by Robot test suites in ROBOT_VARIABLES +ROBOT_VARIABLES="-v VALIDATOR:${HELM_VALIDATOR_IP}:8080" diff --git a/plans/sdc/sdc-helm-validator/teardown.sh b/plans/sdc/sdc-helm-validator/teardown.sh new file mode 100644 index 00000000..9916a424 --- /dev/null +++ b/plans/sdc/sdc-helm-validator/teardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash +echo "Starting teardown script" + +running_container=$(docker ps --filter name=sdc-helm-validator -qa) + +docker stop $running_container +docker rm $running_container diff --git a/plans/sdc/sdc-helm-validator/testplan.txt b/plans/sdc/sdc-helm-validator/testplan.txt new file mode 100644 index 00000000..9133005c --- /dev/null +++ b/plans/sdc/sdc-helm-validator/testplan.txt @@ -0,0 +1,3 @@ +# Test suites are relative paths under [integration/csit.git]/tests/. +# Place the suites in run order. +sdc/sdc-helm-validator diff --git a/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/correct-apiVersion-v2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..a1a99babd5ba614ceac163c8074b4533044db6d7 GIT binary patch literal 651 zcmV;60(AW!iwFSOnNVN=1MOJdZ__Xk@3a4kV|#&#IdQ%!^$l%EARa&(>ML>+H!(|M z%XTu#H2(HoQr1qWPytc-XnqfNVxP};&iA`>Qe})t-8t8CE{v6`K5qkWGAMlCpT;p@ z&8D&cxZVI*r$q?6DRG1VxKbV6YSbdb;2l1sDDEK z^!|?>yZyiAWh3lR1hCKl{V?46f5>>DEH(U zg|~IN0#)}TyCs1Z21;2AO6cX)hbs(00##h7szL?xiz~>av4rN*d40KFKB0@x#_P+6 zh9dX6;^Eb{wYOEY!2LBEma-Ik-^ITfb7qTq&pM$Ysg{)k7;|Ayr;%tKbm z630+3(uVHlg=OectBh|YUcf6XN@#oGN?UkSPtjKB(J{X;G*lW>s8lxMY`#(~c)$gt zuJu$W0S^)`PS~H`|6|9l_CJ?JjqWf9u+9EO6tC<*5m7|;A5kFa?Ee^0rJP|fW&jqA zcFJqJ7z5d}XPs?60_=y^N`u`+Y=YOMc zMEu7|77h^fTKfEz^u?yD5Xzxl73G22+9-G9BG`-@3qS+BLx-C z+MfgP)L6VypD=?PMRHG`ad=x53!rn@jBJ(!1})IxM9%od<%dfOK}agjby<>v$@wK@ z$~wkp$`9IdxqQZ_pY5P6R}J}W&=jjzH?Ia8MN`^eZQx23=-3%|H-^foyr$BZ&9f#w z?CcATm`JO1?cnvrOPbeM{Si~oxXMrtmg3aJ_}ryB!$B7-J9OyqH{&Z;6%j!I8UO%c CQB#Zn literal 0 HcmV?d00001 diff --git a/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz b/tests/sdc/sdc-helm-validator/assets/charts/one-lint-warning-apiVersion-v2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..51f4dd1d9f93a2deae3da3a3c281b751f4a81d79 GIT binary patch literal 738 zcmV<80v-JyiwFR9pip1{1MOB(Z__Xk-m`zjvAw`Vouq9N)Hl>3fk1$^X|Kplyu{Rr zE!#;M5Bzq{NxO|nT|ujcs`Vn-&Ud~$Cug5ORrA4;nOfAvfN62bjTNdMw8P+NXA*{C z8pnWrl*ZxR{zf1grD-@0M+uI@C{D)V5yZP&!e(i#V+NPPtY8n#+ooTA$4$Q{qBiD# zS~BD4HLK)fW8nWt#^cTTABHJJ<8U}i(l8w-fWUARA3?Z_58Ni}{A;0{d0^f){hsFk zHs=#)hooi|pFn2vf#cSZoZHL@?K<^ac+X`8nco+%6s`nS%WIgP%`6yhRbw)4iM!V5 zfoUy8#=IeE*ZxBq(U73(CgO2KqlAXXMy`=+h0$lGiYAgv@bUfM(f(72HyC0vDyQU>mKKaX7#&BB79K z4i$4*iL+m0DUf*XTaI>EB5rY3=jfp3MOPKzr8fLZ{NQqYVF9|d^ zAVtk7p|kUE=ML9~E~cug(81;O9CBeSp@nck*Y?XNbpF!>UAqR#BJj=n)wXqT>u8SO z*J!vB63f>qwM&idoGq|*l^-kgLr#C8#U(RBH5NY1-eFvA)OVgaLPgG5&`(E*(AH)u z=fOTywaw=rF9Ew768P`@FNxF9f&V>)Y{|c80RtR8iU)43|8eqX{)Z7hhx-2%a+m+% znE!CR_HULh-dHMB&_;9aSVR`0&L=R%-=(Uv72BI(1+PU%@+zE^Gsh+bpeK=j>sabr zbVID>nb#FYHM&=NxSwpj2A~gd;fbezmBPA@H_#`sg0*Z6lPmunVsY~}l9?Vu2+OZ0 zp`X2LyuEmi*ouX&LJhjOfSfmCQkQJlp=ilX%Q2!e4PQfe?cnuy2fJ?^{e|%rla}wp Ug>aCA9OT)^ZyN|d_5c(B0QpIK?EnA( literal 0 HcmV?d00001 diff --git a/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot b/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot new file mode 100644 index 00000000..04b570f9 --- /dev/null +++ b/tests/sdc/sdc-helm-validator/sdc-helm-validator.robot @@ -0,0 +1,274 @@ +*** Settings *** +Documentation Testing SDC Helm Validator +Suite Setup Create Validator Session +Library RequestsLibrary +Library OperatingSystem +Library Collections + +*** Variables *** +${VALIDATOR_URL} http://${VALIDATOR} +${VERSIONS_ENDPOINT} /versions +${VALIDATE_ENDPOINT} /validate + +${CHARTS_PATH} %{WORKSPACE}/tests/sdc/sdc-helm-validator/assets/charts/ + +${CHART_CORRECT_V2} /correct-apiVersion-v2.tgz +${CHART_LINT_WARNING_V2} /one-lint-warning-apiVersion-v2.tgz +${CHART_LINT_RENDER_ERROR_V2} /one-lint-one-render-error-apiVersion-v2.tgz +${CHART_INCORRECT} /incorrect-chart.tgz + +${REQ_KEY_VERSION_DESIRED} versionDesired +${REQ_KEY_IS_LINTED} isLinted +${REQ_KEY_IS_STRICT_LINTED} isStrictLinted +${REQ_KEY_FILE} file + +${RESP_KEY_VERSIONS} versions +${RESP_KEY_VERSION_USED} versionUsed +${RESP_KEY_DEPLOYABLE} deployable +${RESP_KEY_VALID} valid +${RESP_KEY_RENDER_ERRORS} renderErrors +${RESP_KEY_LINT_ERROR} lintError +${RESP_KEY_LINT_WARNING} lintWarning + + +*** Test Cases *** + +Verify That Sdc Helm Validator Correctly Responds With Supported Versions Array + [Tags] SDC_HELM_VALIDATOR_1 + [Documentation] Verify that validator correctly responds with supported helm versions array. + ... Send GET request to ask for supported versions array. + ... Should reply with JSON containing an array of Helm versions that are supported by the validator. + [Timeout] 5 minute + + ${resp}= GET On Session ${VALIDATOR_SESSION} ${VERSIONS_ENDPOINT} + Status Should Be 200 ${resp} + + @{versions}= Get From Dictionary ${resp.json()} ${RESP_KEY_VERSIONS} + Should Not Be Empty ${versions} + + FOR ${version} IN @{versions} + Should Match Regexp ${version} \\d+\.\\d+\.\\d+ + END + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Default Version + [Tags] SDC_HELM_VALIDATOR_2 + [Documentation] Verify that validator correctly responds for correct chart validation request with default version. + ... Send POST request to validate correct chart. Input: Helm chart with api version v2, no additional data. + ... Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = [] + [Timeout] 5 minute + + ${other_data}= Create Dictionary + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + + Status Should Be 200 ${resp} + + Dictionary Should Not Contain Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Dictionary Should Not Contain Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Dictionary Should Not Contain Key ${resp.json()} ${RESP_KEY_VALID} + + ${version}= Get By Key ${resp.json()} ${RESP_KEY_VERSION_USED} + Should Start With ${version} 3. + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${errors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${errors} + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Given V3 Version + [Tags] SDC_HELM_VALIDATOR_3 + [Documentation] Verify that validator correctly responds for correct chart validation request with given v3 version. + ... Send POST request to validate correct chart. Input: Helm chart with api version v2, desired version = v3. + ... Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = []. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_VERSION_DESIRED}=v3 + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + + Status Should Be 200 ${resp} + + ${version}= Get By Key ${resp.json()} ${RESP_KEY_VERSION_USED} + Should Start With ${version} 3. + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${errors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${errors} + +Verify That Sdc Helm Validator Responds With Error For Chart Validation Request With Invalid Version + [Tags] SDC_HELM_VALIDATOR_4 + [Documentation] Verify that validator responds with error and 400 status code for validation request with invalid version. + ... Send POST request with correct chart but not supported Helm version. Input: Correct helm chart, desired version = v10. + ... Should reply with JSON containing error message with information regarding not supported Helm version. Response code should be 400. + [Timeout] 5 minute + + ${chart_path} Catenate SEPARATOR= ${CHARTS_PATH} ${CHART_CORRECT_V2} + ${files}= Create Multi Part ${chart_path} + ${other_data}= Create Dictionary ${REQ_KEY_VERSION_DESIRED}=v10 + ${resp}= Post Request ${VALIDATOR_SESSION} ${VALIDATE_ENDPOINT} files=${files} data=${other_data} + + Should Be Equal As Strings ${resp.status_code} 400 + Should Be Equal As Strings ${resp.text} {"message":"Version: 10 is not supported"} + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Random Supported Version + [Tags] SDC_HELM_VALIDATOR_5 + [Documentation] Verify that validator correctly responds for correct chart validation request with random supported version. + ... Send GET request to ask for supported versions array. + ... Should reply with JSON containing an array of Helm versions that are supported by the validator. + ... Select random version from the returned array. + ... Send POST request with correct chart and randomly chosen supported Helm version. Input: Correct helm chart, desired version = [randomly selected]. + ... Response code should be 200. + [Timeout] 5 minute + + ${resp}= GET On Session ${VALIDATOR_SESSION} ${VERSIONS_ENDPOINT} + ${versions}= Get From Dictionary ${resp.json()} ${RESP_KEY_VERSIONS} + ${list_size}= Get length ${versions} + ${random_index}= Evaluate random.randint(0, ${list_size}-1) + ${version}= Get From List ${versions} ${random_index} + Status Should Be 200 ${resp} + + ${other_data}= Create Dictionary ${REQ_KEY_VERSION_DESIRED}=${version} + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + Status Should Be 200 ${resp} + +Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Lint + [Tags] SDC_HELM_VALIDATOR_6 + [Documentation] Verify that validator correctly responds for correct chart validation request with lint. + ... Send POST request to validate correct chart and lint. Input: Helm chart with api version v2, linted = true. + ... Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint errors = [], lint warnings = []. + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true + ${resp}= Send Post Request ${CHART_CORRECT_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Be True ${isValid} + + ${renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${renderErrors} + + ${lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Be Empty ${lintErrors} + + ${lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Be Empty ${lintWarnings} + +Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Warnings + [Tags] SDC_HELM_VALIDATOR_7 + [Documentation] Verify that validator correctly responds for chart validation request with lint warnings. + ... Send POST request to validate chart and lint. Input: Helm chart that should cause lint warning, linted = true. + ... Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint warning = [not empty] + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true + ${resp}= Send Post Request ${CHART_LINT_WARNING_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Be True ${isValid} + + ${renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${renderErrors} + + ${lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Be Empty ${lintErrors} + + @{lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Not Be Empty @{lintWarnings} + Should Contain @{lintWarnings} [WARNING] templates/: directory not found + +Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Strict Checking + [Tags] SDC_HELM_VALIDATOR_8 + [Documentation] Verify that validator correctly responds for chart validation request with lint strict checking. + ... Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint warning, linted = true, strict linted = true. + ... Should reply with JSON containing the following information: deployable = true, valid = false, render errors = [], lint warning = [not empty]. + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true ${REQ_KEY_IS_STRICT_LINTED}=true + ${resp}= Send Post Request ${CHART_LINT_WARNING_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Be True ${isDeployable} + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Not Be True ${isValid} There should be a lint warning, which in strict mode on should make the chart invalid + + ${renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Be Empty ${renderErrors} + + ${lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Be Empty ${lintErrors} + + ${lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Not Be Empty ${lintWarnings} + Should Contain @{lintWarnings} [WARNING] templates/: directory not found + +Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint And Render Errors + [Tags] SDC_HELM_VALIDATOR_9 + [Documentation] Verify that validator correctly responds for chart validation request with lint and render errors. + ... Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint and render errors, linted = true, strict linted = true. + ... Should reply with JSON containing the following information: deployable = false, valid = false, render errors = [not empty], lint errors = [not empty], lint warnings = []. + ... Status code should be 200. + [Timeout] 5 minute + + ${other_data}= Create Dictionary ${REQ_KEY_IS_LINTED}=true ${REQ_KEY_IS_STRICT_LINTED}=true + ${resp}= Send Post Request ${CHART_LINT_RENDER_ERROR_V2} ${other_data} + Status Should Be 200 ${resp} + + ${isDeployable}= Get By Key ${resp.json()} ${RESP_KEY_DEPLOYABLE} + Should Not Be True ${isDeployable} There should be render errors which should make the chart not deployable + + ${isValid}= Get By Key ${resp.json()} ${RESP_KEY_VALID} + Should Not Be True ${isValid} There should be lint errors which should make the chart invalid + + @{renderErrors}= Get By Key ${resp.json()} ${RESP_KEY_RENDER_ERRORS} + Should Not Be Empty @{renderErrors} + Should Contain @{renderErrors} Error: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository + + @{lintErrors}= Get By Key ${resp.json()} ${RESP_KEY_LINT_ERROR} + Should Not Be Empty @{lintErrors} + Should Contain @{lintErrors} [ERROR] templates/: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository + + ${lintWarnings}= Get By Key ${resp.json()} ${RESP_KEY_LINT_WARNING} + Should Be Empty ${lintWarnings} + +*** Keywords *** + +Create Validator Session + Create Session validator_session ${VALIDATOR_URL} + Set Suite Variable ${VALIDATOR_SESSION} validator_session + +Send Post Request + [Arguments] ${chart_name} ${data_dictionary} + ${chart_path} Catenate SEPARATOR= ${CHARTS_PATH} ${chart_name} + ${files}= Create Multi Part ${chart_path} + + ${resp}= POST On Session ${VALIDATOR_SESSION} ${VALIDATE_ENDPOINT} files=${files} data=${data_dictionary} + [Return] ${resp} + +Create Multi Part + [Arguments] ${path} + ${data}= Get Binary File ${path} + ${files}= Create Dictionary + ${fileDir} ${fileName}= Split Path ${path} + ${partData}= Create List ${fileName} ${data} + Set To Dictionary ${files} ${REQ_KEY_FILE}=${partData} + [Return] ${files} + +Get By Key + [Arguments] ${dict} ${key} + Dictionary Should Contain Key ${dict} ${key} + ${value}= Get From Dictionary ${dict} ${key} + [Return] ${value} -- 2.16.6