tests/sdc/sdc-helm-validator/sdc-helm-validator.robot

   1 *** Settings ***
   2 Documentation     Testing SDC Helm Validator
   3 Suite Setup       Create Validator Session
   4 Library           RequestsLibrary
   5 Library           OperatingSystem
   6 Library           Collections
   7
   8 *** Variables ***
   9 ${VALIDATOR_URL}                    http://${VALIDATOR}
  10 ${VERSIONS_ENDPOINT}                /versions
  11 ${VALIDATE_ENDPOINT}                /validate
  12
  13 ${CHARTS_PATH}                      %{WORKSPACE}/tests/sdc/sdc-helm-validator/assets/charts/
  14
  15 ${CHART_CORRECT_V2}                 /correct-apiVersion-v2.tgz
  16 ${CHART_LINT_WARNING_V2}            /one-lint-warning-apiVersion-v2.tgz
  17 ${CHART_LINT_RENDER_ERROR_V2}       /one-lint-one-render-error-apiVersion-v2.tgz
  18 ${CHART_INCORRECT}                  /incorrect-chart.tgz
  19
  20 ${REQ_KEY_VERSION_DESIRED}          versionDesired
  21 ${REQ_KEY_IS_LINTED}                isLinted
  22 ${REQ_KEY_IS_STRICT_LINTED}         isStrictLinted
  23 ${REQ_KEY_FILE}                     file
  24
  25 ${RESP_KEY_VERSIONS}                versions
  26 ${RESP_KEY_VERSION_USED}            versionUsed
  27 ${RESP_KEY_DEPLOYABLE}              deployable
  28 ${RESP_KEY_VALID}                   valid
  29 ${RESP_KEY_RENDER_ERRORS}           renderErrors
  30 ${RESP_KEY_LINT_ERROR}              lintError
  31 ${RESP_KEY_LINT_WARNING}            lintWarning
  32
  33
  34 *** Test Cases ***
  35
  36 Verify That Sdc Helm Validator Correctly Responds With Supported Versions Array
  37     [Tags]                          VERSIONS_ENDPOINT
  38     [Documentation]                 Verify that validator correctly responds with supported helm versions array.
  39     ...                             Send GET request to ask for supported versions array.
  40     ...                             Should reply with JSON containing an array of Helm versions that are supported by the validator.
  41     [Timeout]                       5 minute
  42
  43     ${resp}=                        GET On Session                 ${VALIDATOR_SESSION}          ${VERSIONS_ENDPOINT}
  44     Status Should Be                200                            ${resp}
  45
  46     @{versions}=                    Get From Dictionary            ${resp.json()}                ${RESP_KEY_VERSIONS}
  47     Should Not Be Empty             ${versions}
  48
  49     FOR                             ${version}                     IN                            @{versions}
  50                                     Should Match Regexp            ${version}                    \\d+\.\\d+\.\\d+
  51     END
  52
  53 Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Default Version
  54     [Tags]                          VALIDATE_ENDPOINT    DEPLOYABLE
  55     [Documentation]                 Verify that validator correctly responds for correct chart validation request with default version.
  56     ...                             Send POST request to validate correct chart. Input: Helm chart with api version v2, no additional data.
  57     ...                             Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = []
  58     [Timeout]                       5 minute
  59
  60     ${other_data}=                  Create Dictionary
  61     ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
  62
  63     Status Should Be                200                            ${resp}
  64
  65     Dictionary Should Not Contain Key  ${resp.json()}              ${RESP_KEY_LINT_ERROR}
  66     Dictionary Should Not Contain Key  ${resp.json()}              ${RESP_KEY_LINT_WARNING}
  67     Dictionary Should Not Contain Key  ${resp.json()}              ${RESP_KEY_VALID}
  68
  69     ${version}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VERSION_USED}
  70     Should Start With               ${version}                     3.
  71
  72     ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
  73     Should Be True                  ${isDeployable}
  74
  75     ${errors}=                      Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
  76     Should Be Empty                 ${errors}
  77
  78 Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Given V3 Version
  79     [Tags]                          VALIDATE_ENDPOINT    DEPLOYABLE
  80     [Documentation]                 Verify that validator correctly responds for correct chart validation request with given v3 version.
  81     ...                             Send POST request to validate correct chart. Input: Helm chart with api version v2, desired version = v3.
  82     ...                             Should reply with JSON containing the following information: used version = 3.x.x, deployable = true, render errors = [].
  83     [Timeout]                       5 minute
  84
  85     ${other_data}=                  Create Dictionary              ${REQ_KEY_VERSION_DESIRED}=v3
  86     ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
  87
  88     Status Should Be                200                            ${resp}
  89
  90     ${version}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VERSION_USED}
  91     Should Start With               ${version}                     3.
  92
  93     ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
  94     Should Be True                  ${isDeployable}
  95
  96     ${errors}=                      Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
  97     Should Be Empty                 ${errors}
  98
  99 Verify That Sdc Helm Validator Responds With Error For Chart Validation Request With Invalid Version
 100     [Tags]                          VALIDATE_ENDPOINT    ERROR
 101     [Documentation]                 Verify that validator responds with error and 400 status code for validation request with invalid version.
 102     ...                             Send POST request with correct chart but not supported Helm version. Input: Correct helm chart, desired version = v10.
 103     ...                             Should reply with JSON containing error message with information regarding not supported Helm version. Response code should be 400.
 104     [Timeout]                       5 minute
 105
 106     ${chart_path}                   Catenate                       SEPARATOR=                    ${CHARTS_PATH}                    ${CHART_CORRECT_V2}
 107     ${files}=                       Create Multi Part              ${chart_path}
 108     ${other_data}=                  Create Dictionary              ${REQ_KEY_VERSION_DESIRED}=v10
 109     ${resp}=                        POST On Session                ${VALIDATOR_SESSION}          ${VALIDATE_ENDPOINT}              files=${files}    data=${other_data}     expected_status=anything
 110
 111     Status Should Be                400                            ${resp}
 112     Should Be Equal As Strings      ${resp.text}                   {"message":"Version: 10 is not supported"}
 113
 114 Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Random Supported Version
 115     [Tags]                          VERSIONS_ENDPOINT    VALIDATE_ENDPOINT
 116     [Documentation]                 Verify that validator correctly responds for correct chart validation request with random supported version.
 117     ...                             Send GET request to ask for supported versions array.
 118     ...                             Should reply with JSON containing an array of Helm versions that are supported by the validator.
 119     ...                             Select random version from the returned array.
 120     ...                             Send POST request with correct chart and randomly chosen supported Helm version. Input: Correct helm chart, desired version = [randomly selected].
 121     ...                             Response code should be 200.
 122     [Timeout]                       5 minute
 123
 124     ${resp}=                        GET On Session                 ${VALIDATOR_SESSION}          ${VERSIONS_ENDPOINT}
 125     ${versions}=                    Get From Dictionary            ${resp.json()}                ${RESP_KEY_VERSIONS}
 126     ${list_size}=                   Get length                     ${versions}
 127     ${random_index}=                Evaluate                       random.randint(0, ${list_size}-1)
 128     ${version}=                     Get From List                  ${versions}                   ${random_index}
 129     Status Should Be                200                            ${resp}
 130
 131     ${other_data}=                  Create Dictionary              ${REQ_KEY_VERSION_DESIRED}=${version}
 132     ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
 133     Status Should Be                200                            ${resp}
 134
 135 Verify That Sdc Helm Validator Correctly Responds For Correct Chart Validation Request With Lint
 136     [Tags]                          VALIDATE_ENDPOINT    LINT    DEPLOYABLE    VALID
 137     [Documentation]                 Verify that validator correctly responds for correct chart validation request with lint.
 138     ...                             Send POST request to validate correct chart and lint. Input: Helm chart with api version v2, linted = true.
 139     ...                             Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint errors = [], lint warnings = [].
 140     ...                             Status code should be 200.
 141     [Timeout]                       5 minute
 142
 143     ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true
 144     ${resp}=                        Send Post Request              ${CHART_CORRECT_V2}            ${other_data}
 145     Status Should Be                200                            ${resp}
 146
 147     ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
 148     Should Be True                  ${isDeployable}
 149
 150     ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
 151     Should Be True                  ${isValid}
 152
 153     ${renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
 154     Should Be Empty                 ${renderErrors}
 155
 156     ${lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
 157     Should Be Empty                 ${lintErrors}
 158
 159     ${lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
 160     Should Be Empty                 ${lintWarnings}
 161
 162 Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Warnings
 163     [Tags]                          VALIDATE_ENDPOINT    LINT    DEPLOYABLE    VALID
 164     [Documentation]                 Verify that validator correctly responds for chart validation request with lint warnings.
 165     ...                             Send POST request to validate chart and lint. Input: Helm chart that should cause lint warning, linted = true.
 166     ...                             Should reply with JSON containing the following information: deployable = true, valid = true, render errors = [], lint warning = [not empty]
 167     ...                             Status code should be 200.
 168     [Timeout]                       5 minute
 169
 170     ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true
 171     ${resp}=                        Send Post Request              ${CHART_LINT_WARNING_V2}      ${other_data}
 172     Status Should Be                200                            ${resp}
 173
 174     ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
 175     Should Be True                  ${isDeployable}
 176
 177     ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
 178     Should Be True                  ${isValid}
 179
 180     ${renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
 181     Should Be Empty                 ${renderErrors}
 182
 183     ${lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
 184     Should Be Empty                 ${lintErrors}
 185
 186     @{lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
 187     Should Not Be Empty             @{lintWarnings}
 188     Should Contain                  @{lintWarnings}                [WARNING] templates/: directory not found
 189
 190 Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint Strict Checking
 191     [Tags]                          VALIDATE_ENDPOINT    STRICT_LINT    DEPLOYABLE    NON_VALID
 192     [Documentation]                 Verify that validator correctly responds for chart validation request with lint strict checking.
 193     ...                             Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint warning, linted = true, strict linted = true.
 194     ...                             Should reply with JSON containing the following information: deployable = true, valid = false, render errors = [], lint warning = [not empty].
 195     ...                             Status code should be 200.
 196     [Timeout]                       5 minute
 197
 198     ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true     ${REQ_KEY_IS_STRICT_LINTED}=true
 199     ${resp}=                        Send Post Request              ${CHART_LINT_WARNING_V2}      ${other_data}
 200     Status Should Be                200                            ${resp}
 201
 202     ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
 203     Should Be True                  ${isDeployable}
 204
 205     ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
 206     Should Not Be True              ${isValid}                     There should be a lint warning, which in strict mode on should make the chart invalid
 207
 208     ${renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
 209     Should Be Empty                 ${renderErrors}
 210
 211     ${lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
 212     Should Be Empty                 ${lintErrors}
 213
 214     ${lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
 215     Should Not Be Empty             ${lintWarnings}
 216     Should Contain                  @{lintWarnings}                [WARNING] templates/: directory not found
 217
 218 Verify That Sdc Helm Validator Correctly Responds For Chart Validation Request With Lint And Render Errors
 219     [Tags]                          VALIDATE_ENDPOINT    STRICT_LINT    NON_DEPLOYABLE    NON_VALID
 220     [Documentation]                 Verify that validator correctly responds for chart validation request with lint and render errors.
 221     ...                             Send POST request to validate chart and strictly lint. Input: Helm chart that should cause lint and render errors, linted = true, strict linted = true.
 222     ...                             Should reply with JSON containing the following information: deployable = false, valid = false, render errors = [not empty], lint errors = [not empty], lint warnings = [].
 223     ...                             Status code should be 200.
 224     [Timeout]                       5 minute
 225
 226     ${other_data}=                  Create Dictionary              ${REQ_KEY_IS_LINTED}=true     ${REQ_KEY_IS_STRICT_LINTED}=true
 227     ${resp}=                        Send Post Request              ${CHART_LINT_RENDER_ERROR_V2}  ${other_data}
 228     Status Should Be                200                            ${resp}
 229
 230     ${isDeployable}=                Get By Key                     ${resp.json()}                ${RESP_KEY_DEPLOYABLE}
 231     Should Not Be True              ${isDeployable}                There should be render errors which should make the chart not deployable
 232
 233     ${isValid}=                     Get By Key                     ${resp.json()}                ${RESP_KEY_VALID}
 234     Should Not Be True              ${isValid}                     There should be lint errors which should make the chart invalid
 235
 236     @{renderErrors}=                Get By Key                     ${resp.json()}                ${RESP_KEY_RENDER_ERRORS}
 237     Should Not Be Empty             @{renderErrors}
 238     Should Contain                  @{renderErrors}                Error: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository
 239
 240     @{lintErrors}=                  Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_ERROR}
 241     Should Not Be Empty             @{lintErrors}
 242     Should Contain                  @{lintErrors}                  [ERROR] templates/: template: mychartname/templates/test.yaml:2:18: executing "mychartname/templates/test.yaml" at <.Values.image.repository>: nil pointer evaluating interface {}.repository
 243
 244     ${lintWarnings}=                Get By Key                     ${resp.json()}                ${RESP_KEY_LINT_WARNING}
 245     Should Be Empty                 ${lintWarnings}
 246
 247 *** Keywords ***
 248
 249 Create Validator Session
 250     Create Session                  validator_session              ${VALIDATOR_URL}
 251     Set Suite Variable              ${VALIDATOR_SESSION}           validator_session
 252
 253 Send Post Request
 254     [Arguments]                     ${chart_name}                  ${data_dictionary}
 255     ${chart_path}                   Catenate                       SEPARATOR=                    ${CHARTS_PATH}                    ${chart_name}
 256     ${files}=                       Create Multi Part              ${chart_path}
 257
 258     ${resp}=                        POST On Session                ${VALIDATOR_SESSION}          ${VALIDATE_ENDPOINT}              files=${files}    data=${data_dictionary}
 259     [Return]                        ${resp}
 260
 261 Create Multi Part
 262     [Arguments]                     ${path}
 263     ${data}=                        Get Binary File                ${path}
 264     ${files}=                       Create Dictionary
 265     ${fileDir}  ${fileName}=        Split Path                     ${path}
 266     ${partData}=                    Create List                    ${fileName}                 ${data}
 267     Set To Dictionary               ${files}                       ${REQ_KEY_FILE}=${partData}
 268     [Return]                        ${files}
 269
 270 Get By Key
 271     [Arguments]                     ${dict}                        ${key}
 272     Dictionary Should Contain Key   ${dict}                        ${key}
 273     ${value}=                       Get From Dictionary            ${dict}                     ${key}
 274     [Return]                        ${value}