Define new Suite (js): Endurance

- have one ncmp-test-runner.js with different configs: kpi.json and
  endurance.json
- move scenarios and thresholds settings into the json configs

Issue-ID: CPS-2444

Change-Id: I72f65036dc3f7762284b32903e1d05474dbed6c2
Signed-off-by: halil.cakal <halil.cakal@est.tech>

diff --git a/k6-tests/README.md b/k6-tests/README.md
index f74c9d4..506e6a7 100644 (file)
--- a/k6-tests/README.md
+++ b/k6-tests/README.md
@@ -22,5 +22,5 @@ docker-compose -f docker-compose/docker-compose.yml --profile dmi-stub up
 
 To run an individual test from command line, use
 ```shell
-k6 run ncmp/ncmp-kpi.js
+k6 run ncmp/ncmp-test-runner.js
 ```

diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index 45f6e96..8abe54c 100644 (file)
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -20,13 +20,11 @@
 
 import http from 'k6/http';
 
-const testConfig = JSON.parse(open(`../config/${__ENV.TEST_PROFILE}.json`));
+export const testConfig = JSON.parse(open(`../config/${__ENV.TEST_PROFILE}.json`));
 export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer;
-export const LEGACY_BATCH_TOPIC_NAME = testConfig.kafka.legacyBatchTopic;
-export const DURATION = testConfig.timingConfig.testDuration;
-export const LEGACY_BATCH_THROUGHPUT_TEST_START_TIME = testConfig.timingConfig.legacyBatchThroughputTestStartTime;
 export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl;
 export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl;
+export const LEGACY_BATCH_TOPIC_NAME = 'legacy_batch_topic';
 export const TOTAL_CM_HANDLES = 20000;
 export const REGISTRATION_BATCH_SIZE = 100;
 export const READ_DATA_FOR_CM_HANDLE_DELAY_MS = 300; // must have same value as in docker-compose.yml

diff --git a/k6-tests/ncmp/config/endurance.json b/k6-tests/ncmp/config/endurance.json
index c9def6c..d215d0a 100644 (file)
--- a/k6-tests/ncmp/config/endurance.json
+++ b/k6-tests/ncmp/config/endurance.json
@@ -4,10 +4,98 @@
     "dmiStubUrl": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092",
     "kafkaBootstrapServer": "localhost:9093"
   },
-  "timingConfig": {
-    "testDuration": "2h"
-  },
-  "kafka": {
-    "legacyBatchTopic": "legacy_batch_topic"
+  "scenarios": {
+    "passthrough_read_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughReadScenario",
+      "vus": 2,
+      "duration": "2h"
+    },
+    "passthrough_read_alt_id_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughReadAltIdScenario",
+      "vus": 2,
+      "duration": "2h"
+    },
+    "passthrough_write_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughWriteScenario",
+      "vus": 2,
+      "duration": "2h"
+    },
+    "passthrough_write_alt_id_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughWriteAltIdScenario",
+      "vus": 2,
+      "duration": "2h"
+    },
+    "cm_handle_id_search_nofilter_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchNoFilterScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_search_nofilter_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchNoFilterScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_id_search_module_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchModuleScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_search_module_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchModuleScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_id_search_property_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchPropertyScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_search_property_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchPropertyScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_id_search_cpspath_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchCpsPathScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_search_cpspath_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchCpsPathScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_id_search_trustlevel_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchTrustLevelScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "cm_handle_search_trustlevel_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchTrustLevelScenario",
+      "vus": 1,
+      "duration": "2h"
+    },
+    "legacy_batch_produce_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "legacyBatchProduceScenario",
+      "duration": "2h",
+      "rate": 1,
+      "timeUnit": "1s",
+      "preAllocatedVUs": 1
+    }
   }
 }

diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json
index ad79f92..742321f 100644 (file)
--- a/k6-tests/ncmp/config/kpi.json
+++ b/k6-tests/ncmp/config/kpi.json
@@ -4,11 +4,122 @@
     "dmiStubUrl": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092",
     "kafkaBootstrapServer": "localhost:9092"
   },
-  "timingConfig": {
-    "testDuration": "15m",
-    "legacyBatchThroughputTestStartTime": "15m30s"
+  "scenarios": {
+    "passthrough_read_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughReadScenario",
+      "vus": 2,
+      "duration": "15m"
+    },
+    "passthrough_read_alt_id_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughReadAltIdScenario",
+      "vus": 2,
+      "duration": "15m"
+    },
+    "passthrough_write_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughWriteScenario",
+      "vus": 2,
+      "duration": "15m"
+    },
+    "passthrough_write_alt_id_scenario": {
+      "executor": "constant-vus",
+      "exec": "passthroughWriteAltIdScenario",
+      "vus": 2,
+      "duration": "15m"
+    },
+    "cm_handle_id_search_nofilter_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchNoFilterScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_search_nofilter_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchNoFilterScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_id_search_module_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchModuleScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_search_module_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchModuleScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_id_search_property_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchPropertyScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_search_property_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchPropertyScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_id_search_cpspath_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchCpsPathScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_search_cpspath_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchCpsPathScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_id_search_trustlevel_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleIdSearchTrustLevelScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "cm_handle_search_trustlevel_scenario": {
+      "executor": "constant-vus",
+      "exec": "cmHandleSearchTrustLevelScenario",
+      "vus": 1,
+      "duration": "15m"
+    },
+    "legacy_batch_produce_scenario": {
+      "executor": "shared-iterations",
+      "exec": "legacyBatchProduceScenario",
+      "vus": 2,
+      "iterations": 100
+    },
+    "legacy_batch_consume_scenario": {
+      "executor": "per-vu-iterations",
+      "exec": "legacyBatchConsumeScenario",
+      "vus": 1,
+      "iterations": 1
+    }
   },
-  "kafka": {
-    "legacyBatchTopic": "legacy_batch_topic"
+  "thresholds": {
+    "http_req_failed": ["rate == 0"],
+    "cmhandles_created_per_second": ["avg >= 22"],
+    "cmhandles_deleted_per_second": ["avg >= 22"],
+    "ncmp_overhead_passthrough_read": ["avg <= 40"],
+    "ncmp_overhead_passthrough_write": ["avg <= 40"],
+    "ncmp_overhead_passthrough_read_alt_id": ["avg <= 40"],
+    "ncmp_overhead_passthrough_write_alt_id": ["avg <= 40"],
+    "id_search_nofilter_duration": ["avg <= 2000"],
+    "id_search_module_duration": ["avg <= 2000"],
+    "id_search_property_duration": ["avg <= 2000"],
+    "id_search_cpspath_duration": ["avg <= 2000"],
+    "id_search_trustlevel_duration": ["avg <= 2000"],
+    "cm_search_nofilter_duration": ["avg <= 15000"],
+    "cm_search_module_duration": ["avg <= 15000"],
+    "cm_search_property_duration": ["avg <= 15000"],
+    "cm_search_cpspath_duration": ["avg <= 15000"],
+    "cm_search_trustlevel_duration": ["avg <= 15000"],
+    "legacy_batch_read_cmhandles_per_second": ["avg >= 150"]
   }
 }
\ No newline at end of file

diff --git a/k6-tests/ncmp/ncmp-kpi.js b/k6-tests/ncmp/ncmp-test-runner.js
similarity index 72%
rename from k6-tests/ncmp/ncmp-kpi.js
rename to k6-tests/ncmp/ncmp-test-runner.js
index 20fb1e8..e33ff18 100644 (file)
--- a/k6-tests/ncmp/ncmp-kpi.js
+++ b/k6-tests/ncmp/ncmp-test-runner.js
@@ -24,8 +24,8 @@ import { Reader } from 'k6/x/kafka';
 import {
     TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS,
     makeCustomSummaryReport, makeBatchOfCmHandleIds, LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE,
-    REGISTRATION_BATCH_SIZE, LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS, DURATION,
-    LEGACY_BATCH_THROUGHPUT_TEST_START_TIME, KAFKA_BOOTSTRAP_SERVERS, LEGACY_BATCH_TOPIC_NAME
+    REGISTRATION_BATCH_SIZE, LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS, KAFKA_BOOTSTRAP_SERVERS,
+    LEGACY_BATCH_TOPIC_NAME, testConfig
 } from './common/utils.js';
 import { createCmHandles, deleteCmHandles, waitForAllCmHandlesToBeReady } from './common/cmhandle-crud.js';
 import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
@@ -57,126 +57,8 @@ export const legacyBatchEventReader = new Reader({
 export const options = {
     setupTimeout: '20m',
     teardownTimeout: '20m',
-    scenarios: {
-        passthrough_read_scenario: {
-            executor: 'constant-vus',
-            exec: 'passthroughReadScenario',
-            vus: 2,
-            duration: DURATION,
-        },
-        passthrough_read_alt_id_scenario: {
-            executor: 'constant-vus',
-            exec: 'passthroughReadAltIdScenario',
-            vus: 2,
-            duration: DURATION,
-        },
-        passthrough_write_scenario: {
-            executor: 'constant-vus',
-            exec: 'passthroughWriteScenario',
-            vus: 2,
-            duration: DURATION,
-        },
-        passthrough_write_alt_id_scenario: {
-            executor: 'constant-vus',
-            exec: 'passthroughWriteAltIdScenario',
-            vus: 2,
-            duration: DURATION,
-        },
-        cm_handle_id_search_nofilter_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleIdSearchNoFilterScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_search_nofilter_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleSearchNoFilterScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_id_search_module_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleIdSearchModuleScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_search_module_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleSearchModuleScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_id_search_property_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleIdSearchPropertyScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_search_property_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleSearchPropertyScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_id_search_cpspath_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleIdSearchCpsPathScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_search_cpspath_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleSearchCpsPathScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_id_search_trustlevel_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleIdSearchTrustLevelScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        cm_handle_search_trustlevel_scenario: {
-            executor: 'constant-vus',
-            exec: 'cmHandleSearchTrustLevelScenario',
-            vus: 1,
-            duration: DURATION,
-        },
-        legacy_batch_produce_scenario: {
-            executor: 'shared-iterations',
-            exec: 'legacyBatchProduceScenario',
-            vus: 2,
-            iterations: LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS,
-            startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME,
-        },
-        legacy_batch_consume_scenario: {
-            executor: 'per-vu-iterations',
-            exec: 'legacyBatchConsumeScenario',
-            vus: 1,
-            iterations: 1,
-            startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME,
-        }
-    },
-    thresholds: {
-        'http_req_failed': ['rate == 0'],
-        'cmhandles_created_per_second': ['avg >= 22'],
-        'cmhandles_deleted_per_second': ['avg >= 22'],
-        'ncmp_overhead_passthrough_read': ['avg <= 40'],
-        'ncmp_overhead_passthrough_write': ['avg <= 40'],
-        'ncmp_overhead_passthrough_read_alt_id': ['avg <= 40'],
-        'ncmp_overhead_passthrough_write_alt_id': ['avg <= 40'],
-        'id_search_nofilter_duration': ['avg <= 2000'],
-        'id_search_module_duration': ['avg <= 2000'],
-        'id_search_property_duration': ['avg <= 2000'],
-        'id_search_cpspath_duration': ['avg <= 2000'],
-        'id_search_trustlevel_duration': ['avg <= 2000'],
-        'cm_search_nofilter_duration': ['avg <= 15000'],
-        'cm_search_module_duration': ['avg <= 15000'],
-        'cm_search_property_duration': ['avg <= 15000'],
-        'cm_search_cpspath_duration': ['avg <= 15000'],
-        'cm_search_trustlevel_duration': ['avg <= 15000'],
-        'legacy_batch_read_cmhandles_per_second': ['avg >= 150'],
-    },
+    scenarios: testConfig.scenarios,
+    thresholds: testConfig.thresholds,
 };
 
 export function setup() {

diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index 3457903..3769570 100755 (executable)
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -20,9 +20,9 @@ pushd "$(dirname "$0")" >/dev/null || exit 1
 number_of_failures=0
 testProfile=$1
 summaryFile="${testProfile}Summary.csv"
-
 echo "Running $testProfile performance tests..."
-k6 run ncmp-kpi.js --quiet -e TEST_PROFILE="$testProfile"  > "$summaryFile" 2>/dev/null || ((number_of_failures++))
+
+k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile"  > "$summaryFile" 2>/dev/null || ((number_of_failures++))
 
 if [ -f "$summaryFile" ]; then