[k6] Use constant arrival rate

Currently, k6 KPI tests send requests using 'constant-vus' executor.
This continuously sends requests on different threads - as soon as
one request completes, another is sent. This leads to weird results
where improving performance of one operation ends up degrading other
test cases, due to more requests being sent per second.

This change makes KPI tests send requests at a constant rate,
making the mixed-load tests less prone to such anomalies.

- Send requests at constant rates for test cases 3, 4, 5, 6.
- Throughput tests 1, 2 & 7 still send at the fastest rate.
- Adjust expected performance timings.

Issue-ID: CPS-2519
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: I641ead7d2d9d3b29116e1cf0d4bf8cd345b01a98

diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index b937b4b..589a248 100644 (file)
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -89,21 +89,21 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) {
         makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig),
         makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 50, testResults, scenarioConfig),
         makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 100, testResults, scenarioConfig),
-        makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 500, testResults, scenarioConfig),
-        makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 500, testResults, scenarioConfig),
-        makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1200, testResults, scenarioConfig),
-        makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1200, testResults, scenarioConfig),
-        makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 4200, testResults, scenarioConfig),
-        makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 8000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 10000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 10000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 10000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 13000, testResults, scenarioConfig),
-        makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 30, testResults, scenarioConfig),
-        makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 60, testResults, scenarioConfig),
-        makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 30, testResults, scenarioConfig),
-        makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 60, testResults, scenarioConfig),
-        makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1500, testResults, scenarioConfig),
+        makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig),
+        makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 300, testResults, scenarioConfig),
+        makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig),
+        makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig),
+        makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 3000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 4000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 4500, testResults, scenarioConfig),
+        makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 4500, testResults, scenarioConfig),
+        makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 7000, testResults, scenarioConfig),
+        makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 20, testResults, scenarioConfig),
+        makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 40, testResults, scenarioConfig),
+        makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 20, testResults, scenarioConfig),
+        makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 40, testResults, scenarioConfig),
+        makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 300, testResults, scenarioConfig),
     ];
     return summaryCsvLines.join('\n') + '\n';
 }

diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json
index 742321f..2318702 100644 (file)
--- a/k6-tests/ncmp/config/kpi.json
+++ b/k6-tests/ncmp/config/kpi.json
@@ -6,89 +6,135 @@
   },
   "scenarios": {
     "passthrough_read_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughReadScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "0ms"
     },
     "passthrough_read_alt_id_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughReadAltIdScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "200ms"
     },
+
     "passthrough_write_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughWriteScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "400ms"
     },
     "passthrough_write_alt_id_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughWriteAltIdScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "600ms"
     },
+
     "cm_handle_id_search_nofilter_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchNoFilterScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_nofilter_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchNoFilterScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "0ms"
     },
     "cm_handle_id_search_module_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchModuleScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_module_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchModuleScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "400ms"
     },
     "cm_handle_id_search_property_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchPropertyScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_property_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchPropertyScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "800ms"
     },
     "cm_handle_id_search_cpspath_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchCpsPathScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_cpspath_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchCpsPathScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "1200ms"
     },
     "cm_handle_id_search_trustlevel_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchTrustLevelScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "1600ms"
+    },
+
+    "cm_handle_search_nofilter_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchNoFilterScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "0s"
+    },
+    "cm_handle_search_module_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchModuleScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "3s"
+    },
+    "cm_handle_search_property_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchPropertyScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "6s"
+    },
+    "cm_handle_search_cpspath_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchCpsPathScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "9s"
     },
     "cm_handle_search_trustlevel_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchTrustLevelScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "12s"
     },
+
     "legacy_batch_produce_scenario": {
       "executor": "shared-iterations",
       "exec": "legacyBatchProduceScenario",
@@ -122,4 +168,4 @@
     "cm_search_trustlevel_duration": ["avg <= 15000"],
     "legacy_batch_read_cmhandles_per_second": ["avg >= 150"]
   }
-}
\ No newline at end of file
+}