Adjust K6 thresholds

- Adjust timings based on previous runs.
- Reduce from 5 to 4 parallel clients for CM handle searches
  to avoid OutOfMemoryErrors.

Issue-ID: CPS-2208
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: I339cc3df84b2295bb11371ab73b4948f43615c15

diff --git a/k6-tests/ncmp/1-create-cmhandles.js b/k6-tests/ncmp/1-create-cmhandles.js
index a6f8086..60594c7 100644 (file)
--- a/k6-tests/ncmp/1-create-cmhandles.js
+++ b/k6-tests/ncmp/1-create-cmhandles.js
@@ -35,7 +35,7 @@ export const options = {
     iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 1000'],
+        http_req_duration: ['avg <= 850'],
     },
 };
 
@@ -65,4 +65,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/10-mixed-load-test.js
index b2c20e1..829219b 100644 (file)
--- a/k6-tests/ncmp/10-mixed-load-test.js
+++ b/k6-tests/ncmp/10-mixed-load-test.js
@@ -40,7 +40,7 @@ export const options = {
         cm_search_module: {
             executor: 'constant-vus',
             exec: 'cm_search_module',
-            vus: 5,
+            vus: 4,
             duration: '1m',
         },
     },
@@ -49,9 +49,9 @@ export const options = {
         'http_req_failed{scenario:passthrough_read}': ['rate == 0'],
         'http_req_failed{scenario:id_search_module}': ['rate == 0'],
         'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
-        'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms
-        'http_req_duration{scenario:id_search_module}': ['avg <= 200'],
-        'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'],
+        'http_req_duration{scenario:passthrough_read}': ['avg <= 2600'], // DMI delay + 100 ms
+        'http_req_duration{scenario:id_search_module}': ['avg <= 500'],
+        'http_req_duration{scenario:cm_search_module}': ['avg <= 30000'],
     },
 };
 

diff --git a/k6-tests/ncmp/11-delete-cmhandles.js b/k6-tests/ncmp/11-delete-cmhandles.js
index dabf12c..073d1d0 100644 (file)
--- a/k6-tests/ncmp/11-delete-cmhandles.js
+++ b/k6-tests/ncmp/11-delete-cmhandles.js
@@ -29,7 +29,7 @@ export const options = {
     iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 1000'],
+        http_req_duration: ['avg <= 1050'],
     },
 };
 
@@ -51,4 +51,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
index 4ecadde..5d54c60 100644 (file)
--- a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
+++ b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
@@ -27,7 +27,7 @@ export const options = {
     iterations: 1,
     thresholds: {
         http_req_failed: ['rate == 0'],
-        iteration_duration: ['max <= 300_000'], // 5 minutes
+        iteration_duration: ['max <= 260000'], // 4m20s
     },
 };
 
@@ -67,4 +67,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/3-passthrough-read.js b/k6-tests/ncmp/3-passthrough-read.js
index 0b8c58a..84050b8 100644 (file)
--- a/k6-tests/ncmp/3-passthrough-read.js
+++ b/k6-tests/ncmp/3-passthrough-read.js
@@ -30,7 +30,7 @@ export const options = {
     duration: '30s',
     thresholds: {
         http_req_failed: ['rate == 0'],
-        ncmp_overhead: ['avg <= 40'],
+        ncmp_overhead: ['avg <= 50'],
     },
 };
 
@@ -54,4 +54,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/4-id-search-no-filter.js b/k6-tests/ncmp/4-id-search-no-filter.js
index a99f8b5..1c96665 100644 (file)
--- a/k6-tests/ncmp/4-id-search-no-filter.js
+++ b/k6-tests/ncmp/4-id-search-no-filter.js
@@ -23,10 +23,10 @@ import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
-    duration: '60s',
+    duration: '30s',
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 800'],
+        http_req_duration: ['avg <= 700'],
     },
 };
 
@@ -39,4 +39,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/5-search-no-filter.js b/k6-tests/ncmp/5-search-no-filter.js
index f6a7c99..4ef8a57 100644 (file)
--- a/k6-tests/ncmp/5-search-no-filter.js
+++ b/k6-tests/ncmp/5-search-no-filter.js
@@ -22,11 +22,11 @@ import { searchRequest } from './search-base.js';
 import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
-    vus: 5,
+    vus: 4,
     duration: '60s',
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 35_000'],
+        http_req_duration: ['avg <= 20000'],
     },
 };
 
@@ -38,4 +38,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/6-id-search-public-property.js b/k6-tests/ncmp/6-id-search-public-property.js
index 0a06331..0c5fcc6 100644 (file)
--- a/k6-tests/ncmp/6-id-search-public-property.js
+++ b/k6-tests/ncmp/6-id-search-public-property.js
@@ -26,7 +26,7 @@ export const options = {
     duration: '30s',
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 5000'],
+        http_req_duration: ['avg <= 4500'],
     },
 };
 
@@ -46,4 +46,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/7-search-public-property.js b/k6-tests/ncmp/7-search-public-property.js
index 9a8e339..6a46a03 100644 (file)
--- a/k6-tests/ncmp/7-search-public-property.js
+++ b/k6-tests/ncmp/7-search-public-property.js
@@ -22,11 +22,11 @@ import { searchRequest } from './search-base.js';
 import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
-    vus: 5,
-    duration: '30s',
+    vus: 4,
+    duration: '60s',
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 35_000'],
+        http_req_duration: ['avg <= 25000'],
     },
 };
 
@@ -46,4 +46,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/8-id-search-module.js b/k6-tests/ncmp/8-id-search-module.js
index 299fe62..4c9e03e 100644 (file)
--- a/k6-tests/ncmp/8-id-search-module.js
+++ b/k6-tests/ncmp/8-id-search-module.js
@@ -46,4 +46,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}

diff --git a/k6-tests/ncmp/9-search-module.js b/k6-tests/ncmp/9-search-module.js
index d340bf2..3237e6a 100644 (file)
--- a/k6-tests/ncmp/9-search-module.js
+++ b/k6-tests/ncmp/9-search-module.js
@@ -22,11 +22,11 @@ import { searchRequest } from './search-base.js';
 import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
-    vus: 5,
+    vus: 4,
     duration: '60s',
     thresholds: {
         http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 35_000'],
+        http_req_duration: ['avg <= 20000'],
     },
 };
 
@@ -46,4 +46,4 @@ export function handleSummary(data) {
     return {
         stdout: makeCustomSummaryReport(data, options),
     };
-}
\ No newline at end of file
+}