From 59233aab1cce1de03ea8b69fc2e313ab53a88d37 Mon Sep 17 00:00:00 2001 From: "halil.cakal" Date: Fri, 19 Sep 2025 16:29:07 +0100 Subject: [PATCH] Cleanup k6 test templatters (Split-2) - the current template files make the development of k6 test complex and difficult to manage unnecessarily, need to be removed - so that new ports and variables for the kubernetes cluster can be provieded easily for Split-3 - simplify the algorith to generate a human readeable summary report - remove internal nodePort that is a remained action from commit (Split-1) that already merged - total lines of code removed approx: 600 Issue-ID: CPS-2967 Change-Id: I971c58814890a4b65a283bb09411122d0647fd31 Signed-off-by: halil.cakal --- .gitignore | 6 +- cps-charts/templates/kafka-external-service.yaml | 1 - cps-charts/values.yaml | 2 +- k6-tests/ncmp/common/utils.js | 43 ++-- ...io-execution-definition.json => endurance.json} | 2 +- .../kpi.json} | 23 +- k6-tests/ncmp/config/scenario-metadata.json | 146 ------------- .../ncmp/create-scenario-execution-definition.sh | 106 --------- k6-tests/ncmp/create-scenario-javascript.sh | 115 ---------- k6-tests/ncmp/execute-k6-scenarios.sh | 236 ++++++--------------- ...cenario-javascript.tmpl => ncmp-test-runner.js} | 24 ++- 11 files changed, 133 insertions(+), 571 deletions(-) rename k6-tests/ncmp/config/{endurance-scenario-execution-definition.json => endurance.json} (99%) rename k6-tests/ncmp/{templates/scenario-execution-definition.tmpl => config/kpi.json} (83%) delete mode 100644 k6-tests/ncmp/config/scenario-metadata.json delete mode 100644 k6-tests/ncmp/create-scenario-execution-definition.sh delete mode 100644 k6-tests/ncmp/create-scenario-javascript.sh rename k6-tests/ncmp/{templates/scenario-javascript.tmpl => ncmp-test-runner.js} (85%) diff --git a/.gitignore b/.gitignore index 58cdadfad1..fafb75e4f1 100755 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,5 @@ tmp/ csit/env.properties csit/archives/ -/k6-tests/image/ -k6-tests/ncmp/config/kpi-scenario-execution-definition.json -/k6-tests/ncmp/*Summary.csv -/k6-tests/ncmp/scenario-javascript.js \ No newline at end of file +/k6-tests/logs/ +/k6-tests/image/ \ No newline at end of file diff --git a/cps-charts/templates/kafka-external-service.yaml b/cps-charts/templates/kafka-external-service.yaml index 6d819b4b8e..906c9968e9 100644 --- a/cps-charts/templates/kafka-external-service.yaml +++ b/cps-charts/templates/kafka-external-service.yaml @@ -11,7 +11,6 @@ spec: - name: internal port: {{ .Values.kafka.externalService.ports.internal }} targetPort: {{ .Values.kafka.externalService.ports.internal }} - nodePort: {{ .Values.kafka.externalService.nodePorts.internal }} - name: external port: {{ .Values.kafka.externalService.ports.external }} targetPort: {{ .Values.kafka.externalService.ports.external }} diff --git a/cps-charts/values.yaml b/cps-charts/values.yaml index 1a43f000ca..6fdeacae84 100644 --- a/cps-charts/values.yaml +++ b/cps-charts/values.yaml @@ -66,7 +66,7 @@ kafka: internal: 9092 external: 9093 nodePorts: - external: 30093 # Node port for external communication + external: 30093 zookeeperConnect: "cps-cps-and-ncmp-zookeeper.default.svc.cluster.local" brokerId: 1 listeners: "INTERNAL://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092,NODEPORT://0.0.0.0:9093" diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index fd24680a4c..8f1f2063e0 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -24,8 +24,7 @@ import {check} from 'k6'; import {Trend} from 'k6/metrics'; export const TEST_PROFILE = __ENV.TEST_PROFILE ? __ENV.TEST_PROFILE : 'kpi' -export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}-scenario-execution-definition.json`)); -export const testKpiMetaData = JSON.parse(open(`../config/scenario-metadata.json`)); +export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}.json`)); export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer; export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl; export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl; @@ -120,28 +119,36 @@ export function performGetRequest(url, metricTag) { return http.get(url, {tags: metricTags}); } -export function makeCustomSummaryReport(metrics, thresholds) { +export function makeCustomSummaryReport(testResults, scenarioConfig) { const summaryCsvLines = [ '#,Test Name,Unit,Fs Requirement,Current Expectation,Actual', - ...testKpiMetaData.map(test => { - return makeSummaryCsvLine( - test.label, - test.name, - test.unit, - test.metric, - test.cpsAverage, - metrics, - thresholds - ); - }) + makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), + makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cm_handles_created', 100, testResults, scenarioConfig), + makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cm_handles_deleted', 180, testResults, scenarioConfig), + makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'cm_handle_id_search_no_filter', 550, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'cm_handle_id_search_module_filter', 2300, testResults, scenarioConfig), + makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'cm_handle_id_search_property_filter', 1450, testResults, scenarioConfig), + makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'cm_handle_id_search_cps_path_filter', 1500, testResults, scenarioConfig), + makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'cm_handle_id_search_trust_level_filter', 1600, testResults, scenarioConfig), + makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_handle_search_no_filter', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_handle_search_module_filter', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_handle_search_property_filter', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_handle_search_cps_path_filter', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_handle_search_trust_level_filter', 18000, testResults, scenarioConfig), + makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_read_overhead', 18, testResults, scenarioConfig), + makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_write_overhead', 18, testResults, scenarioConfig), + makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read', 200, testResults, scenarioConfig), + makeSummaryCsvLine('8', 'Write data job scenario - small', 'milliseconds', 'dcm_write_data_job_small', 100, testResults, scenarioConfig), + makeSummaryCsvLine('9', 'Write data job scenario - large', 'milliseconds', 'dcm_write_data_job_large', 8000, testResults, scenarioConfig), ]; return summaryCsvLines.join('\n') + '\n'; } -function makeSummaryCsvLine(testNumber, testName, unit, measurementName, currentExpectation, metrics, thresholds) { - const thresholdCondition = JSON.parse(JSON.stringify(thresholds[measurementName]))[0]; - const [metricsFunction, thresholdOperator, thresholdValue] = thresholdCondition.split(/\s+/); - const actualValue = metrics[measurementName].values[metricsFunction].toFixed(3); +function makeSummaryCsvLine(testNumber, testName, unit, measurementName, currentExpectation, testResults, scenarioConfig) { + const thresholdArray = JSON.parse(JSON.stringify(scenarioConfig.thresholds[measurementName])); + const thresholdString = thresholdArray[0]; + const [thresholdKey, thresholdOperator, thresholdValue] = thresholdString.split(/\s+/); + const actualValue = testResults.metrics[measurementName].values[thresholdKey].toFixed(3); return `${testNumber},${testName},${unit},${thresholdValue},${currentExpectation},${actualValue}`; } diff --git a/k6-tests/ncmp/config/endurance-scenario-execution-definition.json b/k6-tests/ncmp/config/endurance.json similarity index 99% rename from k6-tests/ncmp/config/endurance-scenario-execution-definition.json rename to k6-tests/ncmp/config/endurance.json index 22f502e857..d090603825 100644 --- a/k6-tests/ncmp/config/endurance-scenario-execution-definition.json +++ b/k6-tests/ncmp/config/endurance.json @@ -99,4 +99,4 @@ "duration": "2h" } } -} +} \ No newline at end of file diff --git a/k6-tests/ncmp/templates/scenario-execution-definition.tmpl b/k6-tests/ncmp/config/kpi.json similarity index 83% rename from k6-tests/ncmp/templates/scenario-execution-definition.tmpl rename to k6-tests/ncmp/config/kpi.json index 8eb5415248..2339f4072c 100644 --- a/k6-tests/ncmp/templates/scenario-execution-definition.tmpl +++ b/k6-tests/ncmp/config/kpi.json @@ -169,5 +169,24 @@ "startTime": "923ms" } }, - "thresholds": "#THRESHOLDS-PLACEHOLDER#" -} + "thresholds": { + "http_req_failed": ["rate <= 0.01"], + "cm_handles_created": ["avg >= 22"], + "cm_handles_deleted": ["avg >= 22"], + "cm_handle_id_search_no_filter": ["avg <= 2600"], + "cm_handle_id_search_module_filter": ["avg <= 2600"], + "cm_handle_id_search_property_filter": ["avg <= 2600"], + "cm_handle_id_search_cps_path_filter": ["avg <= 2600"], + "cm_handle_id_search_trust_level_filter": ["avg <= 2600"], + "cm_handle_search_no_filter": ["avg <= 24000"], + "cm_handle_search_module_filter": ["avg <= 24000"], + "cm_handle_search_property_filter": ["avg <= 24000"], + "cm_handle_search_cps_path_filter": ["avg <= 24000"], + "cm_handle_search_trust_level_filter": ["avg <= 24000"], + "ncmp_read_overhead": ["avg <= 40"], + "ncmp_write_overhead": ["avg <= 40"], + "legacy_batch_read": ["avg >= 150"], + "dcm_write_data_job_small": ["avg <= 500"], + "dcm_write_data_job_large": ["avg <= 30000"] + } +} \ No newline at end of file diff --git a/k6-tests/ncmp/config/scenario-metadata.json b/k6-tests/ncmp/config/scenario-metadata.json deleted file mode 100644 index cfbc6ea0b4..0000000000 --- a/k6-tests/ncmp/config/scenario-metadata.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "label": "0", - "name": "HTTP request failures for all tests", - "unit": "rate of failed requests", - "metric": "http_req_failed", - "cpsAverage": 0.00, - "kpiThreshold": 0.01 - }, - { - "label": "1", - "name": "Registration of CM-handles", - "unit": "CM-handles/second", - "metric": "cm_handles_created", - "cpsAverage": 100, - "kpiThreshold": 22 - }, - { - "label": "2", - "name": "De-registration of CM-handles", - "unit": "CM-handles/second", - "metric": "cm_handles_deleted", - "cpsAverage": 180, - "kpiThreshold": 22 - }, - { - "label": "3a", - "name": "CM-handle ID search with No filter", - "unit": "milliseconds", - "metric": "cm_handle_id_search_no_filter", - "cpsAverage": 550, - "kpiThreshold": 2600 - }, - { - "label": "3b", - "name": "CM-handle ID search with Module filter", - "unit": "milliseconds", - "metric": "cm_handle_id_search_module_filter", - "cpsAverage": 2300, - "kpiThreshold": 2600 - }, - { - "label": "3c", - "name": "CM-handle ID search with Property filter", - "unit": "milliseconds", - "metric": "cm_handle_id_search_property_filter", - "cpsAverage": 1450, - "kpiThreshold": 2600 - }, - { - "label": "3d", - "name": "CM-handle ID search with Cps Path filter", - "unit": "milliseconds", - "metric": "cm_handle_id_search_cps_path_filter", - "cpsAverage": 1500, - "kpiThreshold": 2600 - }, - { - "label": "3e", - "name": "CM-handle ID search with Trust Level filter", - "unit": "milliseconds", - "metric": "cm_handle_id_search_trust_level_filter", - "cpsAverage": 1600, - "kpiThreshold": 2600 - }, - { - "label": "4a", - "name": "CM-handle search with No filter", - "unit": "milliseconds", - "metric": "cm_handle_search_no_filter", - "cpsAverage": 18000, - "kpiThreshold": 24000 - }, - { - "label": "4b", - "name": "CM-handle search with Module filter", - "unit": "milliseconds", - "metric": "cm_handle_search_module_filter", - "cpsAverage": 18000, - "kpiThreshold": 24000 - }, - { - "label": "4c", - "name": "CM-handle search with Property filter", - "unit": "milliseconds", - "metric": "cm_handle_search_property_filter", - "cpsAverage": 18000, - "kpiThreshold": 24000 - }, - { - "label": "4d", - "name": "CM-handle search with Cps Path filter", - "unit": "milliseconds", - "metric": "cm_handle_search_cps_path_filter", - "cpsAverage": 18000, - "kpiThreshold": 24000 - }, - { - "label": "4e", - "name": "CM-handle search with Trust Level filter", - "unit": "milliseconds", - "metric": "cm_handle_search_trust_level_filter", - "cpsAverage": 18000, - "kpiThreshold": 24000 - }, - { - "label": "5b", - "name": "NCMP overhead for Synchronous single CM-handle pass-through read with alternate id", - "unit": "milliseconds", - "metric": "ncmp_read_overhead", - "cpsAverage": 18, - "kpiThreshold": 40 - }, - { - "label": "6b", - "name": "NCMP overhead for Synchronous single CM-handle pass-through write with alternate id", - "unit": "milliseconds", - "metric": "ncmp_write_overhead", - "cpsAverage": 18, - "kpiThreshold": 40 - }, - { - "label": "7", - "name": "Legacy batch read operation", - "unit": "events/second", - "metric": "legacy_batch_read", - "cpsAverage": 200, - "kpiThreshold": 150 - }, - { - "label": "8", - "name": "Write data job scenario - small", - "unit": "milliseconds", - "metric": "dcm_write_data_job_small", - "cpsAverage": 100, - "kpiThreshold": 500 - }, - { - "label": "9", - "name": "Write data job scenario - large", - "unit": "milliseconds", - "metric": "dcm_write_data_job_large", - "cpsAverage": 8000, - "kpiThreshold": 30000 - } -] diff --git a/k6-tests/ncmp/create-scenario-execution-definition.sh b/k6-tests/ncmp/create-scenario-execution-definition.sh deleted file mode 100644 index ebe8d75f88..0000000000 --- a/k6-tests/ncmp/create-scenario-execution-definition.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# -# Copyright 2025 OpenInfra Foundation Europe. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script is used to generate K6 performance testing 'thresholds' declarations file by: -# 1) Extracting thresholds from metric metadata JSON (scenario-metadata.json) -# 2) Injects them into a scenario execution template file (scenario-execution-definition.tmpl) and later writes -# it into config file named as {performance-test-profile-name}-scenario-execution-definition.json. -# Note: {performance-test-profile-name} : There are two test profiles that can be run with either: kpi or endurance. -# - -# Path to the JSON file containing metric metadata. -# This JSON holds metric names, units, and threshold values. -SCENARIO_METADATA_FILE="./config/scenario-metadata.json" - -# Scenario JSON template file for scenario execution configuration. -# Contains placeholder (#THRESHOLDS-PLACEHOLDER#) to be replaced with actual threshold values. -SCENARIO_CONFIG_TEMPLATE_FILE="./templates/scenario-execution-definition.tmpl" - -# Final json scenario execution configuration file with thresholds injected. -SCENARIO_CONFIG_OUTPUT_FILE="./config/kpi-scenario-execution-definition.json" - -# ───────────────────────────────────────────────────────────── -# Function: create_thresholds -# Description: -# Prepares threshold expressions for each metric. -# Parameters: -# $1 - Path to the metric metadata JSON file. (scenario-metadata.json) -# Returns: -# JSON object array containing thresholds for each metric. -# ───────────────────────────────────────────────────────────── -create_thresholds() { - local scenario_metadata_json_file="$1" - - # Define the jq script to build the thresholds JSON object - read -r -d '' thresholds_per_metric_as_json << 'EOF' - # Set threshold expression based on metric type. - reduce .[] as $metric ( - {}; - .[$metric.metric] = ( - if $metric.metric == "http_req_failed" then - ["rate <= \($metric.kpiThreshold)"] # For failure rate metric, threshold is rate <= value - elif ($metric.unit | test("/second")) then - ["avg >= \($metric.kpiThreshold)"] # For per-second metrics, expect average >= threshold - else - ["avg <= \($metric.kpiThreshold)"] # Otherwise, average <= threshold - end - ) - ) -EOF - - # This returns a JSON object with: - # - 'thresholds': array of JS declaration strings - jq -r "$thresholds_per_metric_as_json" "$scenario_metadata_json_file" -} - -# ───────────────────────────────────────────────────────────── -# Function: inject_thresholds_into_scenario-execution -# Description: -# Injects the extracted threshold JSON object into the scenario -# configuration template by replacing the `.thresholds` named property. -# Parameters: -# $1 - JSON string of threshold mappings. (scenario-metadata.json) -# $2 - Template scenario config file path. (scenario-execution-definition.tmpl) -# $3 - Output scenario config file path (kpi-scenario-execution-definition.json) -# Returns: -# Writes the updated JSON to output file. (kpi-scenario-execution-definition.json) -# ───────────────────────────────────────────────────────────── -inject_thresholds_into_scenario_execution_config() { - local thresholds_json="$1" - local scenario_execution_template_file="$2" - local scenario_execution_output_file="$3" - - # Use jq to overwrite the `.thresholds` property in the template with the generated thresholds JSON - jq --argjson thresholds "$thresholds_json" '.thresholds = $thresholds' "$scenario_execution_template_file" | jq '.' > "$scenario_execution_output_file" -} - -# ───────────────────────────────────────────────────────────── -# Main script execution starts here -# ───────────────────────────────────────────────────────────── - -# Inform user script is starting threshold generation -echo "Generating thresholds from [$SCENARIO_METADATA_FILE]..." - -# Calling function to extract threshold JSON object from metric metadata JSON file -scenario_execution_thresholds_json=$(create_thresholds "$SCENARIO_METADATA_FILE") - -# Inject the extracted thresholds json block into the scenario config template and write into output file -inject_thresholds_into_scenario_execution_config "$scenario_execution_thresholds_json" "$SCENARIO_CONFIG_TEMPLATE_FILE" "$SCENARIO_CONFIG_OUTPUT_FILE" - -# Final confirmation message on successful injection -echo "Threshold block has been injected into [$SCENARIO_CONFIG_OUTPUT_FILE]" \ No newline at end of file diff --git a/k6-tests/ncmp/create-scenario-javascript.sh b/k6-tests/ncmp/create-scenario-javascript.sh deleted file mode 100644 index 25df997172..0000000000 --- a/k6-tests/ncmp/create-scenario-javascript.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# -# Copyright 2025 OpenInfra Foundation Europe. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script is used to generate K6 performance testing 'Trend' declarations file by: -# 1) Extracting trends from metric metadata JSON (scenario-metadata.json) -# 2) Injects them into a JavaScript template file (scenario-javascript.tmpl) and then write it into final JavaScript -# file named as scenario-javascript.json. -# - -# Path to the JSON file that contains metric definitions (name, unit, threshold, etc.) -# This JSON holds metric names, units, and threshold values. -SCENARIO_METADATA_FILE="./config/scenario-metadata.json" - -# Path to the JS template file where the placeholder `#METRICS-TRENDS-PLACEHOLDER#` exists. -# This is where the generated trend declarations will be inserted. -SCENARIO_JAVASCRIPT_TEMPLATE_FILE="./templates/scenario-javascript.tmpl" - -# Output JavaScript file where the final result (with inserted trend declarations) will be saved. -SCENARIO_JAVASCRIPT_OUTPUT_FILE="scenario-javascript.js" - -# ───────────────────────────────────────────────────────────────────────────────── -# Function: create_trend_declarations -# Description: -# Converts metrics from the metadata JSON into JavaScript `Trend` declarations. -# These declarations are used for K6 performance testing reports. -# Parameters: -# $1 - Accept the path to the metric metadata file as input. (scenario-metadata.json) -# Returns: -# trend declarations as JSON array object -# ───────────────────────────────────────────────────────────────────────────────── - -create_trend_declarations() { - local scenario_metadata_json_file="$1" - - # Read and assign a JQ script to a here-document variable. (trend_declarations) - # `-r` makes jq output raw strings and `-d ''` lets us use multiline input. - read -r -d '' trend_declarations << 'EOF' - # Define a helper function (toCamelCase) that converts metric names from snake_case to camelCase - # (for JS variable names for example: cm_handles_deleted → cmHandlesDeleted) - def toCamelCase: - split("_") as $parts | # Split string by "_" - ($parts[0]) + # Keep first part as it is - ($parts[1:] | map((.[0:1] | ascii_upcase) + .[1:]) # Capitalize rest of each word - | join("")); # Join all parts into one string - - # Loop through each metric item and generate a JavaScript `Trend` declaration if unit matches. - .[] # Iterate through array - | select((.unit == "milliseconds") or (.unit | test("/second"))) # Select based on valid units - | "export let \(.metric | toCamelCase)Trend = new Trend('\(.metric)', \(.unit == "milliseconds"));" - # Output javascript declaration string: `export let abcTrend = new Trend('abc', true/false);` -EOF - # Execute the jq script on the metadata file to generate the trend declarations - jq -r "$trend_declarations" "$scenario_metadata_json_file" -} - -# ───────────────────────────────────────────────────────────── -# Function: inject_trends_into_js_template -# Description: -# Replaces the placeholder line `#METRICS-TRENDS-PLACEHOLDER#` in the template -# file with actual JS trend declarations. -# Parameters: -# $1 - JSON string of threshold mappings. Trend declaration strings. -# for example: export let abcTrend = new Trend('abc', true), from scenario-metadata.json) -# $2 - Template scenario javascript file path. (scenario-javascript.tmpl) -# $3 - Output scenario script file path (scenario-javascript.js) -# Returns: -# Writes the updated JSON to output file. (scenario-javascript.js) -# ───────────────────────────────────────────────────────────── -inject_trends_into_javascript_template() { - local trend_declarations="$1" - local scenario_javascript_template_file="$2" - local scenario_javascript_output_file="$3" - - # Use awk to replace the placeholder line with trend declarations - awk -v trends="$trend_declarations" ' # Pass trends into awk variable - { - if ($0 ~ /#METRICS-TRENDS-PLACEHOLDER#/) { - print trends # Print the trend declarations instead of the placeholder - } else { - print $0 # Otherwise, print the original line - } - } - ' "$scenario_javascript_template_file" > "$scenario_javascript_output_file" # Save the transformed content into the output JS file -} - -# ───────────────────────────────────────────────────────────── -# Main Execution Starts Here -# ───────────────────────────────────────────────────────────── - -# Display log message to inform that generation has started -echo "Generating trend declarations from [$SCENARIO_METADATA_FILE]..." - -# Calling trend generation function -scenario_javascript_trend_declarations=$(create_trend_declarations "$SCENARIO_METADATA_FILE") - -# Inject the generated trends into the JavaScript template and write it into scenario output file -inject_trends_into_javascript_template "$scenario_javascript_trend_declarations" "$SCENARIO_JAVASCRIPT_TEMPLATE_FILE" "$SCENARIO_JAVASCRIPT_OUTPUT_FILE" - -# Final confirmation message to indicate success -echo "Trend declarations inserted into [$SCENARIO_JAVASCRIPT_OUTPUT_FILE]" \ No newline at end of file diff --git a/k6-tests/ncmp/execute-k6-scenarios.sh b/k6-tests/ncmp/execute-k6-scenarios.sh index a25e1e793d..3bb420f7a9 100755 --- a/k6-tests/ncmp/execute-k6-scenarios.sh +++ b/k6-tests/ncmp/execute-k6-scenarios.sh @@ -26,178 +26,85 @@ pushd "$(dirname "$0")" >/dev/null || { # ───────────────────────────────────────────────────────────── # 📌 Global Variables # ───────────────────────────────────────────────────────────── -number_of_failures=0 +threshold_failures=0 testProfile=$1 summaryFile="${testProfile}Summary.csv" -# Path to the JSON file containing metric metadata. -# This JSON holds metric names, units, and threshold values. -SCENARIO_METADATA_FILE="./config/scenario-metadata.json" - -echo -echo "📢 Running NCMP K6 performance test for profile: [$testProfile]" -echo - -chmod +x ./create-scenario-javascript.sh -source ./create-scenario-javascript.sh - -if [[ "$testProfile" == "kpi" ]]; then - chmod +x ./create-scenario-execution-definition.sh - source ./create-scenario-execution-definition.sh -fi +echo "Running $testProfile performance tests..." # ───────────────────────────────────────────────────────────── # Run K6 and Capture Output +# '$?' is immediately captures the exit code after k6 finishes, +# and assign it to k6_exit_code. # ───────────────────────────────────────────────────────────── -k6 run scenario-javascript.js -e TEST_PROFILE="$testProfile" > "$summaryFile" +k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile" > "$summaryFile" k6_exit_code=$? case $k6_exit_code in - 0) echo "✅ K6 executed successfully for profile: [$testProfile]." ;; - 99) echo "⚠️ K6 thresholds failed (exit code 99). Processing failures..." ;; - *) echo "❌ K6 execution error (exit code $k6_exit_code)."; number_of_failures=$((number_of_failures + 1)) ;; + 0) echo "✅ K6 executed successfully for profile: [$testProfile]" ;; + 99) echo "⚠️ K6 thresholds failed (exit code 99)" ;; + *) echo "❌ K6 execution error (exit code $k6_exit_code)";; esac -if [[ "$testProfile" == "kpi" ]]; then -# ───────────────────────────────────────────────────────────── -# Extract and Filter Summary Data -# ───────────────────────────────────────────────────────────── -if [ -f "$summaryFile" ]; then - echo "🔍 Extracting expected test names from metadata..." - expected_tests=() - while IFS= read -r test_name; do - [[ -n "$test_name" ]] && expected_tests+=("$test_name") - done < <(jq -r '.[].name' "$SCENARIO_METADATA_FILE") - - if [[ ${#expected_tests[@]} -eq 0 ]]; then - echo "❌ No test names found in metadata. Aborting." - exit 1 - fi - - filtered_summary=$(mktemp) - - # Extract the CSV header line starting with '#' - grep -m 1 "^#" "$summaryFile" > "$filtered_summary" +############################################################################### +# Adds a “Result” column with ✅ / ❌ to the given summary file +# • Increments global variable `threshold_failures` for each ❌ row +# NR == 1 catches the header only once, appending “Result” +# PASS rules +# • Throughput Tests #1, #2, #7: PASS when Actual ≥ Fs Requirement +# • All others (Duration Tests): PASS when Actual ≤ Fs Requirement +############################################################################### +addResultColumn() { + local summaryFile="$1" + local tmp + tmp=$(mktemp) + + awk -F',' -v OFS=',' ' + NR == 1 { print $0, "Result"; next } + { + throughputTests = ($1 == "1" || $1 == "2" || $1 == "7") + passCondition = throughputTests ? (($6+0) >= ($4+0)) : (($6+0) <= ($4+0)) + print $0, (passCondition ? "✅" : "❌") + } + ' "$summaryFile" > "$tmp" + + mv "$tmp" "$summaryFile" + + # how many failures (❌) occurred? + local newFails + newFails=$(grep -c '❌' "$summaryFile") + threshold_failures=$(( threshold_failures + newFails )) +} - # Match each expected test name with summary rows - for test_name in "${expected_tests[@]}"; do - trimmedTestName="$(echo "$test_name" | xargs)" - matched_line=$(grep -F "$trimmedTestName" "$summaryFile") - [[ -n "$matched_line" ]] && echo "$matched_line" >> "$filtered_summary" || echo "⚠️ Result not found for [$trimmedTestName]" - done +if [ -f "$summaryFile" ]; then # Output raw CSV for plotting job - echo -e "\n📊 -- BEGIN CSV REPORT --" - cat "$filtered_summary" - echo -e "📊 -- -- END CSV REPORT --\n" - - # ───────────────────────────────────────────────────────────── - # Evaluate FS Thresholds - # ───────────────────────────────────────────────────────────── - - # Evaluate FS pass/fail thresholds - annotated_summary=$(mktemp) - threshold_failures=0 - - # Append header with new column "Pass FS" - head -n 1 "$filtered_summary" | awk '{print $0",Pass FS"}' > "$annotated_summary" - tail -n +2 "$filtered_summary" > tmp_input - - # Exit early if no valid test results were found in the filtered summary - if [[ ! -s tmp_input ]]; then - echo "⚠️ No valid test results found in [$summaryFile]. Skipping FS evaluation." - echo "❌ Summary: No tests were executed or matched expected names." - ((number_of_failures++)) - rm -f tmp_input "$summaryFile" "$filtered_summary" - popd >/dev/null || true - exit $number_of_failures - fi - - # Process each test case (skip header and check values) append pass/fail to annotated_summary - while IFS=, read -r id test_name unit fs_requirement current_expected actual_value; do - [[ -z "$test_name" ]] && continue - - # Trim whitespace from fs_requirement and actual - fs_req=$(echo "$fs_requirement" | xargs) - actual_val=$(echo "$actual_value" | xargs) - fs_pass_status="✅" - - # Special case: zero actual is valid, assign ✅ without warning - if [[ "$test_name" == "HTTP request failures for all tests" ]]; then - if [[ "$actual_val" != "0" && "$actual_val" != "0.000" ]]; then - fs_condition_met=$(awk -v a="$actual_val" -v r="$fs_req" 'BEGIN { print (a <= r) ? 1 : 0 }') - [[ "$fs_condition_met" -ne 1 ]] && fs_pass_status="❌" && ((threshold_failures++)) - fi - else - - # For all other tests: if actual is 0 or 0.000, mark as ❌ failure - if [[ "$actual_val" == "0" || "$actual_val" == "0.000" ]]; then - fs_pass_status="❌" - echo "❌ Error: Actual value for metric '$test_name' is 0. This may indicate an error or missing data." - ((threshold_failures++)) - else - if [[ "$unit" == *"millisecond"* || "$unit" == *"rate of failed requests"* ]]; then - fs_condition_met=$(awk -v a="$actual_val" -v r="$fs_req" 'BEGIN { print (a <= r) ? 1 : 0 }') - else - fs_condition_met=$(awk -v a="$actual_val" -v r="$fs_req" 'BEGIN { print (a >= r) ? 1 : 0 }') - fi - [[ "$fs_condition_met" -ne 1 ]] && fs_pass_status="❌" && ((threshold_failures++)) - fi - fi - - echo "$id,$test_name,$unit,$fs_requirement,$current_expected,$actual_value,$fs_pass_status" >> "$annotated_summary" - done < tmp_input - rm -f tmp_input - - # ───────────────────────────────────────────────────────────── - # Print Human-Readable Report - # ───────────────────────────────────────────────────────────── - table_preview=$(column -t -s, "$annotated_summary") - - # Compute table width safely - table_width=$(echo "$table_preview" | awk '{ if (length > max) max = length } END { print max }') + echo "-- BEGIN CSV REPORT" + cat "$summaryFile" + echo "-- END CSV REPORT" + echo - # Fallback if table_width is empty or not a valid number - if ! [[ "$table_width" =~ ^[0-9]+$ ]]; then - table_width=80 + # Output human-readable report + echo "####################################################################################################" + if [ "$testProfile" = "kpi" ]; then + echo "## K 6 K P I P E R F O R M A N C E T E S T R E S U L T S ##" fi + echo "####################################################################################################" + addResultColumn "$summaryFile" + column -t -s, "$summaryFile" + echo - # Now safely create the border line - border_line=$(printf '#%.0s' $(seq 1 "$table_width")) - - format_title_spaced() { - local input="$1" - local result="" - for word in $input; do - for ((i=0; i<${#word}; i++)); do - result+="${word:$i:1} " - done - result+=" " - done - echo "$result" - } - - # Pad title string to center it in the table width - raw_title="K6 ${testProfile^^} PERFORMANCE TEST RESULTS" - - # Dynamically center title within the line - title="$(format_title_spaced "$raw_title")" - title_line=$(printf "## %*s %*s##" \ - $(( (table_width - 6 + ${#title}) / 2 )) "$title" \ - $(( (table_width - 6 - ${#title}) / 2 )) "") - - # Print header - echo "$border_line" - echo "$title_line" - echo "$border_line" + # Clean up + rm -f "$summaryFile" - # Then print the table - echo "$table_preview" +else + echo "Error: Failed to generate $summaryFile" >&2 +fi - # Print closing border after the table - echo "$border_line" - echo +# Change the directory back where it was +popd >/dev/null || exit 1 - # 🎯 Final FS Summary of threshold result +# 🎯 Final FS Summary of threshold result and exit if needed +if [[ "$testProfile" == "kpi" ]]; then if (( threshold_failures > 0 )); then echo "❌ Summary: [$threshold_failures] test(s) failed FS requirements." echo @@ -205,21 +112,12 @@ if [ -f "$summaryFile" ]; then echo "❗ Number of failures or threshold breaches: $threshold_failures" echo "Please check the summary reports and logs above for details." echo "Investigate any failing metrics and consider re-running the tests after fixes." - echo - ((number_of_failures++)) + exit $threshold_failures else echo "✅ All tests passed FS requirements." echo "✅ No threshold violations or execution errors detected." echo "You can review detailed results in the generated summary." fi - - # Cleanup temp files related to reporting - rm -f "$filtered_summary" "$annotated_summary" - -else # no summary file - echo "❌ Error: Summary file [$summaryFile] was not generated. Possible K6 failure." - ((number_of_failures++)) -fi else # ───────────────────────────────────────────────────────────── # Endurance Profile: Investigative Guidance @@ -249,17 +147,9 @@ else echo " - Error rates and failed checks" echo " - Container memory growth over time (especially in endurance tests)" echo - echo " • 📄 Logs and Summary:" - echo " - Check '$summaryFile' for raw execution summary." + echo " • 📄 Logs:" echo " - Inspect logs for timeout/retries/exception patterns." echo echo "ℹ️ Reminder: For KPI validation with FS thresholds, re-run with profile: 'kpi'" - echo -fi # end of testProfile check - -# Cleanup global temp file -rm -f "$summaryFile" - -# final exit -popd >/dev/null || true -exit $number_of_failures \ No newline at end of file + exit 0 +fi \ No newline at end of file diff --git a/k6-tests/ncmp/templates/scenario-javascript.tmpl b/k6-tests/ncmp/ncmp-test-runner.js similarity index 85% rename from k6-tests/ncmp/templates/scenario-javascript.tmpl rename to k6-tests/ncmp/ncmp-test-runner.js index 0f75dcf51e..6fcb02e8cc 100644 --- a/k6-tests/ncmp/templates/scenario-javascript.tmpl +++ b/k6-tests/ncmp/ncmp-test-runner.js @@ -42,7 +42,23 @@ import { passthroughRead, passthroughWrite, legacyBatchRead } from './common/pas import { sendBatchOfKafkaMessages } from './common/produce-avc-event.js'; import { executeWriteDataJob } from "./common/write-data-job.js"; -#METRICS-TRENDS-PLACEHOLDER# +let cmHandlesCreatedTrend = new Trend('cm_handles_created', false); +let cmHandlesDeletedTrend = new Trend('cm_handles_deleted', false); +let cmHandleIdSearchNoFilterTrend = new Trend('cm_handle_id_search_no_filter', true); +let cmHandleIdSearchModuleFilterTrend = new Trend('cm_handle_id_search_module_filter', true); +let cmHandleIdSearchPropertyFilterTrend = new Trend('cm_handle_id_search_property_filter', true); +let cmHandleIdSearchCpsPathFilterTrend = new Trend('cm_handle_id_search_cps_path_filter', true); +let cmHandleIdSearchTrustLevelFilterTrend = new Trend('cm_handle_id_search_trust_level_filter', true); +let cmHandleSearchNoFilterTrend = new Trend('cm_handle_search_no_filter', true); +let cmHandleSearchModuleFilterTrend = new Trend('cm_handle_search_module_filter', true); +let cmHandleSearchPropertyFilterTrend = new Trend('cm_handle_search_property_filter', true); +let cmHandleSearchCpsPathFilterTrend = new Trend('cm_handle_search_cps_path_filter', true); +let cmHandleSearchTrustLevelFilterTrend = new Trend('cm_handle_search_trust_level_filter', true); +let ncmpReadOverheadTrend = new Trend('ncmp_read_overhead', true); +let ncmpWriteOverheadTrend = new Trend('ncmp_write_overhead', true); +let legacyBatchReadTrend = new Trend('legacy_batch_read', false); +let dcmWriteDataJobSmallTrend = new Trend('dcm_write_data_job_small', true); +let dcmWriteDataJobLargeTrend = new Trend('dcm_write_data_job_large', true); const EXPECTED_WRITE_RESPONSE_COUNT = 1; @@ -86,7 +102,7 @@ export function teardown() { const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber); const response = deleteCmHandles(nextBatchOfCmHandleIds); if (response.error_code === 0) { - numberOfDeregisteredCmHandles += REGISTRATION_BATCH_SIZE + numberOfDeregisteredCmHandles += REGISTRATION_BATCH_SIZE } check(response, { 'delete CM-handles status equals 200': (response) => response.status === 200 }); } @@ -200,9 +216,9 @@ export function handleSummary(data) { if (testProfile === 'kpi') { console.log("✅ Generating KPI summary..."); return { - stdout: makeCustomSummaryReport(data.metrics, options.thresholds), + stdout: makeCustomSummaryReport(data, options), }; } console.log("⛔ Skipping KPI summary (not in 'kpi' profile)"); return {}; -} +} \ No newline at end of file -- 2.16.6