From 3e1bd16853106a5bb7939f6777b4ea085ebcefde Mon Sep 17 00:00:00 2001 From: sourabh_sourabh Date: Mon, 7 Jul 2025 13:40:51 +0100 Subject: [PATCH] Refactor K6 test script to use template based code generation - Introduced templates for kpi config and scenario config. - Used these templates to generate thresholds and trends. Issue-ID: CPS-2867 Change-Id: Ia6b6627950ca61c602eef34a08b87e5ab2e9d3c7 Signed-off-by: sourabh_sourabh --- .gitignore | 5 +- k6-tests/ncmp/common/utils.js | 4 +- ...> endurance-scenario-execution-definition.json} | 0 ...st-kpi-metadata.json => scenario-metadata.json} | 0 .../ncmp/create-scenario-execution-definition.sh | 106 +++++++++++++++++++ k6-tests/ncmp/create-scenario-javascript.sh | 115 +++++++++++++++++++++ k6-tests/ncmp/execute-k6-scenarios.sh | 92 +++-------------- .../scenario-execution-definition.tmpl} | 2 +- .../scenario-javascript.tmpl} | 2 +- 9 files changed, 244 insertions(+), 82 deletions(-) rename k6-tests/ncmp/config/{endurance.json => endurance-scenario-execution-definition.json} (100%) rename k6-tests/ncmp/config/{test-kpi-metadata.json => scenario-metadata.json} (100%) create mode 100644 k6-tests/ncmp/create-scenario-execution-definition.sh create mode 100644 k6-tests/ncmp/create-scenario-javascript.sh rename k6-tests/ncmp/{config/kpi.json => templates/scenario-execution-definition.tmpl} (99%) rename k6-tests/ncmp/{scenarios-config.js => templates/scenario-javascript.tmpl} (99%) diff --git a/.gitignore b/.gitignore index f0f1d15e16..58cdadfad1 100755 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,7 @@ tmp/ csit/env.properties csit/archives/ -/k6-tests/image/ \ No newline at end of file +/k6-tests/image/ +k6-tests/ncmp/config/kpi-scenario-execution-definition.json +/k6-tests/ncmp/*Summary.csv +/k6-tests/ncmp/scenario-javascript.js \ No newline at end of file diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index bb4e5ae7a0..fd24680a4c 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -24,8 +24,8 @@ import {check} from 'k6'; import {Trend} from 'k6/metrics'; export const TEST_PROFILE = __ENV.TEST_PROFILE ? __ENV.TEST_PROFILE : 'kpi' -export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}.json`)); -export const testKpiMetaData = JSON.parse(open(`../config/test-kpi-metadata.json`)); +export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}-scenario-execution-definition.json`)); +export const testKpiMetaData = JSON.parse(open(`../config/scenario-metadata.json`)); export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer; export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl; export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl; diff --git a/k6-tests/ncmp/config/endurance.json b/k6-tests/ncmp/config/endurance-scenario-execution-definition.json similarity index 100% rename from k6-tests/ncmp/config/endurance.json rename to k6-tests/ncmp/config/endurance-scenario-execution-definition.json diff --git a/k6-tests/ncmp/config/test-kpi-metadata.json b/k6-tests/ncmp/config/scenario-metadata.json similarity index 100% rename from k6-tests/ncmp/config/test-kpi-metadata.json rename to k6-tests/ncmp/config/scenario-metadata.json diff --git a/k6-tests/ncmp/create-scenario-execution-definition.sh b/k6-tests/ncmp/create-scenario-execution-definition.sh new file mode 100644 index 0000000000..ebe8d75f88 --- /dev/null +++ b/k6-tests/ncmp/create-scenario-execution-definition.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# Copyright 2025 OpenInfra Foundation Europe. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is used to generate K6 performance testing 'thresholds' declarations file by: +# 1) Extracting thresholds from metric metadata JSON (scenario-metadata.json) +# 2) Injects them into a scenario execution template file (scenario-execution-definition.tmpl) and later writes +# it into config file named as {performance-test-profile-name}-scenario-execution-definition.json. +# Note: {performance-test-profile-name} : There are two test profiles that can be run with either: kpi or endurance. +# + +# Path to the JSON file containing metric metadata. +# This JSON holds metric names, units, and threshold values. +SCENARIO_METADATA_FILE="./config/scenario-metadata.json" + +# Scenario JSON template file for scenario execution configuration. +# Contains placeholder (#THRESHOLDS-PLACEHOLDER#) to be replaced with actual threshold values. +SCENARIO_CONFIG_TEMPLATE_FILE="./templates/scenario-execution-definition.tmpl" + +# Final json scenario execution configuration file with thresholds injected. +SCENARIO_CONFIG_OUTPUT_FILE="./config/kpi-scenario-execution-definition.json" + +# ───────────────────────────────────────────────────────────── +# Function: create_thresholds +# Description: +# Prepares threshold expressions for each metric. +# Parameters: +# $1 - Path to the metric metadata JSON file. (scenario-metadata.json) +# Returns: +# JSON object array containing thresholds for each metric. +# ───────────────────────────────────────────────────────────── +create_thresholds() { + local scenario_metadata_json_file="$1" + + # Define the jq script to build the thresholds JSON object + read -r -d '' thresholds_per_metric_as_json << 'EOF' + # Set threshold expression based on metric type. + reduce .[] as $metric ( + {}; + .[$metric.metric] = ( + if $metric.metric == "http_req_failed" then + ["rate <= \($metric.kpiThreshold)"] # For failure rate metric, threshold is rate <= value + elif ($metric.unit | test("/second")) then + ["avg >= \($metric.kpiThreshold)"] # For per-second metrics, expect average >= threshold + else + ["avg <= \($metric.kpiThreshold)"] # Otherwise, average <= threshold + end + ) + ) +EOF + + # This returns a JSON object with: + # - 'thresholds': array of JS declaration strings + jq -r "$thresholds_per_metric_as_json" "$scenario_metadata_json_file" +} + +# ───────────────────────────────────────────────────────────── +# Function: inject_thresholds_into_scenario-execution +# Description: +# Injects the extracted threshold JSON object into the scenario +# configuration template by replacing the `.thresholds` named property. +# Parameters: +# $1 - JSON string of threshold mappings. (scenario-metadata.json) +# $2 - Template scenario config file path. (scenario-execution-definition.tmpl) +# $3 - Output scenario config file path (kpi-scenario-execution-definition.json) +# Returns: +# Writes the updated JSON to output file. (kpi-scenario-execution-definition.json) +# ───────────────────────────────────────────────────────────── +inject_thresholds_into_scenario_execution_config() { + local thresholds_json="$1" + local scenario_execution_template_file="$2" + local scenario_execution_output_file="$3" + + # Use jq to overwrite the `.thresholds` property in the template with the generated thresholds JSON + jq --argjson thresholds "$thresholds_json" '.thresholds = $thresholds' "$scenario_execution_template_file" | jq '.' > "$scenario_execution_output_file" +} + +# ───────────────────────────────────────────────────────────── +# Main script execution starts here +# ───────────────────────────────────────────────────────────── + +# Inform user script is starting threshold generation +echo "Generating thresholds from [$SCENARIO_METADATA_FILE]..." + +# Calling function to extract threshold JSON object from metric metadata JSON file +scenario_execution_thresholds_json=$(create_thresholds "$SCENARIO_METADATA_FILE") + +# Inject the extracted thresholds json block into the scenario config template and write into output file +inject_thresholds_into_scenario_execution_config "$scenario_execution_thresholds_json" "$SCENARIO_CONFIG_TEMPLATE_FILE" "$SCENARIO_CONFIG_OUTPUT_FILE" + +# Final confirmation message on successful injection +echo "Threshold block has been injected into [$SCENARIO_CONFIG_OUTPUT_FILE]" \ No newline at end of file diff --git a/k6-tests/ncmp/create-scenario-javascript.sh b/k6-tests/ncmp/create-scenario-javascript.sh new file mode 100644 index 0000000000..25df997172 --- /dev/null +++ b/k6-tests/ncmp/create-scenario-javascript.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# Copyright 2025 OpenInfra Foundation Europe. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is used to generate K6 performance testing 'Trend' declarations file by: +# 1) Extracting trends from metric metadata JSON (scenario-metadata.json) +# 2) Injects them into a JavaScript template file (scenario-javascript.tmpl) and then write it into final JavaScript +# file named as scenario-javascript.json. +# + +# Path to the JSON file that contains metric definitions (name, unit, threshold, etc.) +# This JSON holds metric names, units, and threshold values. +SCENARIO_METADATA_FILE="./config/scenario-metadata.json" + +# Path to the JS template file where the placeholder `#METRICS-TRENDS-PLACEHOLDER#` exists. +# This is where the generated trend declarations will be inserted. +SCENARIO_JAVASCRIPT_TEMPLATE_FILE="./templates/scenario-javascript.tmpl" + +# Output JavaScript file where the final result (with inserted trend declarations) will be saved. +SCENARIO_JAVASCRIPT_OUTPUT_FILE="scenario-javascript.js" + +# ───────────────────────────────────────────────────────────────────────────────── +# Function: create_trend_declarations +# Description: +# Converts metrics from the metadata JSON into JavaScript `Trend` declarations. +# These declarations are used for K6 performance testing reports. +# Parameters: +# $1 - Accept the path to the metric metadata file as input. (scenario-metadata.json) +# Returns: +# trend declarations as JSON array object +# ───────────────────────────────────────────────────────────────────────────────── + +create_trend_declarations() { + local scenario_metadata_json_file="$1" + + # Read and assign a JQ script to a here-document variable. (trend_declarations) + # `-r` makes jq output raw strings and `-d ''` lets us use multiline input. + read -r -d '' trend_declarations << 'EOF' + # Define a helper function (toCamelCase) that converts metric names from snake_case to camelCase + # (for JS variable names for example: cm_handles_deleted → cmHandlesDeleted) + def toCamelCase: + split("_") as $parts | # Split string by "_" + ($parts[0]) + # Keep first part as it is + ($parts[1:] | map((.[0:1] | ascii_upcase) + .[1:]) # Capitalize rest of each word + | join("")); # Join all parts into one string + + # Loop through each metric item and generate a JavaScript `Trend` declaration if unit matches. + .[] # Iterate through array + | select((.unit == "milliseconds") or (.unit | test("/second"))) # Select based on valid units + | "export let \(.metric | toCamelCase)Trend = new Trend('\(.metric)', \(.unit == "milliseconds"));" + # Output javascript declaration string: `export let abcTrend = new Trend('abc', true/false);` +EOF + # Execute the jq script on the metadata file to generate the trend declarations + jq -r "$trend_declarations" "$scenario_metadata_json_file" +} + +# ───────────────────────────────────────────────────────────── +# Function: inject_trends_into_js_template +# Description: +# Replaces the placeholder line `#METRICS-TRENDS-PLACEHOLDER#` in the template +# file with actual JS trend declarations. +# Parameters: +# $1 - JSON string of threshold mappings. Trend declaration strings. +# for example: export let abcTrend = new Trend('abc', true), from scenario-metadata.json) +# $2 - Template scenario javascript file path. (scenario-javascript.tmpl) +# $3 - Output scenario script file path (scenario-javascript.js) +# Returns: +# Writes the updated JSON to output file. (scenario-javascript.js) +# ───────────────────────────────────────────────────────────── +inject_trends_into_javascript_template() { + local trend_declarations="$1" + local scenario_javascript_template_file="$2" + local scenario_javascript_output_file="$3" + + # Use awk to replace the placeholder line with trend declarations + awk -v trends="$trend_declarations" ' # Pass trends into awk variable + { + if ($0 ~ /#METRICS-TRENDS-PLACEHOLDER#/) { + print trends # Print the trend declarations instead of the placeholder + } else { + print $0 # Otherwise, print the original line + } + } + ' "$scenario_javascript_template_file" > "$scenario_javascript_output_file" # Save the transformed content into the output JS file +} + +# ───────────────────────────────────────────────────────────── +# Main Execution Starts Here +# ───────────────────────────────────────────────────────────── + +# Display log message to inform that generation has started +echo "Generating trend declarations from [$SCENARIO_METADATA_FILE]..." + +# Calling trend generation function +scenario_javascript_trend_declarations=$(create_trend_declarations "$SCENARIO_METADATA_FILE") + +# Inject the generated trends into the JavaScript template and write it into scenario output file +inject_trends_into_javascript_template "$scenario_javascript_trend_declarations" "$SCENARIO_JAVASCRIPT_TEMPLATE_FILE" "$SCENARIO_JAVASCRIPT_OUTPUT_FILE" + +# Final confirmation message to indicate success +echo "Trend declarations inserted into [$SCENARIO_JAVASCRIPT_OUTPUT_FILE]" \ No newline at end of file diff --git a/k6-tests/ncmp/execute-k6-scenarios.sh b/k6-tests/ncmp/execute-k6-scenarios.sh index b050764a7e..a25e1e793d 100755 --- a/k6-tests/ncmp/execute-k6-scenarios.sh +++ b/k6-tests/ncmp/execute-k6-scenarios.sh @@ -16,7 +16,7 @@ # # ───────────────────────────────────────────────────────────── -# 📁 Navigate to Script Directory +# Navigate to Script Directory # ───────────────────────────────────────────────────────────── pushd "$(dirname "$0")" >/dev/null || { echo "❌ Failed to access script directory. Exiting." @@ -29,86 +29,26 @@ pushd "$(dirname "$0")" >/dev/null || { number_of_failures=0 testProfile=$1 summaryFile="${testProfile}Summary.csv" -KPI_METADATA_FILE="./config/test-kpi-metadata.json" -KPI_CONFIG_FILE="./config/kpi.json" -SCENARIOS_CONFIG_SCRIPT="scenarios-config.js" +# Path to the JSON file containing metric metadata. +# This JSON holds metric names, units, and threshold values. +SCENARIO_METADATA_FILE="./config/scenario-metadata.json" echo echo "📢 Running NCMP K6 performance test for profile: [$testProfile]" echo -# ─────────────────────────────────────────────────────────────────────────── -# 1️⃣ Generate trend declarations and (conditionally) thresholds from metadata -# ─────────────────────────────────────────────────────────────────────────── -echo "🔧 Generating trend declarations from [$KPI_METADATA_FILE]..." - -read -r -d '' jq_script << 'EOF' -def toCamelCase: - split("_") as $parts | - ($parts[0]) + ($parts[1:] | map((.[0:1] | ascii_upcase) + .[1:]) | join("")); - -reduce .[] as $item ( - { trends: [], thresholds: {} }; - if ($item.unit == "milliseconds") or ($item.unit | test("/second")) then - .trends += [ - "export let \($item.metric | toCamelCase)Trend = new Trend('\($item.metric)', \($item.unit == "milliseconds"));" - ] - else - . - end - | - .thresholds[$item.metric] = ( - if $item.metric == "http_req_failed" then - ["rate <= \($item.kpiThreshold)"] - elif ($item.unit | test("/second")) then - ["avg >= \($item.kpiThreshold)"] - else - ["avg <= \($item.kpiThreshold)"] - end - ) -) -EOF - -# Execute jq script -jq_output=$(jq -r "$jq_script" "$KPI_METADATA_FILE") - -# Extract trends -trend_declarations=$(echo "$jq_output" | jq -r '.trends[]') - -# Replace placeholder in runner with generated trends -TMP_FILE=$(mktemp) -awk -v trends="$trend_declarations" ' - BEGIN { replaced=0 } - { - if ($0 ~ /#METRICS-TRENDS-PLACE-HOLDER#/ && replaced == 0) { - print trends - replaced=1 - } else { - print $0 - } - } -' "$SCENARIOS_CONFIG_SCRIPT" > "$TMP_FILE" -mv "$TMP_FILE" "$SCENARIOS_CONFIG_SCRIPT" -echo "✅ Trend declarations inserted into [$SCENARIOS_CONFIG_SCRIPT]" +chmod +x ./create-scenario-javascript.sh +source ./create-scenario-javascript.sh -# If profile is KPI, generate threshold config too if [[ "$testProfile" == "kpi" ]]; then - echo "📌 Writing thresholds to [$KPI_CONFIG_FILE]..." - # Update thresholds in KPI config - # Extract thresholds - thresholds_json=$(echo "$jq_output" | jq '.thresholds') - TMP_FILE=$(mktemp) - cp "$KPI_CONFIG_FILE" "$TMP_FILE" - jq --argjson thresholds "$thresholds_json" '.thresholds = $thresholds' "$TMP_FILE" | jq '.' > "$KPI_CONFIG_FILE" - rm -f "$TMP_FILE" - echo "✅ Threshold block has been injected into [$KPI_CONFIG_FILE]" - echo + chmod +x ./create-scenario-execution-definition.sh + source ./create-scenario-execution-definition.sh fi # ───────────────────────────────────────────────────────────── -# 2️⃣ Run K6 and Capture Output +# Run K6 and Capture Output # ───────────────────────────────────────────────────────────── -k6 run scenarios-config.js -e TEST_PROFILE="$testProfile" > "$summaryFile" +k6 run scenario-javascript.js -e TEST_PROFILE="$testProfile" > "$summaryFile" k6_exit_code=$? case $k6_exit_code in @@ -119,14 +59,14 @@ esac if [[ "$testProfile" == "kpi" ]]; then # ───────────────────────────────────────────────────────────── -# 3️⃣ Extract and Filter Summary Data +# Extract and Filter Summary Data # ───────────────────────────────────────────────────────────── if [ -f "$summaryFile" ]; then echo "🔍 Extracting expected test names from metadata..." expected_tests=() while IFS= read -r test_name; do [[ -n "$test_name" ]] && expected_tests+=("$test_name") - done < <(jq -r '.[].name' "$KPI_METADATA_FILE") + done < <(jq -r '.[].name' "$SCENARIO_METADATA_FILE") if [[ ${#expected_tests[@]} -eq 0 ]]; then echo "❌ No test names found in metadata. Aborting." @@ -151,7 +91,7 @@ if [ -f "$summaryFile" ]; then echo -e "📊 -- -- END CSV REPORT --\n" # ───────────────────────────────────────────────────────────── - # 4️⃣ Evaluate FS Thresholds + # Evaluate FS Thresholds # ───────────────────────────────────────────────────────────── # Evaluate FS pass/fail thresholds @@ -209,7 +149,7 @@ if [ -f "$summaryFile" ]; then rm -f tmp_input # ───────────────────────────────────────────────────────────── - # 5️⃣ Print Human-Readable Report + # Print Human-Readable Report # ───────────────────────────────────────────────────────────── table_preview=$(column -t -s, "$annotated_summary") @@ -320,8 +260,6 @@ fi # end of testProfile check # Cleanup global temp file rm -f "$summaryFile" -# ───────────────────────────────────────────────────────────── -# 🔚 Final Exit -# ───────────────────────────────────────────────────────────── +# final exit popd >/dev/null || true exit $number_of_failures \ No newline at end of file diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/templates/scenario-execution-definition.tmpl similarity index 99% rename from k6-tests/ncmp/config/kpi.json rename to k6-tests/ncmp/templates/scenario-execution-definition.tmpl index 13aba8ccc0..8eb5415248 100644 --- a/k6-tests/ncmp/config/kpi.json +++ b/k6-tests/ncmp/templates/scenario-execution-definition.tmpl @@ -169,5 +169,5 @@ "startTime": "923ms" } }, - "thresholds": "#SCENARIO-THRESHOLDS#" + "thresholds": "#THRESHOLDS-PLACEHOLDER#" } diff --git a/k6-tests/ncmp/scenarios-config.js b/k6-tests/ncmp/templates/scenario-javascript.tmpl similarity index 99% rename from k6-tests/ncmp/scenarios-config.js rename to k6-tests/ncmp/templates/scenario-javascript.tmpl index 93bc3209ed..0f75dcf51e 100644 --- a/k6-tests/ncmp/scenarios-config.js +++ b/k6-tests/ncmp/templates/scenario-javascript.tmpl @@ -42,7 +42,7 @@ import { passthroughRead, passthroughWrite, legacyBatchRead } from './common/pas import { sendBatchOfKafkaMessages } from './common/produce-avc-event.js'; import { executeWriteDataJob } from "./common/write-data-job.js"; -#METRICS-TRENDS-PLACE-HOLDER# +#METRICS-TRENDS-PLACEHOLDER# const EXPECTED_WRITE_RESPONSE_COUNT = 1; -- 2.16.6