- Introduced templates for kpi config and scenario config.
- Used these templates to generate thresholds and trends.
Issue-ID: CPS-2867
Change-Id: Ia6b6627950ca61c602eef34a08b87e5ab2e9d3c7
Signed-off-by: sourabh_sourabh <sourabh.sourabh@est.tech>
csit/env.properties
csit/archives/
-/k6-tests/image/
\ No newline at end of file
+/k6-tests/image/
+k6-tests/ncmp/config/kpi-scenario-execution-definition.json
+/k6-tests/ncmp/*Summary.csv
+/k6-tests/ncmp/scenario-javascript.js
\ No newline at end of file
import {Trend} from 'k6/metrics';
export const TEST_PROFILE = __ENV.TEST_PROFILE ? __ENV.TEST_PROFILE : 'kpi'
-export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}.json`));
-export const testKpiMetaData = JSON.parse(open(`../config/test-kpi-metadata.json`));
+export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}-scenario-execution-definition.json`));
+export const testKpiMetaData = JSON.parse(open(`../config/scenario-metadata.json`));
export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer;
export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl;
export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl;
--- /dev/null
+#!/bin/bash
+#
+# Copyright 2025 OpenInfra Foundation Europe. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This script is used to generate K6 performance testing 'thresholds' declarations file by:
+# 1) Extracting thresholds from metric metadata JSON (scenario-metadata.json)
+# 2) Injects them into a scenario execution template file (scenario-execution-definition.tmpl) and later writes
+# it into config file named as {performance-test-profile-name}-scenario-execution-definition.json.
+# Note: {performance-test-profile-name} : There are two test profiles that can be run with either: kpi or endurance.
+#
+
+# Path to the JSON file containing metric metadata.
+# This JSON holds metric names, units, and threshold values.
+SCENARIO_METADATA_FILE="./config/scenario-metadata.json"
+
+# Scenario JSON template file for scenario execution configuration.
+# Contains placeholder (#THRESHOLDS-PLACEHOLDER#) to be replaced with actual threshold values.
+SCENARIO_CONFIG_TEMPLATE_FILE="./templates/scenario-execution-definition.tmpl"
+
+# Final json scenario execution configuration file with thresholds injected.
+SCENARIO_CONFIG_OUTPUT_FILE="./config/kpi-scenario-execution-definition.json"
+
+# ─────────────────────────────────────────────────────────────
+# Function: create_thresholds
+# Description:
+# Prepares threshold expressions for each metric.
+# Parameters:
+# $1 - Path to the metric metadata JSON file. (scenario-metadata.json)
+# Returns:
+# JSON object array containing thresholds for each metric.
+# ─────────────────────────────────────────────────────────────
+create_thresholds() {
+ local scenario_metadata_json_file="$1"
+
+ # Define the jq script to build the thresholds JSON object
+ read -r -d '' thresholds_per_metric_as_json << 'EOF'
+ # Set threshold expression based on metric type.
+ reduce .[] as $metric (
+ {};
+ .[$metric.metric] = (
+ if $metric.metric == "http_req_failed" then
+ ["rate <= \($metric.kpiThreshold)"] # For failure rate metric, threshold is rate <= value
+ elif ($metric.unit | test("/second")) then
+ ["avg >= \($metric.kpiThreshold)"] # For per-second metrics, expect average >= threshold
+ else
+ ["avg <= \($metric.kpiThreshold)"] # Otherwise, average <= threshold
+ end
+ )
+ )
+EOF
+
+ # This returns a JSON object with:
+ # - 'thresholds': array of JS declaration strings
+ jq -r "$thresholds_per_metric_as_json" "$scenario_metadata_json_file"
+}
+
+# ─────────────────────────────────────────────────────────────
+# Function: inject_thresholds_into_scenario-execution
+# Description:
+# Injects the extracted threshold JSON object into the scenario
+# configuration template by replacing the `.thresholds` named property.
+# Parameters:
+# $1 - JSON string of threshold mappings. (scenario-metadata.json)
+# $2 - Template scenario config file path. (scenario-execution-definition.tmpl)
+# $3 - Output scenario config file path (kpi-scenario-execution-definition.json)
+# Returns:
+# Writes the updated JSON to output file. (kpi-scenario-execution-definition.json)
+# ─────────────────────────────────────────────────────────────
+inject_thresholds_into_scenario_execution_config() {
+ local thresholds_json="$1"
+ local scenario_execution_template_file="$2"
+ local scenario_execution_output_file="$3"
+
+ # Use jq to overwrite the `.thresholds` property in the template with the generated thresholds JSON
+ jq --argjson thresholds "$thresholds_json" '.thresholds = $thresholds' "$scenario_execution_template_file" | jq '.' > "$scenario_execution_output_file"
+}
+
+# ─────────────────────────────────────────────────────────────
+# Main script execution starts here
+# ─────────────────────────────────────────────────────────────
+
+# Inform user script is starting threshold generation
+echo "Generating thresholds from [$SCENARIO_METADATA_FILE]..."
+
+# Calling function to extract threshold JSON object from metric metadata JSON file
+scenario_execution_thresholds_json=$(create_thresholds "$SCENARIO_METADATA_FILE")
+
+# Inject the extracted thresholds json block into the scenario config template and write into output file
+inject_thresholds_into_scenario_execution_config "$scenario_execution_thresholds_json" "$SCENARIO_CONFIG_TEMPLATE_FILE" "$SCENARIO_CONFIG_OUTPUT_FILE"
+
+# Final confirmation message on successful injection
+echo "Threshold block has been injected into [$SCENARIO_CONFIG_OUTPUT_FILE]"
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+#
+# Copyright 2025 OpenInfra Foundation Europe. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This script is used to generate K6 performance testing 'Trend' declarations file by:
+# 1) Extracting trends from metric metadata JSON (scenario-metadata.json)
+# 2) Injects them into a JavaScript template file (scenario-javascript.tmpl) and then write it into final JavaScript
+# file named as scenario-javascript.json.
+#
+
+# Path to the JSON file that contains metric definitions (name, unit, threshold, etc.)
+# This JSON holds metric names, units, and threshold values.
+SCENARIO_METADATA_FILE="./config/scenario-metadata.json"
+
+# Path to the JS template file where the placeholder `#METRICS-TRENDS-PLACEHOLDER#` exists.
+# This is where the generated trend declarations will be inserted.
+SCENARIO_JAVASCRIPT_TEMPLATE_FILE="./templates/scenario-javascript.tmpl"
+
+# Output JavaScript file where the final result (with inserted trend declarations) will be saved.
+SCENARIO_JAVASCRIPT_OUTPUT_FILE="scenario-javascript.js"
+
+# ─────────────────────────────────────────────────────────────────────────────────
+# Function: create_trend_declarations
+# Description:
+# Converts metrics from the metadata JSON into JavaScript `Trend` declarations.
+# These declarations are used for K6 performance testing reports.
+# Parameters:
+# $1 - Accept the path to the metric metadata file as input. (scenario-metadata.json)
+# Returns:
+# trend declarations as JSON array object
+# ─────────────────────────────────────────────────────────────────────────────────
+
+create_trend_declarations() {
+ local scenario_metadata_json_file="$1"
+
+ # Read and assign a JQ script to a here-document variable. (trend_declarations)
+ # `-r` makes jq output raw strings and `-d ''` lets us use multiline input.
+ read -r -d '' trend_declarations << 'EOF'
+ # Define a helper function (toCamelCase) that converts metric names from snake_case to camelCase
+ # (for JS variable names for example: cm_handles_deleted → cmHandlesDeleted)
+ def toCamelCase:
+ split("_") as $parts | # Split string by "_"
+ ($parts[0]) + # Keep first part as it is
+ ($parts[1:] | map((.[0:1] | ascii_upcase) + .[1:]) # Capitalize rest of each word
+ | join("")); # Join all parts into one string
+
+ # Loop through each metric item and generate a JavaScript `Trend` declaration if unit matches.
+ .[] # Iterate through array
+ | select((.unit == "milliseconds") or (.unit | test("/second"))) # Select based on valid units
+ | "export let \(.metric | toCamelCase)Trend = new Trend('\(.metric)', \(.unit == "milliseconds"));"
+ # Output javascript declaration string: `export let abcTrend = new Trend('abc', true/false);`
+EOF
+ # Execute the jq script on the metadata file to generate the trend declarations
+ jq -r "$trend_declarations" "$scenario_metadata_json_file"
+}
+
+# ─────────────────────────────────────────────────────────────
+# Function: inject_trends_into_js_template
+# Description:
+# Replaces the placeholder line `#METRICS-TRENDS-PLACEHOLDER#` in the template
+# file with actual JS trend declarations.
+# Parameters:
+# $1 - JSON string of threshold mappings. Trend declaration strings.
+# for example: export let abcTrend = new Trend('abc', true), from scenario-metadata.json)
+# $2 - Template scenario javascript file path. (scenario-javascript.tmpl)
+# $3 - Output scenario script file path (scenario-javascript.js)
+# Returns:
+# Writes the updated JSON to output file. (scenario-javascript.js)
+# ─────────────────────────────────────────────────────────────
+inject_trends_into_javascript_template() {
+ local trend_declarations="$1"
+ local scenario_javascript_template_file="$2"
+ local scenario_javascript_output_file="$3"
+
+ # Use awk to replace the placeholder line with trend declarations
+ awk -v trends="$trend_declarations" ' # Pass trends into awk variable
+ {
+ if ($0 ~ /#METRICS-TRENDS-PLACEHOLDER#/) {
+ print trends # Print the trend declarations instead of the placeholder
+ } else {
+ print $0 # Otherwise, print the original line
+ }
+ }
+ ' "$scenario_javascript_template_file" > "$scenario_javascript_output_file" # Save the transformed content into the output JS file
+}
+
+# ─────────────────────────────────────────────────────────────
+# Main Execution Starts Here
+# ─────────────────────────────────────────────────────────────
+
+# Display log message to inform that generation has started
+echo "Generating trend declarations from [$SCENARIO_METADATA_FILE]..."
+
+# Calling trend generation function
+scenario_javascript_trend_declarations=$(create_trend_declarations "$SCENARIO_METADATA_FILE")
+
+# Inject the generated trends into the JavaScript template and write it into scenario output file
+inject_trends_into_javascript_template "$scenario_javascript_trend_declarations" "$SCENARIO_JAVASCRIPT_TEMPLATE_FILE" "$SCENARIO_JAVASCRIPT_OUTPUT_FILE"
+
+# Final confirmation message to indicate success
+echo "Trend declarations inserted into [$SCENARIO_JAVASCRIPT_OUTPUT_FILE]"
\ No newline at end of file
#
# ─────────────────────────────────────────────────────────────
-# 📁 Navigate to Script Directory
+# Navigate to Script Directory
# ─────────────────────────────────────────────────────────────
pushd "$(dirname "$0")" >/dev/null || {
echo "❌ Failed to access script directory. Exiting."
number_of_failures=0
testProfile=$1
summaryFile="${testProfile}Summary.csv"
-KPI_METADATA_FILE="./config/test-kpi-metadata.json"
-KPI_CONFIG_FILE="./config/kpi.json"
-SCENARIOS_CONFIG_SCRIPT="scenarios-config.js"
+# Path to the JSON file containing metric metadata.
+# This JSON holds metric names, units, and threshold values.
+SCENARIO_METADATA_FILE="./config/scenario-metadata.json"
echo
echo "📢 Running NCMP K6 performance test for profile: [$testProfile]"
echo
-# ───────────────────────────────────────────────────────────────────────────
-# 1️⃣ Generate trend declarations and (conditionally) thresholds from metadata
-# ───────────────────────────────────────────────────────────────────────────
-echo "🔧 Generating trend declarations from [$KPI_METADATA_FILE]..."
-
-read -r -d '' jq_script << 'EOF'
-def toCamelCase:
- split("_") as $parts |
- ($parts[0]) + ($parts[1:] | map((.[0:1] | ascii_upcase) + .[1:]) | join(""));
-
-reduce .[] as $item (
- { trends: [], thresholds: {} };
- if ($item.unit == "milliseconds") or ($item.unit | test("/second")) then
- .trends += [
- "export let \($item.metric | toCamelCase)Trend = new Trend('\($item.metric)', \($item.unit == "milliseconds"));"
- ]
- else
- .
- end
- |
- .thresholds[$item.metric] = (
- if $item.metric == "http_req_failed" then
- ["rate <= \($item.kpiThreshold)"]
- elif ($item.unit | test("/second")) then
- ["avg >= \($item.kpiThreshold)"]
- else
- ["avg <= \($item.kpiThreshold)"]
- end
- )
-)
-EOF
-
-# Execute jq script
-jq_output=$(jq -r "$jq_script" "$KPI_METADATA_FILE")
-
-# Extract trends
-trend_declarations=$(echo "$jq_output" | jq -r '.trends[]')
-
-# Replace placeholder in runner with generated trends
-TMP_FILE=$(mktemp)
-awk -v trends="$trend_declarations" '
- BEGIN { replaced=0 }
- {
- if ($0 ~ /#METRICS-TRENDS-PLACE-HOLDER#/ && replaced == 0) {
- print trends
- replaced=1
- } else {
- print $0
- }
- }
-' "$SCENARIOS_CONFIG_SCRIPT" > "$TMP_FILE"
-mv "$TMP_FILE" "$SCENARIOS_CONFIG_SCRIPT"
-echo "✅ Trend declarations inserted into [$SCENARIOS_CONFIG_SCRIPT]"
+chmod +x ./create-scenario-javascript.sh
+source ./create-scenario-javascript.sh
-# If profile is KPI, generate threshold config too
if [[ "$testProfile" == "kpi" ]]; then
- echo "📌 Writing thresholds to [$KPI_CONFIG_FILE]..."
- # Update thresholds in KPI config
- # Extract thresholds
- thresholds_json=$(echo "$jq_output" | jq '.thresholds')
- TMP_FILE=$(mktemp)
- cp "$KPI_CONFIG_FILE" "$TMP_FILE"
- jq --argjson thresholds "$thresholds_json" '.thresholds = $thresholds' "$TMP_FILE" | jq '.' > "$KPI_CONFIG_FILE"
- rm -f "$TMP_FILE"
- echo "✅ Threshold block has been injected into [$KPI_CONFIG_FILE]"
- echo
+ chmod +x ./create-scenario-execution-definition.sh
+ source ./create-scenario-execution-definition.sh
fi
# ─────────────────────────────────────────────────────────────
-# 2️⃣ Run K6 and Capture Output
+# Run K6 and Capture Output
# ─────────────────────────────────────────────────────────────
-k6 run scenarios-config.js -e TEST_PROFILE="$testProfile" > "$summaryFile"
+k6 run scenario-javascript.js -e TEST_PROFILE="$testProfile" > "$summaryFile"
k6_exit_code=$?
case $k6_exit_code in
if [[ "$testProfile" == "kpi" ]]; then
# ─────────────────────────────────────────────────────────────
-# 3️⃣ Extract and Filter Summary Data
+# Extract and Filter Summary Data
# ─────────────────────────────────────────────────────────────
if [ -f "$summaryFile" ]; then
echo "🔍 Extracting expected test names from metadata..."
expected_tests=()
while IFS= read -r test_name; do
[[ -n "$test_name" ]] && expected_tests+=("$test_name")
- done < <(jq -r '.[].name' "$KPI_METADATA_FILE")
+ done < <(jq -r '.[].name' "$SCENARIO_METADATA_FILE")
if [[ ${#expected_tests[@]} -eq 0 ]]; then
echo "❌ No test names found in metadata. Aborting."
echo -e "📊 -- -- END CSV REPORT --\n"
# ─────────────────────────────────────────────────────────────
- # 4️⃣ Evaluate FS Thresholds
+ # Evaluate FS Thresholds
# ─────────────────────────────────────────────────────────────
# Evaluate FS pass/fail thresholds
rm -f tmp_input
# ─────────────────────────────────────────────────────────────
- # 5️⃣ Print Human-Readable Report
+ # Print Human-Readable Report
# ─────────────────────────────────────────────────────────────
table_preview=$(column -t -s, "$annotated_summary")
# Cleanup global temp file
rm -f "$summaryFile"
-# ─────────────────────────────────────────────────────────────
-# 🔚 Final Exit
-# ─────────────────────────────────────────────────────────────
+# final exit
popd >/dev/null || true
exit $number_of_failures
\ No newline at end of file
"startTime": "923ms"
}
},
- "thresholds": "#SCENARIO-THRESHOLDS#"
+ "thresholds": "#THRESHOLDS-PLACEHOLDER#"
}
import { sendBatchOfKafkaMessages } from './common/produce-avc-event.js';
import { executeWriteDataJob } from "./common/write-data-job.js";
-#METRICS-TRENDS-PLACE-HOLDER#
+#METRICS-TRENDS-PLACEHOLDER#
const EXPECTED_WRITE_RESPONSE_COUNT = 1;
Code Review / cps.git / commitdiff