containers:
- name: cps
image: "{{ .Values.cps.image.repository }}:{{ .Values.cps.image.tag }}"
+ imagePullPolicy: {{ .Values.cps.image.pullPolicy }}
ports:
- containerPort: 8080
- containerPort: 5701
image:
repository: "nexus3.onap.org:10003/onap/cps-and-ncmp"
tag: "latest"
+ pullPolicy: Always
replicas: 2
servicePort: 8080
service:
export const TEST_PROFILE = __ENV.TEST_PROFILE ? __ENV.TEST_PROFILE : 'kpi'
export const testConfig = JSON.parse(open(`../config/${TEST_PROFILE}.json`));
+export const scenarioMetaData = JSON.parse(open(`../config/scenario-metadata.json`));
export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer;
export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl;
export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl;
export function makeCustomSummaryReport(testResults, scenarioConfig) {
const summaryCsvLines = [
'#,Test Name,Unit,Fs Requirement,Current Expectation,Actual',
- makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig),
- makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cm_handles_created', 100, testResults, scenarioConfig),
- makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cm_handles_deleted', 180, testResults, scenarioConfig),
- makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'cm_handle_id_search_no_filter', 550, testResults, scenarioConfig),
- makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'cm_handle_id_search_module_filter', 2300, testResults, scenarioConfig),
- makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'cm_handle_id_search_property_filter', 1450, testResults, scenarioConfig),
- makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'cm_handle_id_search_cps_path_filter', 1500, testResults, scenarioConfig),
- makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'cm_handle_id_search_trust_level_filter', 1600, testResults, scenarioConfig),
- makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_handle_search_no_filter', 18000, testResults, scenarioConfig),
- makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_handle_search_module_filter', 18000, testResults, scenarioConfig),
- makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_handle_search_property_filter', 18000, testResults, scenarioConfig),
- makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_handle_search_cps_path_filter', 18000, testResults, scenarioConfig),
- makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_handle_search_trust_level_filter', 18000, testResults, scenarioConfig),
- makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_read_overhead', 18, testResults, scenarioConfig),
- makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_write_overhead', 18, testResults, scenarioConfig),
- makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read', 200, testResults, scenarioConfig),
- makeSummaryCsvLine('8', 'Write data job scenario - small', 'milliseconds', 'dcm_write_data_job_small', 100, testResults, scenarioConfig),
- makeSummaryCsvLine('9', 'Write data job scenario - large', 'milliseconds', 'dcm_write_data_job_large', 8000, testResults, scenarioConfig),
+ ...scenarioMetaData.map(kpiTest => {
+ return makeSummaryCsvLine(
+ kpiTest.testNumber,
+ kpiTest.testName,
+ kpiTest.unit,
+ kpiTest.measurementName,
+ kpiTest.currentExpectation,
+ testResults,
+ scenarioConfig
+ );
+ })
];
return summaryCsvLines.join('\n') + '\n';
}
--- /dev/null
+[
+ {
+ "testNumber" : "0",
+ "testName" : "HTTP request failures for all tests",
+ "unit" : "rate of failed requests",
+ "measurementName" : "http_req_failed",
+ "currentExpectation" : 0
+ },
+ {
+ "testNumber" : "1",
+ "testName" : "Registration of CM-handles",
+ "unit" : "CM-handles/second",
+ "measurementName" : "cm_handles_created",
+ "currentExpectation" : 100
+ },
+ {
+ "testNumber" : "2",
+ "testName" : "De-registration of CM-handles",
+ "unit" : "CM-handles/second",
+ "measurementName" : "cm_handles_deleted",
+ "currentExpectation" : 180
+ },
+ {
+ "testNumber" : "3a",
+ "testName" : "CM-handle ID search with No filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_id_search_no_filter",
+ "currentExpectation" : 550
+ },
+ {
+ "testNumber" : "3b",
+ "testName" : "CM-handle ID search with Module filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_id_search_module_filter",
+ "currentExpectation" : 2300
+ },
+ {
+ "testNumber" : "3c",
+ "testName" : "CM-handle ID search with Property filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_id_search_property_filter",
+ "currentExpectation" : 1450
+ },
+ {
+ "testNumber" : "3d",
+ "testName" : "CM-handle ID search with Cps Path filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_id_search_cps_path_filter",
+ "currentExpectation" : 1500
+ },
+ {
+ "testNumber" : "3e",
+ "testName" : "CM-handle ID search with Trust Level filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_id_search_trust_level_filter",
+ "currentExpectation" : 1600
+ },
+ {
+ "testNumber" : "4a",
+ "testName" : "CM-handle search with No filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_search_no_filter",
+ "currentExpectation" : 18000
+ },
+ {
+ "testNumber" : "4b",
+ "testName" : "CM-handle search with Module filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_search_module_filter",
+ "currentExpectation" : 18000
+ },
+ {
+ "testNumber" : "4c",
+ "testName" : "CM-handle search with Property filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_search_property_filter",
+ "currentExpectation" : 18000
+ },
+ {
+ "testNumber" : "4d",
+ "testName" : "CM-handle search with Cps Path filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_search_cps_path_filter",
+ "currentExpectation" : 18000
+ },
+ {
+ "testNumber" : "4e",
+ "testName" : "CM-handle search with Trust Level filter",
+ "unit" : "milliseconds",
+ "measurementName" : "cm_handle_search_trust_level_filter",
+ "currentExpectation" : 18000
+ },
+ {
+ "testNumber" : "5b",
+ "testName" : "NCMP overhead for Synchronous single CM-handle pass-through read with alternate id",
+ "unit" : "milliseconds",
+ "measurementName" : "ncmp_read_overhead",
+ "currentExpectation" : 18
+ },
+ {
+ "testNumber" : "6b",
+ "testName" : "NCMP overhead for Synchronous single CM-handle pass-through write with alternate id",
+ "unit" : "milliseconds",
+ "measurementName" : "ncmp_write_overhead",
+ "currentExpectation" : 18
+ },
+ {
+ "testNumber" : "7",
+ "testName" : "Legacy batch read operation",
+ "unit" : "events/second",
+ "measurementName" : "legacy_batch_read",
+ "currentExpectation" : 200
+ },
+ {
+ "testNumber" : "8",
+ "testName" : "Write data job scenario - small",
+ "unit" : "milliseconds",
+ "measurementName" : "dcm_write_data_job_small",
+ "currentExpectation" : 100
+ },
+ {
+ "testNumber" : "9",
+ "testName" : "Write data job scenario - large",
+ "unit" : "milliseconds",
+ "measurementName" : "dcm_write_data_job_large",
+ "currentExpectation" : 8000
+ }
+]
\ No newline at end of file
--- /dev/null
+cm_handles_created
+cm_handles_deleted
+cm_handle_id_search_no_filter
+cm_handle_id_search_module_filter
+cm_handle_id_search_property_filter
+cm_handle_id_search_cps_path_filter
+cm_handle_id_search_trust_level_filter
+cm_handle_search_no_filter
+cm_handle_search_module_filter
+cm_handle_search_property_filter
+cm_handle_search_cps_path_filter
+cm_handle_search_trust_level_filter
+ncmp_read_overhead
+ncmp_write_overhead
+legacy_batch_read
+dcm_write_data_job_small
+dcm_write_data_job_large
\ No newline at end of file
local tmp
tmp=$(mktemp)
- awk -F',' -v OFS=',' '
- NR == 1 { print $0, "Result"; next }
- {
- throughputTests = ($1 == "1" || $1 == "2" || $1 == "7")
- passCondition = throughputTests ? (($6+0) >= ($4+0)) : (($6+0) <= ($4+0))
- print $0, (passCondition ? "✅" : "❌")
+awk -F',' -v OFS=',' '
+ function initRowVariables() {
+ titleRow = $0
+ testNumber = $1
+ fsRequirement = $4 + 0
+ actual = $6 + 0
}
- ' "$summaryFile" > "$tmp"
+
+ NR == 1 { # block for header
+ titleRow = $0
+ print titleRow, "Result"
+ next
+ }
+
+ { # block for every data row
+ initRowVariables()
+ isThroughput = (testNumber=="0" || testNumber=="1" || \
+ testNumber=="2" || testNumber=="7")
+
+ if (actual == 0 && testNumber != "0")
+ pass = 0
+ else if (isThroughput)
+ pass = (actual >= fsRequirement)
+ else
+ pass = (actual <= fsRequirement)
+
+ print titleRow, (pass ? "✅" : "❌")
+ }
+' "$summaryFile" > "$tmp"
mv "$tmp" "$summaryFile"
import { check, sleep } from 'k6';
import { Trend } from 'k6/metrics';
import { Reader } from 'k6/x/kafka';
-import { testConfig,
+import {
+ testConfig,
validateResponseAndRecordMetricWithOverhead,
validateResponseAndRecordMetric,
makeCustomSummaryReport,
import { sendBatchOfKafkaMessages } from './common/produce-avc-event.js';
import { executeWriteDataJob } from "./common/write-data-job.js";
-let cmHandlesCreatedTrend = new Trend('cm_handles_created', false);
-let cmHandlesDeletedTrend = new Trend('cm_handles_deleted', false);
-let cmHandleIdSearchNoFilterTrend = new Trend('cm_handle_id_search_no_filter', true);
-let cmHandleIdSearchModuleFilterTrend = new Trend('cm_handle_id_search_module_filter', true);
-let cmHandleIdSearchPropertyFilterTrend = new Trend('cm_handle_id_search_property_filter', true);
-let cmHandleIdSearchCpsPathFilterTrend = new Trend('cm_handle_id_search_cps_path_filter', true);
-let cmHandleIdSearchTrustLevelFilterTrend = new Trend('cm_handle_id_search_trust_level_filter', true);
-let cmHandleSearchNoFilterTrend = new Trend('cm_handle_search_no_filter', true);
-let cmHandleSearchModuleFilterTrend = new Trend('cm_handle_search_module_filter', true);
-let cmHandleSearchPropertyFilterTrend = new Trend('cm_handle_search_property_filter', true);
-let cmHandleSearchCpsPathFilterTrend = new Trend('cm_handle_search_cps_path_filter', true);
-let cmHandleSearchTrustLevelFilterTrend = new Trend('cm_handle_search_trust_level_filter', true);
-let ncmpReadOverheadTrend = new Trend('ncmp_read_overhead', true);
-let ncmpWriteOverheadTrend = new Trend('ncmp_write_overhead', true);
-let legacyBatchReadTrend = new Trend('legacy_batch_read', false);
-let dcmWriteDataJobSmallTrend = new Trend('dcm_write_data_job_small', true);
-let dcmWriteDataJobLargeTrend = new Trend('dcm_write_data_job_large', true);
+
+const throughputTrends = ['cm_handles_created', 'cm_handles_deleted', 'legacy_batch_read'];
+const kpiTrendDeclarations = {};
+
+for (const trendName of open('./config/trendNames.txt').trim().split('\n')) {
+ const isTimeTrend = !throughputTrends.includes(trendName);
+ kpiTrendDeclarations[trendName] = new Trend(trendName, isTimeTrend);
+}
const EXPECTED_WRITE_RESPONSE_COUNT = 1;
const endTimeInMillis = Date.now();
const totalRegistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
- cmHandlesCreatedTrend.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
+ kpiTrendDeclarations.cm_handles_created.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
}
export function teardown() {
const endTimeInMillis = Date.now();
const totalDeregistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
- cmHandlesDeletedTrend.add(numberOfDeregisteredCmHandles / totalDeregistrationTimeInSeconds);
+ kpiTrendDeclarations.cm_handles_deleted.add(numberOfDeregisteredCmHandles / totalDeregistrationTimeInSeconds);
sleep(CONTAINER_COOL_DOWW_TIME_IN_SECONDS);
}
export function passthroughReadAltIdScenario() {
const response = passthroughRead();
- validateResponseAndRecordMetricWithOverhead(response, 200, 'passthrough read with alternate Id status equals 200', READ_DATA_FOR_CM_HANDLE_DELAY_MS, ncmpReadOverheadTrend);
+ validateResponseAndRecordMetricWithOverhead(response, 200, 'passthrough read with alternate Id status equals 200', READ_DATA_FOR_CM_HANDLE_DELAY_MS, kpiTrendDeclarations.ncmp_read_overhead);
}
export function passthroughWriteAltIdScenario() {
const response = passthroughWrite();
- validateResponseAndRecordMetricWithOverhead(response, 201, 'passthrough write with alternate Id status equals 201', WRITE_DATA_FOR_CM_HANDLE_DELAY_MS, ncmpWriteOverheadTrend);
+ validateResponseAndRecordMetricWithOverhead(response, 201, 'passthrough write with alternate Id status equals 201', WRITE_DATA_FOR_CM_HANDLE_DELAY_MS, kpiTrendDeclarations.ncmp_write_overhead);
}
export function cmHandleIdSearchNoFilterScenario() {
const response = executeCmHandleIdSearch('no-filter');
- validateResponseAndRecordMetric(response, 200, 'CM handle ID no-filter search', TOTAL_CM_HANDLES, cmHandleIdSearchNoFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle ID no-filter search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_id_search_no_filter);
}
export function cmHandleSearchNoFilterScenario() {
const response = executeCmHandleSearch('no-filter');
- validateResponseAndRecordMetric(response, 200, 'CM handle no-filter search', TOTAL_CM_HANDLES, cmHandleSearchNoFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle no-filter search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_search_no_filter);
}
export function cmHandleIdSearchModuleScenario() {
const response = executeCmHandleIdSearch('module');
- validateResponseAndRecordMetric(response, 200, 'CM handle ID module search', TOTAL_CM_HANDLES, cmHandleIdSearchModuleFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle ID module search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_id_search_module_filter);
}
export function cmHandleSearchModuleScenario() {
const response = executeCmHandleSearch('module');
- validateResponseAndRecordMetric(response, 200, 'CM handle module search', TOTAL_CM_HANDLES, cmHandleSearchModuleFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle module search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_search_module_filter);
}
export function cmHandleIdSearchPropertyScenario() {
const response = executeCmHandleIdSearch('property');
- validateResponseAndRecordMetric(response, 200, 'CM handle ID property search', TOTAL_CM_HANDLES, cmHandleIdSearchPropertyFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle ID property search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_id_search_property_filter);
}
export function cmHandleSearchPropertyScenario() {
const response = executeCmHandleSearch('property');
- validateResponseAndRecordMetric(response, 200, 'CM handle property search', TOTAL_CM_HANDLES, cmHandleSearchPropertyFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle property search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_search_property_filter);
}
export function cmHandleIdSearchCpsPathScenario() {
const response = executeCmHandleIdSearch('cps-path-for-ready-cm-handles');
- validateResponseAndRecordMetric(response, 200, 'CM handle ID cps path search', TOTAL_CM_HANDLES, cmHandleIdSearchCpsPathFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle ID cps path search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_id_search_cps_path_filter);
}
export function cmHandleSearchCpsPathScenario() {
const response = executeCmHandleSearch('cps-path-for-ready-cm-handles');
- validateResponseAndRecordMetric(response, 200, 'CM handle cps path search', TOTAL_CM_HANDLES, cmHandleSearchCpsPathFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle cps path search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_search_cps_path_filter);
}
export function cmHandleIdSearchTrustLevelScenario() {
const response = executeCmHandleIdSearch('trust-level');
- validateResponseAndRecordMetric(response, 200, 'CM handle ID trust level search', TOTAL_CM_HANDLES, cmHandleIdSearchTrustLevelFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle ID trust level search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_id_search_trust_level_filter);
}
export function cmHandleSearchTrustLevelScenario() {
const response = executeCmHandleSearch('trust-level');
- validateResponseAndRecordMetric(response, 200, 'CM handle trust level search', TOTAL_CM_HANDLES, cmHandleSearchTrustLevelFilterTrend);
+ validateResponseAndRecordMetric(response, 200, 'CM handle trust level search', TOTAL_CM_HANDLES, kpiTrendDeclarations.cm_handle_search_trust_level_filter);
}
export function legacyBatchProduceScenario() {
const messages = legacyBatchEventReader.consume({ limit: 220, expectTimeout: true });
const timestamp2 = (new Date()).toISOString();
console.debug(`✅ From ${timestamp1} to ${timestamp2} consumed ${messages.length} messages by legacy batch read\``);
- legacyBatchReadTrend.add(messages.length);
+ kpiTrendDeclarations.legacy_batch_read.add(messages.length);
} catch (error) {
const timestamp2 = (new Date()).toISOString();
console.error(`❌ From ${timestamp1} to ${timestamp2} Consume error (legacy batch read): ${error.message}`);
export function writeDataJobLargeScenario() {
const response = executeWriteDataJob(100000);
- validateResponseAndRecordMetric(response, 200, 'Large writeDataJob', EXPECTED_WRITE_RESPONSE_COUNT, dcmWriteDataJobLargeTrend);
+ validateResponseAndRecordMetric(response, 200, 'Large writeDataJob', EXPECTED_WRITE_RESPONSE_COUNT, kpiTrendDeclarations.dcm_write_data_job_large);
}
export function writeDataJobSmallScenario() {
const response = executeWriteDataJob(100);
- validateResponseAndRecordMetric(response, 200, 'Small writeDataJob', EXPECTED_WRITE_RESPONSE_COUNT, dcmWriteDataJobSmallTrend);
+ validateResponseAndRecordMetric(response, 200, 'Small writeDataJob', EXPECTED_WRITE_RESPONSE_COUNT, kpiTrendDeclarations.dcm_write_data_job_small);
}
export function produceAvcEventsScenario() {
Code Review / cps.git / commitdiff