From 66254ca1c5df0d7971764799088c8f20c79f4ca7 Mon Sep 17 00:00:00 2001 From: Remigiusz Janeczek Date: Thu, 23 Apr 2020 08:02:47 +0200 Subject: [PATCH] Add metric for total latency without routing Add metric for time between Producer and HV-VES output without sending to Kafka Refactor metrics test Add new latencies and individual cores usage to Grafana Issue-ID: DCAEGEN2-1576 Signed-off-by: Remigiusz Janeczek Change-Id: I6112db76be1c7108c18336b50f9f12d5ce62c24a --- .../dcae/collectors/veshv/tests/fakes/metrics.kt | 2 + .../veshv/main/metrics/MicrometerMetrics.kt | 8 +- .../collectors/veshv/main/MicrometerMetricsTest.kt | 98 +- .../cloud/grafana/dashboards/k8s-metrics.json | 1101 +++++++++++--------- .../cloud/grafana/dashboards/latencies.json | 75 +- .../cloud/grafana/dashboards/processing.json | 2 +- 6 files changed, 735 insertions(+), 551 deletions(-) diff --git a/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt b/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt index 12555965..4a5474c6 100644 --- a/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt +++ b/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt @@ -42,6 +42,7 @@ class FakeMetrics : Metrics { var lastProcessingTimeMicros: Double = -1.0; private set var lastProcessingTimeWithoutRoutingMicros: Double = -1.0; private set var lastToCollectorTravelTime: Double = -1.0; private set + var lastTotalLatencyWithoutRouting: Double = -1.0; private set var messagesSentCount: Int = 0; private set var clientRejectionCause = mutableMapOf(); private set @@ -63,6 +64,7 @@ class FakeMetrics : Metrics { override fun notifyMessageReadyForRouting(msg: VesMessage) { lastProcessingTimeWithoutRoutingMicros = Duration.between(msg.wtpFrame.receivedAt, Instant.now()).toNanos() / 1000.0 + lastTotalLatencyWithoutRouting = Duration.between(TimeUtils.epochMicroToInstant(msg.header.lastEpochMicrosec), Instant.now()).toNanos() / 1000.0 } override fun notifyMessageSent(msg: RoutedMessage) { diff --git a/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt b/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt index e0d99fc6..a949803f 100644 --- a/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt +++ b/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt @@ -66,6 +66,10 @@ class MicrometerMetrics internal constructor( .maximumExpectedValue(MAX_BUCKET_DURATION) .publishPercentileHistogram(true) .register(registry) + private val totalLatencyWithoutRouting = Timer.builder(name(MESSAGES, LATENCY, WITHOUT, ROUTING)) + .maximumExpectedValue(MAX_BUCKET_DURATION) + .publishPercentileHistogram(true) + .register(registry) private val totalLatency = Timer.builder(name(MESSAGES, LATENCY)) .maximumExpectedValue(MAX_BUCKET_DURATION) .publishPercentileHistogram(true) @@ -104,7 +108,9 @@ class MicrometerMetrics internal constructor( } override fun notifyMessageReadyForRouting(msg: VesMessage) { - processingTimeWithoutRouting.record(Duration.between(msg.wtpFrame.receivedAt, Instant.now())) + val now = Instant.now() + processingTimeWithoutRouting.record(Duration.between(msg.wtpFrame.receivedAt, now)) + totalLatencyWithoutRouting.record(Duration.between(epochMicroToInstant(msg.header.lastEpochMicrosec), now)) } override fun notifyMessageReceived(msg: WireFrameMessage) { diff --git a/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt b/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt index efd353ec..dd206d08 100644 --- a/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt +++ b/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt @@ -146,8 +146,66 @@ object MicrometerMetricsTest : Spek({ "$PREFIX.messages.received.payload.bytes" ) } + + on("$PREFIX.messages.to.collector.travel.time") { + val counterName = "$PREFIX.messages.to.collector.travel.time" + val toCollectorTravelTimeMs = 100L + + it("should update timer") { + val now = Instant.now() + val vesMessage = vesMessageReceivedAt(now, sentAt = now.minusMillis(toCollectorTravelTimeMs)) + cut.notifyMessageReceived(vesMessage) + + registry.verifyTimer(counterName) { timer -> + assertThat(timer.mean(TimeUnit.MILLISECONDS)).isEqualTo(toCollectorTravelTimeMs.toDouble()) + } + + verifyCountersAndTimersAreUnchangedBut(counterName) + } + } } + describe("notifyMessageReadyForRouting"){ + on("$PREFIX.messages.processing.time.without.routing") { + val counterName = "$PREFIX.messages.processing.time.without.routing" + val processingTimeMs = 100L + + it("should update timer") { + + cut.notifyMessageReadyForRouting(vesMessageReceivedAt(Instant.now().minusMillis(processingTimeMs))) + + registry.verifyTimer(counterName) { timer -> + assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(processingTimeMs.toDouble()) + } + verifyCountersAndTimersAreUnchangedBut( + counterName, + "$PREFIX.messages.latency.without.routing" + ) + } + } + + on("$PREFIX.messages.latency.without.routing") { + val counterName = "$PREFIX.messages.latency.without.routing" + val latencyWithoutRoutingMs = 200L + + it("should update timer") { + + val sentAt = Instant.now().minusMillis(latencyWithoutRoutingMs) + + cut.notifyMessageReadyForRouting(vesMessageSentAt(sentAt)) + + registry.verifyTimer(counterName) { timer -> + assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(latencyWithoutRoutingMs.toDouble()) + } + verifyCountersAndTimersAreUnchangedBut( + counterName, + "$PREFIX.messages.processing.time.without.routing" + ) + } + } + } + + describe("notifyMessageSent") { val topicName1 = "PERF3GPP" val topicName2 = "CALLTRACE" @@ -206,39 +264,6 @@ object MicrometerMetricsTest : Spek({ } } - on("$PREFIX.messages.to.collector.travel.time") { - val counterName = "$PREFIX.messages.to.collector.travel.time" - val toCollectorTravelTimeMs = 100L - - it("should update timer") { - val now = Instant.now() - val vesMessage = vesMessageReceivedAt(now, sentAt = now.minusMillis(toCollectorTravelTimeMs)) - cut.notifyMessageReceived(vesMessage) - - registry.verifyTimer(counterName) { timer -> - assertThat(timer.mean(TimeUnit.MILLISECONDS)).isEqualTo(toCollectorTravelTimeMs.toDouble()) - } - - verifyCountersAndTimersAreUnchangedBut(counterName) - } - } - - on("$PREFIX.messages.processing.time.without.routing") { - val counterName = "$PREFIX.messages.processing.time.without.routing" - val processingTimeMs = 100L - - it("should update timer") { - - cut.notifyMessageReadyForRouting(vesMessageReceivedAt(Instant.now().minusMillis(processingTimeMs))) - - registry.verifyTimer(counterName) { timer -> - assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(processingTimeMs.toDouble()) - } - - verifyCountersAndTimersAreUnchangedBut(counterName) - } - } - on("$PREFIX.messages.latency") { val counterName = "$PREFIX.messages.latency" val latencyMs = 1666L @@ -398,6 +423,13 @@ object MicrometerMetricsTest : Spek({ } }) +private fun vesMessageSentAt(sentAt: Instant): VesMessage { + val lastEpochMicrosec = sentAt.epochSecond * 1000000 + sentAt.nano / 1000 + val commonHeader = commonHeader(lastEpochMicrosec = lastEpochMicrosec) + return VesMessage(commonHeader, + wireProtocolFrame(commonHeader, ByteString.copyFromUtf8("highvolume measurements"))) +} + private fun vesMessageReceivedAt(receivedAt: Instant, sentAt: Instant): VesMessage { val lastEpochMicrosec = sentAt.epochSecond * 1000000 + sentAt.nano / 1000 val commonHeader = commonHeader(lastEpochMicrosec = lastEpochMicrosec) diff --git a/tools/performance/cloud/grafana/dashboards/k8s-metrics.json b/tools/performance/cloud/grafana/dashboards/k8s-metrics.json index 676d0a93..7841f77d 100644 --- a/tools/performance/cloud/grafana/dashboards/k8s-metrics.json +++ b/tools/performance/cloud/grafana/dashboards/k8s-metrics.json @@ -1,530 +1,619 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 5, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 8, - "options": { - "fieldOptions": { - "calcs": [ - "mean" - ], - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "id": 6, + "options": { + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "percent" - }, - "overrides": [], - "values": false + { + "color": "red", + "value": 80 + } + ] }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true + "unit": "percent" }, - "pluginVersion": "6.7.2", - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{id=\"/\", instance=~\".*worker.*\"}) by(instance) / (sum(machine_memory_bytes{instance=~\".*worker.*\"}) by (instance))*100", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Nodes RAM Usage", - "type": "gauge" + "overrides": [], + "values": false }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.7.2", + "targets": [ { - "datasource": null, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 6, - "options": { - "fieldOptions": { - "calcs": [ - "mean" - ], - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [], - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.7.2", - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", instance=~\".*worker.*\"}[1m])) by(instance) / (sum(machine_cpu_cores{instance=~\".*worker.*\"}) by (instance))*100", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Nodes CPU Usage", - "type": "gauge" + "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", instance=~\".*worker.*\"}[1m])) by(instance) / (sum(machine_cpu_cores{instance=~\".*worker.*\"}) by (instance))*100", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Nodes CPU Usage", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[1m])) by(instance) / (sum(machine_cpu_cores) by (instance))*100", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": "20m", + "timeRegions": [], + "timeShift": null, + "title": "Nodes Total CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 12, - "w": 8, - "x": 16, - "y": 0 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "C" - }, - { - "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))", - "hide": false, - "interval": "", - "legendFormat": "Total Receive", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Network Usage - Receive", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "C" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 16, - "w": 8, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{id=\"/\"}) by (instance) / (sum(machine_memory_bytes) by (instance))*100", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Nodes Total RAM Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "Total Receive", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": "20m", + "timeRegions": [], + "timeShift": null, + "title": "Network Usage - Receive", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 16, - "w": 8, - "x": 8, - "y": 7 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[1m])) by(instance) / (sum(machine_cpu_cores) by (instance))*100", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 8, + "options": { + "fieldOptions": { + "calcs": [ + "mean" ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Nodes Total CPU usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "unit": "percent" + }, + "overrides": [], + "values": false }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.7.2", + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 8, - "x": 16, - "y": 12 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - }, - { - "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))", - "hide": false, - "interval": "", - "legendFormat": "Total Transmit", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Network Usage - Transmit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "sum(container_memory_working_set_bytes{id=\"/\", instance=~\".*worker.*\"}) by(instance) / (sum(machine_memory_bytes{instance=~\".*worker.*\"}) by (instance))*100", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Nodes RAM Usage", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 8, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{id=\"/\"}) by (instance) / (sum(machine_memory_bytes) by (instance))*100", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": "20m", + "timeRegions": [], + "timeShift": null, + "title": "Nodes Total RAM Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100- sum(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) by (node_name, cpu) * 100", + "interval": "", + "legendFormat": "Instance: {{node_name}} cpu: {{cpu}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": "20m", + "timeRegions": [], + "timeShift": null, + "title": "Usage of each core", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "refresh": "5s", - "schemaVersion": 22, - "style": "dark", - "tags": [], - "templating": { - "list": [] + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "time": { - "from": "now-6h", - "to": "now" + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "Total Transmit", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": "20m", + "timeRegions": [], + "timeShift": null, + "title": "Network Usage - Transmit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "timezone": "", - "title": "K8s metrics", - "uid": "RxjnWpjZk", - "variables": { - "list": [] + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "version": 1 + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } + ], + "refresh": "5s", + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "K8s metrics", + "uid": "RxjnWpjZk", + "variables": { + "list": [] + }, + "version": 1 +} diff --git a/tools/performance/cloud/grafana/dashboards/latencies.json b/tools/performance/cloud/grafana/dashboards/latencies.json index 43ec7f4f..9efe8485 100644 --- a/tools/performance/cloud/grafana/dashboards/latencies.json +++ b/tools/performance/cloud/grafana/dashboards/latencies.json @@ -15,7 +15,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 6, + "id": 5, "links": [], "panels": [ { @@ -33,8 +33,10 @@ "y": 0 }, "hiddenSeries": false, - "id": 2, + "id": 8, + "interval": "", "legend": { + "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, @@ -47,12 +49,13 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -61,10 +64,32 @@ "steppedLine": false, "targets": [ { - "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])", + "expr": "rate(hvves_messages_to_collector_travel_time_seconds_sum[1h])/rate(hvves_messages_to_collector_travel_time_seconds_count[1h])", + "format": "time_series", + "hide": false, + "instant": false, "interval": "", + "intervalFactor": 1, "legendFormat": "average", "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )", + "interval": "", + "legendFormat": "90th percentile", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )", + "interval": "", + "legendFormat": "95th percentile", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )", + "interval": "", + "legendFormat": "99th precentile", + "refId": "D" } ], "thresholds": [], @@ -87,6 +112,7 @@ }, "yaxes": [ { + "decimals": 2, "format": "s", "label": null, "logBase": 1, @@ -123,8 +149,10 @@ "y": 0 }, "hiddenSeries": false, - "id": 7, + "id": 9, + "interval": "", "legend": { + "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, @@ -137,12 +165,13 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -151,10 +180,32 @@ "steppedLine": false, "targets": [ { - "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h]) + rate(hvves_messages_processing_time_without_routing_seconds_sum[1h])/rate(hvves_messages_processing_time_without_routing_seconds_count[1h])", + "expr": "rate(hvves_messages_latency_without_routing_seconds_sum[1h])/rate(hvves_messages_latency_without_routing_seconds_count[1h])", + "format": "time_series", + "hide": false, + "instant": false, "interval": "", + "intervalFactor": 1, "legendFormat": "average", "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )", + "interval": "", + "legendFormat": "90th percentile", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )", + "interval": "", + "legendFormat": "95th percentile", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )", + "interval": "", + "legendFormat": "99th precentile", + "refId": "D" } ], "thresholds": [], @@ -177,6 +228,7 @@ }, "yaxes": [ { + "decimals": 2, "format": "s", "label": null, "logBase": 1, @@ -353,19 +405,22 @@ "steppedLine": false, "targets": [ { - "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])", + "expr": "rate(hvves_messages_to_collector_travel_time_seconds_sum[1h])/rate(hvves_messages_to_collector_travel_time_seconds_count[1h])", + "hide": false, "interval": "", "legendFormat": "Producer to HV-VES input", "refId": "A" }, { - "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h]) + rate(hvves_messages_processing_time_without_routing_seconds_sum[1h])/rate(hvves_messages_processing_time_without_routing_seconds_count[1h])", + "expr": "rate(hvves_messages_latency_without_routing_seconds_sum[1h])/rate(hvves_messages_latency_without_routing_seconds_count[1h])", + "hide": false, "interval": "", "legendFormat": "Producer to HV-VES output (without sending to Kafka)", "refId": "B" }, { "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h])", + "hide": false, "interval": "", "legendFormat": "Producer to HV-VES output (with sending to Kafka)", "refId": "C" @@ -444,5 +499,5 @@ "variables": { "list": [] }, - "version": 0 + "version": 1 } diff --git a/tools/performance/cloud/grafana/dashboards/processing.json b/tools/performance/cloud/grafana/dashboards/processing.json index 7b606e44..f8912578 100644 --- a/tools/performance/cloud/grafana/dashboards/processing.json +++ b/tools/performance/cloud/grafana/dashboards/processing.json @@ -660,7 +660,7 @@ } ], "thresholds": [], - "timeFrom": null, + "timeFrom": "20m", "timeRegions": [], "timeShift": null, "title": "Memory usage of HV-VES", -- 2.16.6