--- /dev/null
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Grafana Dashboard for Policy OPA-PDP",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 6,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 19,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Quick Info",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "description": "Uptime per pod for Policy OPA-PDP application calculated in days",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "decimals": 1,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 1
+ },
+ "id": 4,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "vertical",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "text": {
+ "titleSize": 16
+ },
+ "textMode": "value_and_name",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "(time() - process_start_time_seconds{job=\"opa-pdp-metrics\"})",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "process_uptime: {{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Process Uptime",
+ "transparent": true,
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "description": "Policy OPA-PDP CPU Usage Monitoring",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "decimals": 2,
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "process-1h: dev-policy-api-69f54f45cc-wq6xq"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-purple",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "process_1h: dev-policy-api-69f54f45cc-wq6xq"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-purple",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 1
+ },
+ "id": 26,
+ "options": {
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": false,
+ "sizing": "auto",
+ "text": {}
+ },
+ "pluginVersion": "11.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "exemplar": true,
+ "expr": "irate(process_cpu_seconds_total{job=\"opa-pdp-metrics\"}[$__range]) * 100",
+ "interval": "",
+ "legendFormat": "{{pod}}",
+ "refId": "A"
+ }
+ ],
+ "title": "CPU Usage",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "description": "Policy OPA-PDP Memory Usage Monitoring",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "{area=\"heap\", container=\"policy-api\", endpoint=\"policy-api\", id=\"Tenured Gen\", instance=\"10.42.7.19:6969\", job=\"policy-api\", namespace=\"onap\", pod=\"dev-policy-api-69f54f45cc-sb56v\", service=\"policy-api\"}"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "text",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Tenured Gen - dev-policy-api-69f54f45cc-wq6xq"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-purple",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 12,
+ "y": 1
+ },
+ "id": 27,
+ "options": {
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": false,
+ "sizing": "auto"
+ },
+ "pluginVersion": "11.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "exemplar": true,
+ "expr": "100 * process_resident_memory_bytes{job=\"opa-pdp-metrics\"} / (1024 * 1024 * 1024)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "RSS : {{ pod }}",
+ "refId": "A"
+ }
+ ],
+ "title": "Memory Usage",
+ "type": "gauge"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 7
+ },
+ "id": 17,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "System",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "description": "Policy OPA-PDP CPU Usage Monitoring",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 1,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "process-1h: dev-policy-api-69f54f45cc-wq6xq"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-purple",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "process_1h: dev-policy-api-69f54f45cc-wq6xq"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-purple",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 8,
+ "x": 0,
+ "y": 8
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [
+ "max",
+ "mean",
+ "last"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Max",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "irate(process_cpu_seconds_total{job=\"opa-pdp-metrics\"}[$__range])*100",
+ "interval": "",
+ "legendFormat": "opa: {{pod}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "CPU Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "description": "Policy OPA-PDP Memory Usage Monitoring",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 1,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "{area=\"heap\", container=\"policy-api\", endpoint=\"policy-api\", id=\"Tenured Gen\", instance=\"10.42.7.19:6969\", job=\"policy-api\", namespace=\"onap\", pod=\"dev-policy-api-69f54f45cc-sb56v\", service=\"policy-api\"}"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "text",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Tenured Gen - dev-policy-api-69f54f45cc-wq6xq"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-purple",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 7,
+ "x": 8,
+ "y": 8
+ },
+ "id": 13,
+ "options": {
+ "legend": {
+ "calcs": [
+ "max",
+ "mean",
+ "last"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last",
+ "sortDesc": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "editorMode": "code",
+ "exemplar": true,
+ "expr": "100 * process_resident_memory_bytes{job=\"opa-pdp-metrics\"} / (1024 * 1024 * 1024)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Alloc : {{ pod }}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 21,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Requests",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "description": "Policy Decisions Total for OPA-PDP per instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 27
+ },
+ "id": 30,
+ "options": {
+ "displayMode": "gradient",
+ "maxVizHeight": 300,
+ "minVizHeight": 16,
+ "minVizWidth": 8,
+ "namePlacement": "auto",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "sizing": "auto",
+ "valueMode": "color"
+ },
+ "pluginVersion": "11.1.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "dkSf71fnz"
+ },
+ "exemplar": true,
+ "expr": "opa_decision_response_time_seconds_count{instance=\"policy-opa-pdp:8282\", job=\"opa-pdp-metrics\"}",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Decision Count",
+ "refId": "B"
+ }
+ ],
+ "title": "Policy Decisions",
+ "type": "bargauge"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 39,
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-30m",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Policy OPA-PDP",
+ "uid": "go-U1x9FQmA",
+ "version": 3,
+ "weekStart": ""
+}
--- /dev/null
+*** Settings ***
+Library OperatingSystem
+Resource common-library.robot
+
+*** Test Cases ***
+WaitForPrometheusServer
+ [Documentation] Sleep time to wait for Prometheus server to gather all metrics
+ Sleep 1 minute
+
+ValidateOPAPolicyDecisionsTotalCounter
+ [Documentation] Validate opa policy decision counters using prometheus metrics
+ ValidateOPAPrometheusMetric opa_decision_response_time_seconds_count{instance="policy-opa-pdp:8282", job="opa-pdp-metrics"} 9
+
+ValidateOPAPolicyDataTotalCounter
+ [Documentation] Validate opa policy data counters using prometheus metrics
+ ValidateOPAPrometheusMetric opa_data_response_time_seconds_count{instance="policy-opa-pdp:8282", job="opa-pdp-metrics"} 12
+
+ValidateOPADecisionAverageResponseTime
+ [Documentation] Ensure average response time is less than 10ms
+ ValidateOPADecisionAverageResponseTimeMetric 1.5
+
+ValidateOPADataAverageResponseTime
+ [Documentation] Ensure average response time is less than 10ms
+ ValidateOPADataAverageResponseTimeMetric 1.5
+
+
+*** Keywords ***
+ValidateOPAPrometheusMetric
+ [Arguments] ${url} ${expectedLimit}
+ [Documentation] Check the policy decision/data execution count
+ ${resp}= QueryPrometheus ${url}
+ ${actualValue}= Evaluate ${resp['data']['result'][0]['value'][1]}
+ Should Be True ${actual_value} == ${expectedLimit}
+
+ValidateOPADecisionAverageResponseTimeMetric
+ [Arguments] ${threshold}
+ [Documentation] Validate that the average response time is below the threshold
+
+ ${sum_resp}= QueryPrometheus opa_decision_response_time_seconds_sum{instance="policy-opa-pdp:8282", job="opa-pdp-metrics"}
+ ${count_resp}= QueryPrometheus opa_decision_response_time_seconds_count{instance="policy-opa-pdp:8282", job="opa-pdp-metrics"}
+
+ ${sum_value}= Evaluate ${sum_resp['data']['result'][0]['value'][1]}
+ ${count_value}= Evaluate ${count_resp['data']['result'][0]['value'][1]}
+
+ ${avg_response_time}= Evaluate float(${sum_value}) / float(${count_value})
+ Should Be True ${avg_response_time} < ${threshold} msg=Average response time exceeded ${threshold}
+
+
+ValidateOPADataAverageResponseTimeMetric
+ [Arguments] ${threshold}
+ [Documentation] Validate that the average response time is below the threshold
+
+ ${sum_resp}= QueryPrometheus opa_data_response_time_seconds_sum{instance="policy-opa-pdp:8282", job="opa-pdp-metrics"}
+ ${count_resp}= QueryPrometheus opa_data_response_time_seconds_count{instance="policy-opa-pdp:8282", job="opa-pdp-metrics"}
+
+ ${sum_value}= Evaluate ${sum_resp['data']['result'][0]['value'][1]}
+ ${count_value}= Evaluate ${count_resp['data']['result'][0]['value'][1]}
+
+ ${avg_response_time}= Evaluate float(${sum_value}) / float(${count_value})
+ Should Be True ${avg_response_time} < ${threshold} msg=Average response time exceeded ${threshold}