Update apex-pdp CSIT tests to include metrics

Validate metrics as documented in
https://wiki.onap.org/display/DW/Policy+Framework%3A+Key+metrics+to+monitor

Issue-ID: POLICY-3766
Signed-off-by: Rashmi Pujar <rashmi.pujar1@bell.ca>
Change-Id: Iac1c0298c8e9a35c5ef25aec15e3b9bcbabebf7e

diff --git a/csit/apex-pdp/tests/apex-pdp-test.robot b/csit/apex-pdp/tests/apex-pdp-test.robot
index ffa501d..b0e4c72 100644 (file)
--- a/csit/apex-pdp/tests/apex-pdp-test.robot
+++ b/csit/apex-pdp/tests/apex-pdp-test.robot
@@ -16,12 +16,6 @@ Healthcheck
      Should Be Equal As Strings    ${resp.json()['code']}    200
      Set Suite Variable    ${pdpName}    ${resp.json()['name']}
 
-Metrics
-     [Documentation]  Verify policy-apex-pdp is exporting prometheus metrics
-     ${hcauth}=  HealthCheckAuth
-     ${resp}=  PerformGetRequest  ${APEX_IP}  /metrics  200  null  ${hcauth}
-     Should Contain  ${resp.text}  jvm_threads_current
-
 ExecuteApexSampleDomainPolicy
      Set Test Variable    ${policyName}    onap.policies.native.apex.Sampledomain
      ${postjson}=  Get file  ${CURDIR}/data/${policyName}.json
@@ -65,6 +59,26 @@ ExecuteApexTestPnfPolicyWithMetadataSet
       Should Be Equal As Integers    ${result.rc}    0
       Wait Until Keyword Succeeds    2 min    5 sec    TriggerAndVerifyTestPnfPolicy
 
+Metrics
+     [Documentation]  Verify policy-apex-pdp is exporting prometheus metrics
+     ${auth}=  HealthCheckAuth
+     ${resp}=  PerformGetRequest  ${APEX_IP}  /metrics  200  null  ${auth}
+     Should Contain  ${resp.text}  pdpa_policy_deployments_total{operation="deploy",status="TOTAL",} 4.0
+     Should Contain  ${resp.text}  pdpa_policy_deployments_total{operation="deploy",status="SUCCESS",} 4.0
+     Should Contain  ${resp.text}  pdpa_policy_executions_total{status="SUCCESS",} 3.0
+     Should Contain  ${resp.text}  pdpa_policy_executions_total{status="TOTAL",} 3.0
+     Should Match  ${resp.text}  *pdpa_engine_event_executions{engine_instance_id="NSOApexEngine-*:0.0.1",}*
+     Should Match  ${resp.text}  *pdpa_engine_event_executions{engine_instance_id="MyApexEngine-*:0.0.1",}*
+     Should Match  ${resp.text}  *pdpa_engine_state{engine_instance_id=*,} 2.0*
+     Should Contain  ${resp.text}  pdpa_engine_event_executions
+     Should Contain  ${resp.text}  pdpa_engine_average_execution_time_seconds
+     Should Contain  ${resp.text}  pdpa_engine_last_execution_time_bucket
+     Should Contain  ${resp.text}  pdpa_engine_last_execution_time_count
+     Should Contain  ${resp.text}  pdpa_engine_last_execution_time_sum
+     Should Match  ${resp.text}  *pdpa_engine_last_start_timestamp_epoch{engine_instance_id="NSOApexEngine-*:0.0.1",}*E12*
+     Should Match  ${resp.text}  *pdpa_engine_last_start_timestamp_epoch{engine_instance_id="MyApexEngine-*:0.0.1",}*E12*
+     Should Contain  ${resp.text}  jvm_threads_current
+
 *** Keywords ***
 
 DeployPolicy

diff --git a/csit/apex-pdp/tests/data/onap.pnf.metadataSet.Test.json b/csit/apex-pdp/tests/data/onap.pnf.metadataSet.Test.json
index c59f695..b59efc0 100644 (file)
--- a/csit/apex-pdp/tests/data/onap.pnf.metadataSet.Test.json
+++ b/csit/apex-pdp/tests/data/onap.pnf.metadataSet.Test.json
@@ -1232,7 +1232,8 @@
                               }
                             }
                           ]
-                        }
+                        },
+                        "toscaPolicyState": "EXIT_FAILURE"
                       },
                       "outputEvents": {
                         "entry": [
@@ -1293,7 +1294,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           }
                         ]
@@ -1379,7 +1381,8 @@
                               }
                             }
                           ]
-                        }
+                        },
+                        "toscaPolicyState": "EXIT_SUCCESS"
                       },
                       "outputEvents": {
                         "entry": [
@@ -1440,7 +1443,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           }
                         ]
@@ -1493,7 +1497,8 @@
                               }
                             }
                           ]
-                        }
+                        },
+                        "toscaPolicyState": "EXIT_FAILURE"
                       },
                       "outputEvents": {
                         "entry": [
@@ -1554,7 +1559,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           }
                         ]
@@ -1706,7 +1712,8 @@
                               }
                             }
                           ]
-                        }
+                        },
+                        "toscaPolicyState": ""
                       },
                       "outputEvents": {
                         "entry": [
@@ -1756,7 +1763,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           },
                           {
@@ -1816,7 +1824,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           }
                         ]
@@ -1869,7 +1878,8 @@
                               }
                             }
                           ]
-                        }
+                        },
+                        "toscaPolicyState": "ENTRY"
                       },
                       "outputEvents": {
                         "entry": [
@@ -1897,7 +1907,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           },
                           {
@@ -1957,7 +1968,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             }
                           }
                         ]
@@ -2016,7 +2028,8 @@
                           }
                         ]
                       }
-                    }
+                    },
+                    "toscaPolicyState": "EXIT_FAILURE"
                   },
                   {
                     "key": {
@@ -2045,7 +2058,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": ""
                     }
                   },
                   {
@@ -2174,7 +2188,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": ""
                     }
                   },
                   {
@@ -2237,7 +2252,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": "EXIT_FAILURE"
                     }
                   },
                   {
@@ -2289,7 +2305,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": ""
                     }
                   },
                   {
@@ -2352,7 +2369,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": "EXIT_SUCCESS"
                     }
                   },
                   {
@@ -2415,7 +2433,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": ""
                     }
                   },
                   {
@@ -2445,7 +2464,8 @@
                             }
                           }
                         ]
-                      }
+                      },
+                      "toscaPolicyState": "ENTRY"
                     }
                   }
                 ]
@@ -2732,4 +2752,4 @@
       }
     }
   }
-}
\ No newline at end of file
+}

diff --git a/csit/apex-pdp/tests/data/onap.policies.apex.pnf.Test.json b/csit/apex-pdp/tests/data/onap.policies.apex.pnf.Test.json
index e64e0c9..5d9cbf9 100644 (file)
--- a/csit/apex-pdp/tests/data/onap.policies.apex.pnf.Test.json
+++ b/csit/apex-pdp/tests/data/onap.policies.apex.pnf.Test.json
@@ -1254,7 +1254,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": "EXIT_FAILURE"
                             },
                             "outputEvents": {
                               "entry": [
@@ -1315,7 +1316,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 }
                               ]
@@ -1401,7 +1403,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": "EXIT_SUCCESS"
                             },
                             "outputEvents": {
                               "entry": [
@@ -1462,7 +1465,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 }
                               ]
@@ -1515,7 +1519,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": "EXIT_FAILURE"
                             },
                             "outputEvents": {
                               "entry": [
@@ -1576,7 +1581,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 }
                               ]
@@ -1728,7 +1734,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": ""
                             },
                             "outputEvents": {
                               "entry": [
@@ -1778,7 +1785,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 },
                                 {
@@ -1838,7 +1846,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 }
                               ]
@@ -1891,7 +1900,8 @@
                                     }
                                   }
                                 ]
-                              }
+                              },
+                              "toscaPolicyState": "ENTRY"
                             },
                             "outputEvents": {
                               "entry": [
@@ -1919,7 +1929,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 },
                                 {
@@ -1979,7 +1990,8 @@
                                           }
                                         }
                                       ]
-                                    }
+                                    },
+                                    "toscaPolicyState": ""
                                   }
                                 }
                               ]
@@ -2038,7 +2050,8 @@
                                 }
                               ]
                             }
-                          }
+                          },
+                          "toscaPolicyState": "EXIT_FAILURE"
                         },
                         {
                           "key": {
@@ -2067,7 +2080,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2196,7 +2210,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2259,7 +2274,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "EXIT_FAILURE"
                           }
                         },
                         {
@@ -2311,7 +2327,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2374,7 +2391,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "EXIT_SUCCESS"
                           }
                         },
                         {
@@ -2437,7 +2455,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2467,7 +2486,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "ENTRY"
                           }
                         }
                       ]

diff --git a/csit/apex-pdp/tests/data/onap.policies.apex.vnf.Test.json b/csit/apex-pdp/tests/data/onap.policies.apex.vnf.Test.json
index 90684c4..d000de7 100644 (file)
--- a/csit/apex-pdp/tests/data/onap.policies.apex.vnf.Test.json
+++ b/csit/apex-pdp/tests/data/onap.policies.apex.vnf.Test.json
@@ -2164,7 +2164,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "EXIT_FAILURE"
                           }
                         },
                         {
@@ -2194,7 +2195,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2356,7 +2358,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2419,7 +2422,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "EXIT_FAILURE"
                           }
                         },
                         {
@@ -2471,7 +2475,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2534,7 +2539,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2597,7 +2603,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "EXIT_FAILURE"
                           }
                         },
                         {
@@ -2649,7 +2656,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2712,7 +2720,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "EXIT_SUCCESS"
                           }
                         },
                         {
@@ -2742,7 +2751,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": ""
                           }
                         },
                         {
@@ -2772,7 +2782,8 @@
                                   }
                                 }
                               ]
-                            }
+                            },
+                            "toscaPolicyState": "ENTRY"
                           }
                         }
                       ]