Integration tests report enrichment

Improve the step description for better reporting
Add duration step in pythonsdk-test reporting

Issue-ID: TEST-271
Issue-ID: TEST-272

Signed-off-by: Michal Jagiello <michal.jagiello@t-mobile.pl>
Change-Id: I6d46cb38ae236bc578eb15982c2c0b8f2b0c0791

diff --git a/src/onaptests/steps/base.py b/src/onaptests/steps/base.py
index 57217fb..8d70405 100644 (file)
--- a/src/onaptests/steps/base.py
+++ b/src/onaptests/steps/base.py
@@ -1,12 +1,13 @@
 import logging
 import logging.config
+import time
 
 from abc import ABC, abstractmethod
 from typing import List
 from onapsdk.configuration import settings
 from onapsdk.aai.business import Customer
 
-from .reports_collection import ReportsCollection
+from .reports_collection import Report, ReportsCollection, ReportStepStatus
 
 
 class BaseStep(ABC):
@@ -93,16 +94,50 @@ class BaseStep(ABC):
         """Step name."""
         return self.__class__.__name__
 
+    @property
+    @abstractmethod
+    def description(self) -> str:
+        """Step description.
+
+        Used for reports
+
+        Returns:
+            str: Step description
+
+        """
+
+    @property
+    @abstractmethod
+    def component(self) -> str:
+        """Component name.
+
+        Name of component which step is related with.
+            Most is the name of ONAP component.
+
+        Returns:
+            str: Component name
+
+        """
+
     @classmethod
     def store_state(cls, fun):
         def wrapper(self, *args, **kwargs):
             try:
+                start_time: float = time.time()
                 ret = fun(self, *args, **kwargs)
-                self.reports_collection.put({self.name: "PASS"})
+                execution_status: ReportStepStatus = ReportStepStatus.PASS
                 return ret
             except Exception:
-                self.reports_collection.put({self.name: "FAIL"})
+                execution_status: ReportStepStatus = ReportStepStatus.FAIL
                 raise
+            finally:
+                self.reports_collection.put(
+                    Report(
+                        step_description=f"[{self.component}] {self.name}: {self.description}",
+                        step_execution_status=execution_status,
+                        step_execution_duration=time.time() - start_time
+                    )
+                )
         return wrapper
 
     def execute(self) -> None:

diff --git a/src/onaptests/steps/cloud/complex_create.py b/src/onaptests/steps/cloud/complex_create.py
index b4d110d..1bb7e51 100644 (file)
--- a/src/onaptests/steps/cloud/complex_create.py
+++ b/src/onaptests/steps/cloud/complex_create.py
@@ -7,6 +7,16 @@ from ..base import BaseStep
 class ComplexCreateStep(BaseStep):
     """Complex creation step."""
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Create complex."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "AAI"
+
     @BaseStep.store_state
     def execute(self):
         """Create complex.

diff --git a/src/onaptests/steps/cloud/connect_service_subscription_to_cloud_region.py b/src/onaptests/steps/cloud/connect_service_subscription_to_cloud_region.py
index 1d0dbfb..97a7823 100644 (file)
--- a/src/onaptests/steps/cloud/connect_service_subscription_to_cloud_region.py
+++ b/src/onaptests/steps/cloud/connect_service_subscription_to_cloud_region.py
@@ -28,6 +28,16 @@ class ConnectServiceSubToCloudRegionStep(BaseStep):
         self.add_step(LinkCloudRegionToComplexStep(cleanup=cleanup))
         self.add_step(CustomerServiceSubscriptionCreateStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Connect service subscription with cloud region."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "AAI"
+
     @BaseStep.store_state
     def execute(self):
         """Connect service subsription to cloud region and tenant.

diff --git a/src/onaptests/steps/cloud/customer_create.py b/src/onaptests/steps/cloud/customer_create.py
index 3c0ef11..b99fdb1 100644 (file)
--- a/src/onaptests/steps/cloud/customer_create.py
+++ b/src/onaptests/steps/cloud/customer_create.py
@@ -7,6 +7,16 @@ from ..base import BaseStep
 class CustomerCreateStep(BaseStep):
     """Customer creation step."""
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Create customer."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "AAI"
+
     @BaseStep.store_state
     def execute(self):
         """Create cutomer.

diff --git a/src/onaptests/steps/cloud/customer_service_subscription_create.py b/src/onaptests/steps/cloud/customer_service_subscription_create.py
index 707d79b..b318dc7 100644 (file)
--- a/src/onaptests/steps/cloud/customer_service_subscription_create.py
+++ b/src/onaptests/steps/cloud/customer_service_subscription_create.py
@@ -18,6 +18,16 @@ class CustomerServiceSubscriptionCreateStep(BaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(CustomerCreateStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Create customer's service subscription."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "AAI"
+
     @BaseStep.store_state
     def execute(self):
         """Create customer service subsription.

diff --git a/src/onaptests/steps/cloud/k8s_connectivity_info_create.py b/src/onaptests/steps/cloud/k8s_connectivity_info_create.py
index 6106d7e..279ae0d 100644 (file)
--- a/src/onaptests/steps/cloud/k8s_connectivity_info_create.py
+++ b/src/onaptests/steps/cloud/k8s_connectivity_info_create.py
@@ -6,6 +6,16 @@ from ..base import BaseStep
 class K8SConnectivityInfoStep(BaseStep):
     """CreateConnnectivityInfoStep."""
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Create K8S connectivity info."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "K8S plugin"
+
     @BaseStep.store_state
     def execute(self):
         """Creation k8s connectivity information

diff --git a/src/onaptests/steps/cloud/link_cloud_to_complex.py b/src/onaptests/steps/cloud/link_cloud_to_complex.py
index 22c4aac..fcfa711 100644 (file)
--- a/src/onaptests/steps/cloud/link_cloud_to_complex.py
+++ b/src/onaptests/steps/cloud/link_cloud_to_complex.py
@@ -18,6 +18,16 @@ class LinkCloudRegionToComplexStep(BaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(ComplexCreateStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Connect cloud region with complex."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "AAI"
+
     @BaseStep.store_state
     def execute(self):
         """Link cloud region to complex.

diff --git a/src/onaptests/steps/cloud/register_cloud.py b/src/onaptests/steps/cloud/register_cloud.py
index 8931847..341fc1c 100644 (file)
--- a/src/onaptests/steps/cloud/register_cloud.py
+++ b/src/onaptests/steps/cloud/register_cloud.py
@@ -10,6 +10,16 @@ from ..base import BaseStep
 class RegisterCloudRegionStep(BaseStep):
     """Cloud region registration step."""
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Register cloud region."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "AAI"
+
     @BaseStep.store_state
     def execute(self):
         """Register cloud region

diff --git a/src/onaptests/steps/instantiate/k8s_profile_create.py b/src/onaptests/steps/instantiate/k8s_profile_create.py
index 0bc6c2d..a875f11 100644 (file)
--- a/src/onaptests/steps/instantiate/k8s_profile_create.py
+++ b/src/onaptests/steps/instantiate/k8s_profile_create.py
@@ -24,6 +24,16 @@ class K8SProfileStep(BaseStep):
         self._service_instance: ServiceInstance = None
         self.add_step(YamlTemplateVnfAlaCarteInstantiateStep(cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Create K8S profile."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "K8S plugin"
+
     @property
     def yaml_template(self) -> dict:
         """Step YAML template.

diff --git a/src/onaptests/steps/instantiate/service_ala_carte.py b/src/onaptests/steps/instantiate/service_ala_carte.py
index 47fc532..d5e5096 100644 (file)
--- a/src/onaptests/steps/instantiate/service_ala_carte.py
+++ b/src/onaptests/steps/instantiate/service_ala_carte.py
@@ -31,6 +31,16 @@ class ServiceAlaCarteInstantiateStep(BaseStep):
             self.add_step(ServiceOnboardStep(cleanup))
             self.add_step(ConnectServiceSubToCloudRegionStep(cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Instantiate service using SO a'la carte method."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SO"
+
     @BaseStep.store_state
     def execute(self):
         """Instantiate service.
@@ -92,6 +102,16 @@ class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep):
             self.add_step(YamlTemplateServiceOnboardStep(cleanup))
             self.add_step(ConnectServiceSubToCloudRegionStep(cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Instantiate service described in YAML using SO a'la carte method."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SO"
+
     @property
     def yaml_template(self) -> dict:
         """Step YAML template.

diff --git a/src/onaptests/steps/instantiate/vf_module_ala_carte.py b/src/onaptests/steps/instantiate/vf_module_ala_carte.py
index 264b719..115966a 100644 (file)
--- a/src/onaptests/steps/instantiate/vf_module_ala_carte.py
+++ b/src/onaptests/steps/instantiate/vf_module_ala_carte.py
@@ -34,7 +34,15 @@ class YamlTemplateVfModuleAlaCarteInstantiateStep(YamlTemplateBaseStep):
         else:
             self.add_step(YamlTemplateVnfAlaCarteInstantiateStep(cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Instantiate VF module described in YAML using SO a'la carte method."
 
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SO"
 
     @property
     def yaml_template(self) -> dict:

diff --git a/src/onaptests/steps/instantiate/vl_ala_carte.py b/src/onaptests/steps/instantiate/vl_ala_carte.py
index 3a77c73..f2815a4 100644 (file)
--- a/src/onaptests/steps/instantiate/vl_ala_carte.py
+++ b/src/onaptests/steps/instantiate/vl_ala_carte.py
@@ -30,6 +30,16 @@ class YamlTemplateVlAlaCarteInstantiateStep(YamlTemplateBaseStep):
         self._service_instance: ServiceInstance = None
         self.add_step(YamlTemplateServiceAlaCarteInstantiateStep(cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Instantiate network link described in YAML using SO a'la carte method."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SO"
+
     @property
     def yaml_template(self) -> dict:
         """Step YAML template.

diff --git a/src/onaptests/steps/instantiate/vnf_ala_carte.py b/src/onaptests/steps/instantiate/vnf_ala_carte.py
index e9ea1ba..6b9c4a4 100644 (file)
--- a/src/onaptests/steps/instantiate/vnf_ala_carte.py
+++ b/src/onaptests/steps/instantiate/vnf_ala_carte.py
@@ -28,6 +28,16 @@ class YamlTemplateVnfAlaCarteInstantiateStep(YamlTemplateBaseStep):
         self._service_instance: ServiceInstance = None
         self.add_step(YamlTemplateServiceAlaCarteInstantiateStep(cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Instantiate vnf described in YAML using SO a'la carte method."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SO"
+
     @property
     def yaml_template(self) -> dict:
         """Step YAML template.

diff --git a/src/onaptests/steps/onboard/service.py b/src/onaptests/steps/onboard/service.py
index 86a2e6a..3c8b183 100644 (file)
--- a/src/onaptests/steps/onboard/service.py
+++ b/src/onaptests/steps/onboard/service.py
@@ -21,6 +21,16 @@ class ServiceOnboardStep(BaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(VfOnboardStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard service in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @BaseStep.store_state
     def execute(self):
         """Onboard service.
@@ -57,6 +67,16 @@ class YamlTemplateServiceOnboardStep(YamlTemplateBaseStep):
         self._yaml_template: dict = None
         self.add_step(YamlTemplateVfOnboardStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard service described in YAML file in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @property
     def yaml_template(self) -> dict:
         """Step YAML template.

diff --git a/src/onaptests/steps/onboard/vendor.py b/src/onaptests/steps/onboard/vendor.py
index 49d721b..71b28b9 100644 (file)
--- a/src/onaptests/steps/onboard/vendor.py
+++ b/src/onaptests/steps/onboard/vendor.py
@@ -7,6 +7,16 @@ from ..base import BaseStep
 class VendorOnboardStep(BaseStep):
     """Vendor onboard step."""
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard vendor in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @BaseStep.store_state
     def execute(self):
         """Onboard vendor.

diff --git a/src/onaptests/steps/onboard/vf.py b/src/onaptests/steps/onboard/vf.py
index 18b56f3..817b412 100644 (file)
--- a/src/onaptests/steps/onboard/vf.py
+++ b/src/onaptests/steps/onboard/vf.py
@@ -18,6 +18,16 @@ class VfOnboardStep(BaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(VspOnboardStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard vf in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @BaseStep.store_state
     def execute(self):
         """Onboard Vf.
@@ -45,6 +55,16 @@ class YamlTemplateVfOnboardStep(YamlTemplateBaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(YamlTemplateVspOnboardStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard vf described in YAML file in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @property
     def yaml_template(self) -> dict:
         """YAML template.

diff --git a/src/onaptests/steps/onboard/vsp.py b/src/onaptests/steps/onboard/vsp.py
index e0761ab..18f73d8 100644 (file)
--- a/src/onaptests/steps/onboard/vsp.py
+++ b/src/onaptests/steps/onboard/vsp.py
@@ -19,6 +19,16 @@ class VspOnboardStep(BaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(VendorOnboardStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard vsp in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @BaseStep.store_state
     def execute(self):
         """Onboard Vsp.
@@ -47,6 +57,16 @@ class YamlTemplateVspOnboardStep(YamlTemplateBaseStep):
         super().__init__(cleanup=cleanup)
         self.add_step(VendorOnboardStep(cleanup=cleanup))
 
+    @property
+    def description(self) -> str:
+        """Step description."""
+        return "Onboard vsp described in YAML file in SDC."
+
+    @property
+    def component(self) -> str:
+        """Component name."""
+        return "SDC"
+
     @property
     def yaml_template(self) -> dict:
         """YAML template.

diff --git a/src/onaptests/steps/reports_collection.py b/src/onaptests/steps/reports_collection.py
index 6879c2b..e1eae51 100644 (file)
--- a/src/onaptests/steps/reports_collection.py
+++ b/src/onaptests/steps/reports_collection.py
@@ -1,8 +1,25 @@
 import sys
-from typing import Dict
+from dataclasses import dataclass
+from enum import Enum
+from typing import List
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 from onapsdk.configuration import settings
 
+
+class ReportStepStatus(Enum):
+    """Enum which stores steps execution statuses."""
+    PASS = "PASS"
+    FAIL = "FAIL"
+
+
+@dataclass
+class Report:
+    """Step execution report."""
+    step_description: str
+    step_execution_status: ReportStepStatus
+    step_execution_duration: float
+
+
 class ReportsCollection:
     """Collection to store steps execution statuses."""
 
@@ -10,36 +27,39 @@ class ReportsCollection:
         """Initialize collection."""
         self._collection: list = []
 
-    def put(self, item: Dict[str, str]) -> None:
+    def put(self, item: Report) -> None:
         """Put execution status dictionary.
 
         Args:
-            item (Dict[str, str]): Step name with status dictionary
+            item (Report): Step report
 
         """
-        self._collection.append(item)
+        self._collection.insert(0, item)
 
     @property
-    def report(self) -> Dict[str, str]:
+    def report(self) -> List[Report]:
         """Get report.
 
         Build a dictionary with execution statuses.
 
         Returns:
-            Dict[str, str]: Steps name with status dictionary
+            List[str, str]: Steps name with status dictionary
+
+        """
+        return self._collection
+
+    @property
+    def failed_steps_num(self) -> int:
+        """Number of failed steps in report.
+
+        Returns:
+            int: How many steps failed
 
         """
-        report: Dict[str, str] = {}
-        for element in self._collection[::-1]:
-            report.update(element)
-        return report
+        return sum((1 for step_report in self.report if \
+            step_report.step_execution_status == ReportStepStatus.FAIL))
 
     def generate_report(self) -> None:
-        step_list = self.report
-        failing_steps = {}
-        for step,status in step_list.items():
-            if 'FAIL' in status:
-                failing_steps[step] = status
         usecase = settings.SERVICE_NAME
         try:
             details = settings.SERVICE_DETAILS
@@ -56,8 +76,7 @@ class ReportsCollection:
             loader=FileSystemLoader(sys.path[-1] + '/onaptests/templates/reporting'))
 
         jinja_env.get_template('reporting.html.j2').stream(
-            failing_steps=failing_steps,
-            steps=step_list,
+            report=self,
             usecase=usecase,
             details=details,
             components=components,

diff --git a/src/onaptests/templates/reporting/reporting.html.j2 b/src/onaptests/templates/reporting/reporting.html.j2
index 5dafe3f..05c00b7 100644 (file)
--- a/src/onaptests/templates/reporting/reporting.html.j2
+++ b/src/onaptests/templates/reporting/reporting.html.j2
@@ -2,7 +2,7 @@
 {% block title %}Summary{% endblock %}
 {% block content %}
     {{ summary('Results', "", [
-      { 'title': 'Pythonsdk Tests', 'failing': (failing_steps | length), 'total': (steps | length)},
+      { 'title': 'Pythonsdk Tests', 'failing': report.failed_steps_num, 'total': (report.report | length)},
       ])
     }}
 
@@ -23,16 +23,20 @@
             <tr>
               <th>Name</th>
               <th>Status</th>
+              <th>Duration (seconds)</th>
             </tr>
           </thead>
           <tbody>
-          {% for step,value in steps.items() %}
-            <tr {% if  value == 'FAIL' %} class="has-background-danger"  {% else %} class="has-background-success-light" {% endif %}>
+          {% for step_report in report.report %}
+            <tr {% if  step_report.step_execution_status.value == 'FAIL' %} class="has-background-danger"  {% else %} class="has-background-success-light" {% endif %}>
             <td>
-            {{ step }}
+            {{ step_report.step_description }}
             </td>
             <td>
-            {{ value }}
+            {{ step_report.step_execution_status.value }}
+            </td>
+            <td>
+            {{ step_report.step_execution_duration | round(2) }}
             </td>
             </tr>
           {% endfor %}

diff --git a/tests/test_reports_collection.py b/tests/test_reports_collection.py
index 264b6b4..8491b91 100644 (file)
--- a/tests/test_reports_collection.py
+++ b/tests/test_reports_collection.py
@@ -1,10 +1,34 @@
 
-from onaptests.steps.reports_collection import ReportsCollection
+from onaptests.steps.reports_collection import Report, ReportsCollection, ReportStepStatus
 
 
 def test_reports_collection():
     rc = ReportsCollection()
-    assert rc.report == {}
+    assert rc.report == []
 
-    rc.put({"a": "b"})
-    assert rc.report == {"a": "b"}
+    rc.put(Report(
+        "test",
+        ReportStepStatus.PASS,
+        0.0
+    ))
+    assert len(rc.report) == 1
+
+
+def test_reports_collection_failed_steps_num():
+
+    rc = ReportsCollection()
+    assert rc.failed_steps_num == 0
+
+    rc.put(Report(
+        "test",
+        ReportStepStatus.PASS,
+        0.0
+    ))
+    assert rc.failed_steps_num == 0
+
+    rc.put(Report(
+        "test",
+        ReportStepStatus.FAIL,
+        0.0
+    ))
+    assert rc.failed_steps_num == 1

diff --git a/tests/test_store_state.py b/tests/test_store_state.py
index e0c8b6b..f5fcc62 100644 (file)
--- a/tests/test_store_state.py
+++ b/tests/test_store_state.py
@@ -1,13 +1,23 @@
 import pytest
+
 from onaptests.steps.base import BaseStep
 
 
+
 class TestStep(BaseStep):
 
     @BaseStep.store_state
     def execute(self):
         return super().execute()
 
+    @property
+    def description(self):
+        return "Test pass step"
+
+    @property
+    def component(self) -> str:
+        return "Test"
+
 
 class TestFailStep(BaseStep):
 
@@ -16,14 +26,33 @@ class TestFailStep(BaseStep):
         super().execute()
         raise Exception
 
+    @property
+    def description(self):
+        return "Test fail step"
+
+    @property
+    def component(self) -> str:
+        return "Test"
+
 
 def test_store_state():
     ts = TestStep()
     ts.execute()
-    assert ts.reports_collection.report == {"TestStep": "PASS"}
+    assert len(ts.reports_collection.report) == 1
+    rep = ts.reports_collection.report[0]
+    assert rep.step_description == "[Test] TestStep: Test pass step"
+    assert rep.step_execution_status.value == "PASS"
+    assert rep.step_execution_duration != 0
 
     fs = TestFailStep()
     fs.add_step(TestStep())
     with pytest.raises(Exception):
         fs.execute()
-    fs.reports_collection.report == {"TestFailStep": "FAIL", "TestStep": "PASS"}
+    rep_f, rep_s = fs.reports_collection.report
+    assert rep_f.step_description == "[Test] TestFailStep: Test fail step"
+    assert rep_f.step_execution_status.value == "FAIL"
+    assert rep_f.step_execution_duration != 0
+
+    assert rep_s.step_description == "[Test] TestStep: Test pass step"
+    assert rep_s.step_execution_status.value == "PASS"
+    assert rep_s.step_execution_duration != 0