implement regex support in PM Mapper filtering mechanism 37/110237/3
authormarcinrzepeckiwroc <marcin.rzepecki@nokia.com>
Wed, 15 Jul 2020 11:26:51 +0000 (13:26 +0200)
committermarcinrzepeckiwroc <marcin.rzepecki@nokia.com>
Fri, 17 Jul 2020 11:43:43 +0000 (13:43 +0200)
Issue-ID: DCAEGEN2-2347
Signed-off-by: marcinrzepeckiwroc <marcin.rzepecki@nokia.com>
Change-Id: I3fdbae33ae318800ccc2758a191604fe79c2aaf4

src/main/java/org/onap/dcaegen2/services/pmmapper/App.java
src/main/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandler.java
src/test/java/org/onap/dcaegen2/services/pmmapper/AppTest.java
src/test/java/org/onap/dcaegen2/services/pmmapper/filtering/MeasFilterHandlerTest.java
src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml [new file with mode: 0644]
src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml [new file with mode: 0644]
src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml [new file with mode: 0644]
src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml [new file with mode: 0644]

index 21e4441..6aeeaba 100644 (file)
@@ -59,7 +59,6 @@ import javax.net.ssl.SSLContext;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
 import java.util.List;
 
 @Data
@@ -213,6 +212,7 @@ public class App {
         try {
             hasMatchingFilter = filterHandler.filterByMeasType(events);
             if(!hasMatchingFilter) {
+                logger.unwrap().info("No filter match from all measurement files.");
                 sendEventProcessed(config,event);
             }
         } catch (Exception exception) {
@@ -223,25 +223,25 @@ public class App {
     }
 
     public static Flux<List<Event>> map(Mapper mapper, List<Event> events, MapperConfig config) {
-        List<Event> mappedEvents  = new ArrayList<>();
+        List<Event> mappedEvents;
         try {
             mappedEvents = mapper.mapEvents(events);
         } catch (Exception exception) {
             logger.unwrap().error("Unable to map XML to VES",exception);
             sendEventProcessed(config,events.get(0));
-            return Flux.<List<Event>>empty();
+            return Flux.empty();
         }
         return Flux.just(mappedEvents);
     }
 
     public static Flux<List<Event>> split(MeasSplitter splitter, Event event, MapperConfig config) {
-        List<Event> splitEvents  = new ArrayList<>();
+        List<Event> splitEvents;
         try {
             splitEvents = splitter.split(event);
         } catch (Exception exception) {
             logger.unwrap().error("Unable to split MeasCollecFile",exception);
             sendEventProcessed(config,event);
-            return Flux.<List<Event>>empty();
+            return Flux.empty();
         }
         return Flux.just(splitEvents);
     }
index fb5772e..7f2a5d6 100644 (file)
 package org.onap.dcaegen2.services.pmmapper.filtering;
 
 import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.FilenameUtils;
@@ -68,16 +71,9 @@ public class MeasFilterHandler {
         logger.unwrap().info("Filtering the measurement file by measTypes.");
         MeasurementData measData = measurementFile.getMeasurementData().get().get(0);
         List<MeasurementInfo> measInfos = measData.getMeasurementInfo();
-        List<MeasurementInfo> filteredMeasInfos = new ArrayList<>();
+        List<Pattern> filterPatternList = getPatternsFromFilters(filter);
 
-        for (MeasurementInfo currentMeasInfo : measInfos) {
-            List<String> measTypesNode = currentMeasInfo.getMeasTypes();
-            if (measTypesNode != null && !measTypesNode.isEmpty()) {
-                setMeasInfosFromMeasTypes(currentMeasInfo, filteredMeasInfos, filter.get());
-            } else {
-                setMeasInfoFromMeasType(currentMeasInfo, filteredMeasInfos, filter.get());
-            }
-        }
+        List<MeasurementInfo> filteredMeasInfos = filterMeasInfos(measInfos, filterPatternList);
 
         if (filteredMeasInfos.isEmpty()) {
             logger.unwrap().info("No filter match from the current measurement file.");
@@ -90,27 +86,43 @@ public class MeasFilterHandler {
         return true;
     }
 
-    /**
-     * Filters each measInfo node in the list for measTypes that match the given measTypes from filters.
-     **/
-    public boolean filterByMeasType(List<Event> events) {
-        boolean hasMatchAnyFilter = false;
-        for (int i = 0; i < events.size(); i++) {
-            Event currentEvent = events.get(i);
-            boolean hasMatchingFilter = filterByMeasType(currentEvent);
-            if (hasMatchingFilter) {
-                hasMatchAnyFilter = true;
+    private List<MeasurementInfo> filterMeasInfos(List<MeasurementInfo> measInfos, List<Pattern> filterPatternList) {
+        List<MeasurementInfo> filteredMeasInfos = new LinkedList<>();
+        for (MeasurementInfo currentMeasInfo : measInfos) {
+            List<String> measTypesNode = currentMeasInfo.getMeasTypes();
+            if (measTypesNode != null && !measTypesNode.isEmpty()) {
+                getFilteredMeasInfosFromMeasTypes(currentMeasInfo, filterPatternList).ifPresent(filteredMeasInfos::add);
             } else {
-                events.remove(events.get(i));
+                getFilteredMeasInfoFromMeasType(currentMeasInfo, filterPatternList).ifPresent(filteredMeasInfos::add);
             }
         }
+        return  filteredMeasInfos;
+    }
 
-        if (!hasMatchAnyFilter) {
-            logger.unwrap().info("No filter match from all measurement files.");
-            return false;
+    private List<Pattern> getPatternsFromFilters(Optional<Filter> filters) {
+        List<Pattern> patternList = new LinkedList<>();
+        for (String filter : filters.get().getMeasTypes()) {
+            tryToCompileFilter(filter).ifPresent(patternList::add);
+        }
+        return patternList;
+    }
+    private Optional<Pattern> tryToCompileFilter(String measType) {
+        try {
+            return Optional.of(Pattern.compile("^" + measType + "$"));
+        } catch (PatternSyntaxException exception) {
+            logger.unwrap().warn("Can not parse measType filter: ", exception);
         }
+        return Optional.empty();
+    }
 
-        return true;
+    /**
+     * Filters each measInfo node in the list for measTypes that match the given measTypes from filters.
+     **/
+    public boolean filterByMeasType(List<Event> events) {
+        List<Event> filteredList = events.stream().filter(this::filterByMeasType).collect(Collectors.toList());
+        events.clear();
+        events.addAll(filteredList);
+        return !events.isEmpty();
     }
 
     private boolean hasNoFilters(Optional<Filter> filter) {
@@ -162,9 +174,9 @@ public class MeasFilterHandler {
        return hasResults;
     }
 
-    private void setMeasInfoFromMeasType(MeasurementInfo currentMeasInfo, List<MeasurementInfo> filteredMeasInfos, Filter filter) {
+    private Optional<MeasurementInfo> getFilteredMeasInfoFromMeasType(MeasurementInfo currentMeasInfo, List<Pattern> filters) {
         List<MeasurementInfo.MeasType> filteredMeasTypes = currentMeasInfo.getMeasType().stream()
-                .filter(mt -> filter.getMeasTypes().contains(mt.getValue()))
+                .filter(mt -> matchFilters(filters, mt.getValue()))
                 .collect(Collectors.toList());
 
         if(!filteredMeasTypes.isEmpty()) {
@@ -173,11 +185,16 @@ public class MeasFilterHandler {
                     .collect(Collectors.toList());
             currentMeasInfo.replaceMeasType(filteredMeasTypes);
             currentMeasInfo.replaceMeasValue(filteredMeasValues);
-            filteredMeasInfos.add(currentMeasInfo);
+            return Optional.of(currentMeasInfo);
         }
+        return Optional.empty();
+    }
+
+    private boolean matchFilters(List<Pattern> filters, String measType) {
+        return filters.stream().anyMatch(filter -> filter.matcher(measType).matches());
     }
 
-    private void setMeasInfosFromMeasTypes(MeasurementInfo currentMeasInfo, List<MeasurementInfo> filteredMeasInfos, Filter filter) {
+    private Optional<MeasurementInfo> getFilteredMeasInfosFromMeasTypes(MeasurementInfo currentMeasInfo, List<Pattern> filters) {
         MeasValue currentMeasValue = currentMeasInfo.getMeasValue()
                 .get(0);
         List<String> measTypesNode = currentMeasInfo.getMeasTypes();
@@ -187,8 +204,7 @@ public class MeasFilterHandler {
         List<String> filteredMeasTypes = new ArrayList<>();
         for (int j = 0; j < measTypesNode.size(); j++) {
             String currentMeasType = measTypesNode.get(j);
-            List<String> measTypeFilters = filter.getMeasTypes();
-            if (measTypeFilters.contains(currentMeasType)) {
+            if (matchFilters(filters, currentMeasType)) {
                 filteredMeasTypes.add(currentMeasType);
                 filteredMeasResults.add(measResultsNode.get(j));
             }
@@ -197,8 +213,9 @@ public class MeasFilterHandler {
         if (!filteredMeasTypes.isEmpty()) {
             currentMeasInfo.replaceMeasTypes(filteredMeasTypes);
             currentMeasValue.replaceMeasResults(filteredMeasResults);
-            filteredMeasInfos.add(currentMeasInfo);
+            return Optional.of(currentMeasInfo);
         }
+        return Optional.empty();
     }
 
 }
index ce051e4..0b8cdfc 100644 (file)
@@ -33,13 +33,14 @@ import static org.mockserver.integration.ClientAndServer.startClientAndServer;
 import static org.mockserver.model.HttpResponse.response;
 import static utils.ConfigUtils.getMapperConfigFromFile;
 
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Collections;
 
 import com.google.gson.Gson;
 import io.undertow.server.HttpServerExchange;
@@ -261,7 +262,7 @@ class AppTest {
     @Test
     void testFilter_success() {
         Event mockEvent = Mockito.mock(Event.class);
-        List<Event> mockEvents = Arrays.asList(mockEvent);
+        List<Event> mockEvents = new LinkedList<>(Collections.singletonList(mockEvent));
         MapperConfig mockConfig = Mockito.mock(MapperConfig.class);
         boolean result = App.filter(new MeasFilterHandler(new MeasConverter()), mockEvents, mockConfig);
         assertTrue(result);
index 0462616..3eea765 100644 (file)
@@ -30,6 +30,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -58,7 +59,7 @@ class MeasFilterHandlerTest {
 
     private static final Path FILTER_DIRECTORY = Paths.get("src/test/resources/filter_test/");
     private static final String baseDir = "src/test/resources/filter_test/";
-    private static MeasConverter converter = new MeasConverter();
+    private static final MeasConverter converter = new MeasConverter();
     private MeasFilterHandler objUnderTest;
     @Mock
     private HttpServerExchange exchange;
@@ -100,11 +101,26 @@ class MeasFilterHandlerTest {
         assertEquals(1, events.size());
     }
 
+    @Test
+    void remove_all_events_that_does_not_match_filter() {
+        String inputPath = baseDir + "lte/meas_type_and_r_manyinfo/test.xml";
+
+        Filter noMatchFilter = new MeasFilterConfig().new Filter();
+        noMatchFilter.setMeasTypes(Arrays.asList("ad", "bs"));
+        Event eventNoMatch = generateEvent(inputPath, noMatchFilter);
+
+        List<Event> events = new ArrayList<>();
+        events.add(eventNoMatch);
+        events.add(eventNoMatch);
+        assertFalse(objUnderTest.filterByMeasType(events));
+        assertEquals(0, events.size());
+    }
+
     @Test
     void skip_mapping_when_MeasData_isEmpty() {
         String inputPath = baseDir + "lte/meas_results/test.xml";
         Event event = generateEvent(inputPath, generateValidFilter());
-        event.getMeasurement().replacementMeasurementData(Arrays.asList());
+        event.getMeasurement().replacementMeasurementData(Collections.emptyList());
 
         assertFalse(objUnderTest.filterByMeasType(event));
     }
@@ -114,7 +130,7 @@ class MeasFilterHandlerTest {
         String inputPath = baseDir + "lte/meas_results/test.xml";
 
         Filter emptyMeastypesFilter = new MeasFilterConfig().new Filter();
-        emptyMeastypesFilter.setMeasTypes(Arrays.asList());
+        emptyMeastypesFilter.setMeasTypes(Collections.emptyList());
 
         Event event = generateEvent(inputPath, emptyMeastypesFilter);
         MeasurementFile originalMeasCollec = event.getMeasurement();
@@ -148,6 +164,18 @@ class MeasFilterHandlerTest {
         });
     }
 
+    @Test
+    void invalid_pattern_test() {
+        String inputPath = baseDir + "lte/meas_type_and_r_manyinfo/test.xml";
+        Filter matchFilter = new MeasFilterConfig().new Filter();
+        matchFilter.setMeasTypes(Arrays.asList("a", "b", "......****"));
+        Event eventMatch = generateEvent(inputPath, matchFilter);
+        List<Event> events = new ArrayList<>();
+        events.add(eventMatch);
+        assertTrue(objUnderTest.filterByMeasType(events));
+        assertEquals(1, events.size());
+    }
+
     @ParameterizedTest
     @MethodSource("getEvents")
     void filter_valid_measurements(Event expectedEvent, Event testEvent) {
@@ -170,7 +198,7 @@ class MeasFilterHandlerTest {
         Filter filter;
         filter = new MeasFilterConfig().new Filter();
         filter.setDictionaryVersion("1.0");
-        filter.setMeasTypes(Arrays.asList("a", "b"));
+        filter.setMeasTypes(Arrays.asList("a", "b", "aab.*", ".*3", ".******test"));
         return filter;
     }
 
diff --git a/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/expected.xml
new file mode 100644 (file)
index 0000000..d1044ad
--- /dev/null
@@ -0,0 +1,42 @@
+<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec">
+    <fileHeader fileFormatVersion="32.435 V10.0" vendorName="FooBar Ltd" dnPrefix="some dnPrefix">
+        <fileSender localDn="Dublin"/>
+        <measCollec beginTime="2018-10-02T12:00:00+01:00"/>
+    </fileHeader>
+    <measData>
+        <managedElement localDn="Dublin" swVersion="r0.1"/>
+        <measInfo measInfoId="this will be filtered out">
+            <job jobId="jobId"/>
+            <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/>
+            <repPeriod duration="PT900S"/>
+            <measTypes>zz3</measTypes>
+            <measValue measObjLdn="objLdn">
+                <measResults>27</measResults>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="some measInfoId">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measTypes>a b</measTypes>
+            <measValue measObjLdn="objLdn">
+                <measResults>1 2</measResults>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="some measInfoId2">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measTypes>aab1 b</measTypes>
+            <measValue measObjLdn="objLdn">
+                <measResults>1 2</measResults>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+    </measData>
+    <fileFooter>
+        <measCollec endTime="2018-10-02T12:15:00+01:00"/>
+    </fileFooter>
+</measCollecFile>
diff --git a/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml b/src/test/resources/filter_test/lte/meas_results_manyinfo_regex/test.xml
new file mode 100644 (file)
index 0000000..890fee4
--- /dev/null
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec">
+    <fileHeader dnPrefix="some dnPrefix" vendorName="FooBar Ltd"
+                fileFormatVersion="32.435 V10.0">
+        <fileSender localDn="Dublin"/>
+        <measCollec beginTime="2018-10-02T12:00:00+01:00"/>
+    </fileHeader>
+    <measData>
+        <managedElement swVersion="r0.1" localDn="Dublin"/>
+        <measInfo measInfoId="this will be filtered out">
+            <job jobId="jobId"/>
+            <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/>
+            <repPeriod duration="PT900S"/>
+            <measTypes>z aa zz3 bb</measTypes>
+            <measValue measObjLdn="objLdn">
+                <measResults>99 1 27 2</measResults>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="some measInfoId">
+            <job jobId="jobId"/>
+            <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/>
+            <repPeriod duration="PT900S"/>
+            <measTypes>z a zz b</measTypes>
+            <measValue measObjLdn="objLdn">
+                <measResults>99 1 27 2</measResults>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="some measInfoId2">
+            <job jobId="jobId"/>
+            <granPeriod endTime="2018-10-02T12:15:00Z" duration="PT900S"/>
+            <repPeriod duration="PT900S"/>
+            <measTypes>z aab1 zz b</measTypes>
+            <measValue measObjLdn="objLdn">
+                <measResults>99 1 27 2</measResults>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+    </measData>
+    <fileFooter>
+        <measCollec endTime="2018-10-02T12:15:00+01:00"/>
+    </fileFooter>
+</measCollecFile>
diff --git a/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/expected.xml
new file mode 100644 (file)
index 0000000..2a2a967
--- /dev/null
@@ -0,0 +1,48 @@
+<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec">
+    <fileHeader fileFormatVersion="32.435 V10.0" vendorName="FooBar Ltd" dnPrefix="some dnPrefix">
+        <fileSender localDn="Dublin"/>
+        <measCollec beginTime="2018-10-02T12:00:00+01:00"/>
+    </fileHeader>
+    <measData>
+        <managedElement localDn="Dublin" swVersion="r0.1"/>
+        <measInfo measInfoId="some measInfoId">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measType p="1">aab1</measType>
+            <measType p="2">aab2</measType>
+            <measType p="4">b</measType>
+            <measValue measObjLdn="some measObjLdn">
+                <r p="1">1</r>
+                <r p="2">99</r>
+                <r p="4">2</r>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="filter will disregard this measInfo">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measType p="1">aab1</measType>
+            <measValue measObjLdn="some measObjLdn">
+                <r p="1">1</r>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="some measInfoId2">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measType p="1">aab1</measType>
+            <measType p="3">bb3</measType>
+            <measValue measObjLdn="some measObjLdn">
+                <r p="1">1</r>
+                <r p="3">2</r>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+    </measData>
+    <fileFooter>
+        <measCollec endTime="2018-10-02T12:15:00+01:00"/>
+    </fileFooter>
+</measCollecFile>
diff --git a/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml b/src/test/resources/filter_test/lte/meas_type_and_r_manyinfo_regex/test.xml
new file mode 100644 (file)
index 0000000..9aa668e
--- /dev/null
@@ -0,0 +1,56 @@
+<measCollecFile xmlns="http://www.3gpp.org/ftp/specs/archive/32_series/32.435#measCollec">
+    <fileHeader fileFormatVersion="32.435 V10.0" vendorName="FooBar Ltd" dnPrefix="some dnPrefix">
+        <fileSender localDn="Dublin"/>
+        <measCollec beginTime="2018-10-02T12:00:00+01:00"/>
+    </fileHeader>
+    <measData>
+        <managedElement localDn="Dublin" swVersion="r0.1"/>
+        <measInfo measInfoId="some measInfoId">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measType p="1">aab1</measType>
+            <measType p="2">aab2</measType>
+            <measType p="3">z</measType>
+            <measType p="4">b</measType>
+            <measValue measObjLdn="some measObjLdn">
+                <r p="1">1</r>
+                <r p="2">99</r>
+                <r p="3">3</r>
+                <r p="4">2</r>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="filter will disregard this measInfo">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measType p="1">aab1</measType>
+            <measType p="2">z</measType>
+            <measType p="3">bb</measType>
+            <measValue measObjLdn="some measObjLdn">
+                <r p="1">1</r>
+                <r p="2">99</r>
+                <r p="3">2</r>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+        <measInfo measInfoId="some measInfoId2">
+            <job jobId="jobId"/>
+            <granPeriod duration="PT900S" endTime="2018-10-02T12:15:00Z"/>
+            <repPeriod duration="PT900S"/>
+            <measType p="1">aab1</measType>
+            <measType p="2">z</measType>
+            <measType p="3">bb3</measType>
+            <measValue measObjLdn="some measObjLdn">
+                <r p="1">1</r>
+                <r p="2">99</r>
+                <r p="3">2</r>
+                <suspect>false</suspect>
+            </measValue>
+        </measInfo>
+    </measData>
+    <fileFooter>
+        <measCollec endTime="2018-10-02T12:15:00+01:00"/>
+    </fileFooter>
+</measCollecFile>