Fix to not remove whitespaces in the payload to ES

Change-Id: I3547e79ff27220a49539d9cfdf48ac534f155f10
Issue-ID: AAI-1596
Signed-off-by: Edwin Lawrance <Edwin.Lawrance@amdocs.com>

diff --git a/pom.xml b/pom.xml
index 3a3a67e..e6768cd 100644 (file)
--- a/pom.xml
+++ b/pom.xml
@@ -69,7 +69,6 @@ limitations under the License.
             <groupId>com.jayway.jsonpath</groupId>
             <artifactId>json-path</artifactId>
             <version>2.2.0</version>
-            <scope>test</scope>
         </dependency>
 
         <dependency>

diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
index 90adbd7..98a254c 100644 (file)
--- a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
@@ -370,7 +370,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
     try {
        attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(sb.toString()));
     } catch(IOException e) {
-       throw new DocumentStoreOperationException("Error in translating Index payload to make it ES compliant.", e);
+       logger.error(SearchDbMsgs.INDEX_CREATE_FAILURE, e);
+       throw new DocumentStoreOperationException(e.getMessage(), e);
     }
 
     logger.debug("\ncreateTable(), Sending 'PUT' request to URL : " + conn.getURL());
@@ -422,7 +423,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
     try {
        attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(settingsAndMappings));
     } catch(IOException e) {
-       throw new DocumentStoreOperationException("Error in translating DynamicIndex payload to make it ES compliant.", e);
+       logger.error(SearchDbMsgs.INDEX_CREATE_FAILURE, e);
+       throw new DocumentStoreOperationException(e.getMessage());
     }
     handleResponse(conn, result);
 
@@ -571,12 +573,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 //    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
     conn.setRequestProperty("Content-Type", "application/json");
     conn.setRequestProperty("Connection", "Close");
-    
-    try {
-       attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(doc.getContentInJson()));
-    } catch(IOException e) {
-       throw new DocumentStoreOperationException("Error in translating Document payload to make it ES compliant.", e);
-    }
+    attachContent(conn, doc.getContentInJson());
   }
 
   private DocumentOperationResult checkDocumentExistence(String indexName,

diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
index 2e97103..97f0ca1 100644 (file)
--- a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
@@ -23,14 +23,20 @@ package org.onap.aai.sa.searchdbabstraction.util;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.onap.aai.cl.api.Logger;
 import org.onap.aai.cl.eelf.LoggerFactory;
 import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs;
 
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+
 
 /**
  * This class as the name suggests is to translate the payload of PUT & POST requests
@@ -52,8 +58,9 @@ public class ElasticSearchPayloadTranslator {
 
 
        /**
-        *  Using String replacement to translate the payload to ES compatible version
-        * 
+        *  Using JSON Path query to filter objects to translate the payload to ES compatible version
+        *  The filter queries and the replacement attributes are configured in the es-payload-translation.json file.
+        *  
         * @param source
         * @return translated payload in String
         * @throws IOException
@@ -61,23 +68,33 @@ public class ElasticSearchPayloadTranslator {
        public static String translateESPayload(String source) throws IOException {
                logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "translateESPayload, method-params[ source=" + source + "]");
                String pathToTranslationFile = CONFIG_DIRECTORY + File.separator + ES_PAYLOAD_TRANSLATION_FILE;
-               
-               String translatedPayload = source.replaceAll("\\s+", ""); // removing whitespaces
-               JSONObject translationConfigPayload = new JSONObject(IOUtils.toString(
-                               new FileInputStream(new File(pathToTranslationFile)), "UTF-8"));
 
-               JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations");
+               try {
+                       
+                       JSONObject translationConfigPayload = new JSONObject(IOUtils.toString(
+                                       new FileInputStream(new File(pathToTranslationFile)), "UTF-8"));
+                       JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations");
+                       DocumentContext payloadToTranslate = JsonPath.parse(source);
 
-               for(Object obj : attrTranslations) {
-                       JSONObject jsonObj = ((JSONObject)obj);
-                       String from = jsonObj.get("from").toString();
-                       String to = jsonObj.get("to").toString();
-                       if(translatedPayload.indexOf(from) > 0) {
-                               translatedPayload = translatedPayload.replaceAll(from, to);
+                       for(Object obj : attrTranslations) {
+                               JSONObject jsonObj = ((JSONObject) obj);
+                               String query = jsonObj.get("query").toString();
+                               JSONObject attrToUpdate = (JSONObject) jsonObj.get("update");
+                               List<Map<String, Object>> filteredObjects = payloadToTranslate.read(query);
+                               for(Map<String, Object> objMap : filteredObjects) {
+                                       objMap.putAll(attrToUpdate.toMap());
+                               }
                        }
-               }
-
-               logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+translatedPayload);
-               return translatedPayload;
+                       
+                       logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+payloadToTranslate.jsonString());
+                       return payloadToTranslate.jsonString();
+                       
+               } catch (JSONException | IOException e) {
+                       logger.error(SearchDbMsgs.FILTERS_CONFIG_FAILURE, e, ES_PAYLOAD_TRANSLATION_FILE, e.getMessage());
+                       if(e instanceof JSONException) {
+                               throw new IOException("Payload translation configuration looks corrupted. Please correct!", e);
+                       }
+                       throw new IOException("Error in configuring payload translation file. Please check if it exists.", e);
+               }       
        }
 }

diff --git a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java
index 11b2acb..ba5cb48 100644 (file)
--- a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java
+++ b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java
@@ -21,16 +21,19 @@
 package org.onap.aai.sa.searchdbabstraction.util;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.io.File;
+import java.io.FileInputStream;
 
+import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
 import org.onap.aai.sa.rest.TestUtils;
 
 public class ElasticSearchPayloadTranslatorTest {
        
-       private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/simpleDocument.json";
+       private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/index-mapping.json";
 
        @Before
        public void setup() throws Exception {
@@ -38,13 +41,17 @@ public class ElasticSearchPayloadTranslatorTest {
        }
        
        @Test
-       public void testPayloadTranslation_FromStringToText() throws Exception {
+       public void testPayloadTranslation() throws Exception {
+               String expectedErrMsg = "Sample error message for whitespace check";
                File schemaFile = new File(SIMPLE_DOC_SCHEMA_JSON);
-           String documentJson = TestUtils.readFileToString(schemaFile);
-           assertTrue(documentJson.contains("\"data-type\":\"string\""));
-               assertTrue(documentJson.contains("\"searchable\":true"));
+           String documentJson = IOUtils.toString(new FileInputStream(schemaFile), "UTF-8");
+           assertTrue(documentJson.contains("\"type\": \"string\""));
+               assertTrue(documentJson.contains("\"index\": \"analyzed\""));
                String translatedPayload = ElasticSearchPayloadTranslator.translateESPayload(documentJson);
-               assertTrue(translatedPayload.contains("\"data-type\":\"text\""));
+               assertTrue(translatedPayload.contains("\"type\":\"text\""));
                assertTrue(translatedPayload.contains("\"index\":true"));
+               assertTrue(translatedPayload.contains("\"fielddata\":true"));
+               assertFalse(documentJson.contains("\"index\":\"analyzed\""));
+               assertTrue(translatedPayload.contains("\"errMsg\":\""+expectedErrMsg+"\""));
        }
 }

diff --git a/src/test/resources/json/es-payload-translation.json b/src/test/resources/json/es-payload-translation.json
index e5290b0..58ed8f6 100644 (file)
--- a/src/test/resources/json/es-payload-translation.json
+++ b/src/test/resources/json/es-payload-translation.json
@@ -1,24 +1,16 @@
 {
-  "attr-translations": [
-    { 
-      "from": "\"data-type\":\"string\"",
-      "to": "\"data-type\":\"text\",\"fielddata\":true"
-    },    
+  "attr-translations": [       
     {
-      "from": "\"type\":\"string\",\"index\":\"analyzed\"",
-      "to": "\"type\":\"text\",\"index\":\"true\",\"fielddata\":true"
+      "query": "$..[?(@.type=='string' && @.index=='analyzed')]",
+      "update": {"type": "text", "index": true, "fielddata": true}
     },
     {
-      "from": "\"type\":\"string\",\"index\":\"not_analyzed\"",
-      "to": "\"type\":\"keyword\",\"index\":\"true\""
+      "query": "$..[?(@.type=='string' && @.index=='not_analyzed')]",
+      "update": {"type": "keyword", "index": true}
     },
     {
-      "from": "\"type\":\"string\"",
-      "to": "\"type\":\"text\",\"fielddata\":true"
-    },
-    {
-      "from": "searchable",
-      "to": "index"
+      "query": "$..[?(@.type=='string' && !@.index)]",
+      "update": {"type": "text", "fielddata": true}
     }
   ]
 }
\ No newline at end of file

diff --git a/src/test/resources/json/index-mapping.json b/src/test/resources/json/index-mapping.json
new file mode 100644 (file)
index 0000000..e447092
--- /dev/null
+++ b/src/test/resources/json/index-mapping.json
@@ -0,0 +1,28 @@
+{
+   "fields": [
+              {"name": "validationId", "type": "string", "searchable": false},
+              {"name": "validationTimestamp1", "type": "date", "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM\/dd\/yyyy||yyyyMMdd'T'HHmmssZ"},
+              {"name": "entityId", "type": "nested"},
+              {"name": "entityType", "type": "string", "index": "analyzed"},
+              {"name": "entityLink", "type": "string"},
+              
+              {"name": "resourceVersion", "type": "string", "index": "not_analyzed"},
+              {"name": "violations", "type": "nested", "sub-fields": [
+                     {"name": "violationId", "type": "string"},
+                     {"name": "violationTimestamp", "type": "date", "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM\/dd\/yyyy||yyyyMMdd'T'HHmmssZ"},            
+                     {"name": "modelName", "type": "string"},
+                     {"name": "category", "type": "string"},
+                     {"name": "severity", "type": "string"},
+                     {"name": "violationType", "type": "string"},
+                     {"name": "validationRule", "type": "string"},
+                     {"name": "violationDetails", "type": "nested", "sub-fields": [
+                           {"name": "entityId", "type": "nested"},
+                           {"name": "entityType", "type": "string"},
+                           {"name": "modelName", "type": "string"},
+                           {"name": "MISSING_REL", "type": "string"}
+                     ]},
+                     {"name": "errorMessage", "type": "string", "errMsg": "Sample error message for whitespace check"}
+              ]}
+       ]
+   
+}