From 82658daad05f982a8358581623f44a1554238a48 Mon Sep 17 00:00:00 2001 From: Edwin Lawrance Date: Thu, 13 Sep 2018 11:19:43 +0100 Subject: [PATCH] Fix to not remove whitespaces in the payload to ES Change-Id: I3547e79ff27220a49539d9cfdf48ac534f155f10 Issue-ID: AAI-1596 Signed-off-by: Edwin Lawrance --- pom.xml | 1 - .../dao/ElasticSearchHttpController.java | 13 +++--- .../util/ElasticSearchPayloadTranslator.java | 51 ++++++++++++++-------- .../util/ElasticSearchPayloadTranslatorTest.java | 19 +++++--- .../resources/json/es-payload-translation.json | 22 +++------- src/test/resources/json/index-mapping.json | 28 ++++++++++++ 6 files changed, 87 insertions(+), 47 deletions(-) create mode 100644 src/test/resources/json/index-mapping.json diff --git a/pom.xml b/pom.xml index 3a3a67e..e6768cd 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,6 @@ limitations under the License. com.jayway.jsonpath json-path 2.2.0 - test diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java index 90adbd7..98a254c 100644 --- a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java @@ -370,7 +370,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(sb.toString())); } catch(IOException e) { - throw new DocumentStoreOperationException("Error in translating Index payload to make it ES compliant.", e); + logger.error(SearchDbMsgs.INDEX_CREATE_FAILURE, e); + throw new DocumentStoreOperationException(e.getMessage(), e); } logger.debug("\ncreateTable(), Sending 'PUT' request to URL : " + conn.getURL()); @@ -422,7 +423,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(settingsAndMappings)); } catch(IOException e) { - throw new DocumentStoreOperationException("Error in translating DynamicIndex payload to make it ES compliant.", e); + logger.error(SearchDbMsgs.INDEX_CREATE_FAILURE, e); + throw new DocumentStoreOperationException(e.getMessage()); } handleResponse(conn, result); @@ -571,12 +573,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { // conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); conn.setRequestProperty("Content-Type", "application/json"); conn.setRequestProperty("Connection", "Close"); - - try { - attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(doc.getContentInJson())); - } catch(IOException e) { - throw new DocumentStoreOperationException("Error in translating Document payload to make it ES compliant.", e); - } + attachContent(conn, doc.getContentInJson()); } private DocumentOperationResult checkDocumentExistence(String indexName, diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java index 2e97103..97f0ca1 100644 --- a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java @@ -23,14 +23,20 @@ package org.onap.aai.sa.searchdbabstraction.util; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.apache.commons.io.IOUtils; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.onap.aai.cl.api.Logger; import org.onap.aai.cl.eelf.LoggerFactory; import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; + /** * This class as the name suggests is to translate the payload of PUT & POST requests @@ -52,8 +58,9 @@ public class ElasticSearchPayloadTranslator { /** - * Using String replacement to translate the payload to ES compatible version - * + * Using JSON Path query to filter objects to translate the payload to ES compatible version + * The filter queries and the replacement attributes are configured in the es-payload-translation.json file. + * * @param source * @return translated payload in String * @throws IOException @@ -61,23 +68,33 @@ public class ElasticSearchPayloadTranslator { public static String translateESPayload(String source) throws IOException { logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "translateESPayload, method-params[ source=" + source + "]"); String pathToTranslationFile = CONFIG_DIRECTORY + File.separator + ES_PAYLOAD_TRANSLATION_FILE; - - String translatedPayload = source.replaceAll("\\s+", ""); // removing whitespaces - JSONObject translationConfigPayload = new JSONObject(IOUtils.toString( - new FileInputStream(new File(pathToTranslationFile)), "UTF-8")); - JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations"); + try { + + JSONObject translationConfigPayload = new JSONObject(IOUtils.toString( + new FileInputStream(new File(pathToTranslationFile)), "UTF-8")); + JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations"); + DocumentContext payloadToTranslate = JsonPath.parse(source); - for(Object obj : attrTranslations) { - JSONObject jsonObj = ((JSONObject)obj); - String from = jsonObj.get("from").toString(); - String to = jsonObj.get("to").toString(); - if(translatedPayload.indexOf(from) > 0) { - translatedPayload = translatedPayload.replaceAll(from, to); + for(Object obj : attrTranslations) { + JSONObject jsonObj = ((JSONObject) obj); + String query = jsonObj.get("query").toString(); + JSONObject attrToUpdate = (JSONObject) jsonObj.get("update"); + List> filteredObjects = payloadToTranslate.read(query); + for(Map objMap : filteredObjects) { + objMap.putAll(attrToUpdate.toMap()); + } } - } - - logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+translatedPayload); - return translatedPayload; + + logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+payloadToTranslate.jsonString()); + return payloadToTranslate.jsonString(); + + } catch (JSONException | IOException e) { + logger.error(SearchDbMsgs.FILTERS_CONFIG_FAILURE, e, ES_PAYLOAD_TRANSLATION_FILE, e.getMessage()); + if(e instanceof JSONException) { + throw new IOException("Payload translation configuration looks corrupted. Please correct!", e); + } + throw new IOException("Error in configuring payload translation file. Please check if it exists.", e); + } } } diff --git a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java index 11b2acb..ba5cb48 100644 --- a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java +++ b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java @@ -21,16 +21,19 @@ package org.onap.aai.sa.searchdbabstraction.util; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import java.io.File; +import java.io.FileInputStream; +import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; import org.onap.aai.sa.rest.TestUtils; public class ElasticSearchPayloadTranslatorTest { - private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/simpleDocument.json"; + private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/index-mapping.json"; @Before public void setup() throws Exception { @@ -38,13 +41,17 @@ public class ElasticSearchPayloadTranslatorTest { } @Test - public void testPayloadTranslation_FromStringToText() throws Exception { + public void testPayloadTranslation() throws Exception { + String expectedErrMsg = "Sample error message for whitespace check"; File schemaFile = new File(SIMPLE_DOC_SCHEMA_JSON); - String documentJson = TestUtils.readFileToString(schemaFile); - assertTrue(documentJson.contains("\"data-type\":\"string\"")); - assertTrue(documentJson.contains("\"searchable\":true")); + String documentJson = IOUtils.toString(new FileInputStream(schemaFile), "UTF-8"); + assertTrue(documentJson.contains("\"type\": \"string\"")); + assertTrue(documentJson.contains("\"index\": \"analyzed\"")); String translatedPayload = ElasticSearchPayloadTranslator.translateESPayload(documentJson); - assertTrue(translatedPayload.contains("\"data-type\":\"text\"")); + assertTrue(translatedPayload.contains("\"type\":\"text\"")); assertTrue(translatedPayload.contains("\"index\":true")); + assertTrue(translatedPayload.contains("\"fielddata\":true")); + assertFalse(documentJson.contains("\"index\":\"analyzed\"")); + assertTrue(translatedPayload.contains("\"errMsg\":\""+expectedErrMsg+"\"")); } } diff --git a/src/test/resources/json/es-payload-translation.json b/src/test/resources/json/es-payload-translation.json index e5290b0..58ed8f6 100644 --- a/src/test/resources/json/es-payload-translation.json +++ b/src/test/resources/json/es-payload-translation.json @@ -1,24 +1,16 @@ { - "attr-translations": [ - { - "from": "\"data-type\":\"string\"", - "to": "\"data-type\":\"text\",\"fielddata\":true" - }, + "attr-translations": [ { - "from": "\"type\":\"string\",\"index\":\"analyzed\"", - "to": "\"type\":\"text\",\"index\":\"true\",\"fielddata\":true" + "query": "$..[?(@.type=='string' && @.index=='analyzed')]", + "update": {"type": "text", "index": true, "fielddata": true} }, { - "from": "\"type\":\"string\",\"index\":\"not_analyzed\"", - "to": "\"type\":\"keyword\",\"index\":\"true\"" + "query": "$..[?(@.type=='string' && @.index=='not_analyzed')]", + "update": {"type": "keyword", "index": true} }, { - "from": "\"type\":\"string\"", - "to": "\"type\":\"text\",\"fielddata\":true" - }, - { - "from": "searchable", - "to": "index" + "query": "$..[?(@.type=='string' && !@.index)]", + "update": {"type": "text", "fielddata": true} } ] } \ No newline at end of file diff --git a/src/test/resources/json/index-mapping.json b/src/test/resources/json/index-mapping.json new file mode 100644 index 0000000..e447092 --- /dev/null +++ b/src/test/resources/json/index-mapping.json @@ -0,0 +1,28 @@ +{ + "fields": [ + {"name": "validationId", "type": "string", "searchable": false}, + {"name": "validationTimestamp1", "type": "date", "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM\/dd\/yyyy||yyyyMMdd'T'HHmmssZ"}, + {"name": "entityId", "type": "nested"}, + {"name": "entityType", "type": "string", "index": "analyzed"}, + {"name": "entityLink", "type": "string"}, + + {"name": "resourceVersion", "type": "string", "index": "not_analyzed"}, + {"name": "violations", "type": "nested", "sub-fields": [ + {"name": "violationId", "type": "string"}, + {"name": "violationTimestamp", "type": "date", "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM\/dd\/yyyy||yyyyMMdd'T'HHmmssZ"}, + {"name": "modelName", "type": "string"}, + {"name": "category", "type": "string"}, + {"name": "severity", "type": "string"}, + {"name": "violationType", "type": "string"}, + {"name": "validationRule", "type": "string"}, + {"name": "violationDetails", "type": "nested", "sub-fields": [ + {"name": "entityId", "type": "nested"}, + {"name": "entityType", "type": "string"}, + {"name": "modelName", "type": "string"}, + {"name": "MISSING_REL", "type": "string"} + ]}, + {"name": "errorMessage", "type": "string", "errMsg": "Sample error message for whitespace check"} + ]} + ] + +} -- 2.16.6