From: Edwin Lawrance Date: Thu, 12 Jul 2018 15:28:57 +0000 (+0100) Subject: Updating Search service to be ES 6.1.2 compliant X-Git-Tag: 1.3.1~20^2 X-Git-Url: https://gerrit.onap.org/r/gitweb?p=aai%2Fsearch-data-service.git;a=commitdiff_plain;h=8fb6b69f8ddc607d0413dc851f629a2e300e1be9 Updating Search service to be ES 6.1.2 compliant Payload to Elastic Search is translated to comply ES6.1.2 PUT and POST calls now have content-type header Added functionality for dynamic templates Change-Id: I2a44a8a9999ec01a3bad1fb6999fe35bb6ef70d1 Issue-ID: AAI-1376 Signed-off-by: Edwin Lawrance --- diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java index 626ea8c..90adbd7 100644 --- a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java @@ -48,6 +48,7 @@ import org.onap.aai.sa.searchdbabstraction.entity.SearchOperationResult; import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs; import org.onap.aai.sa.searchdbabstraction.util.AggregationParsingUtil; import org.onap.aai.sa.searchdbabstraction.util.DocumentSchemaUtil; +import org.onap.aai.sa.searchdbabstraction.util.ElasticSearchPayloadTranslator; import org.onap.aai.sa.searchdbabstraction.util.SearchDbConstants; import org.onap.aai.cl.api.LogFields; import org.onap.aai.cl.api.LogLine; @@ -179,7 +180,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { //result.setResult("{\"index\": \"" + index + ", \"type\": \"" + DEFAULT_TYPE + "\"}"); } - } catch (DocumentStoreOperationException e) { + } catch (DocumentStoreOperationException | IOException e) { result.setFailureCause("Document store operation failure. Cause: " + e.getMessage()); } @@ -350,6 +351,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { conn.setRequestMethod("PUT"); + conn.setRequestProperty("Content-Type", "application/json"); } catch (ProtocolException e) { shutdownConnection(conn); throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e); @@ -365,7 +367,11 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { sb.append(indexMappings); sb.append("}}"); - attachContent(conn, sb.toString()); + try { + attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(sb.toString())); + } catch(IOException e) { + throw new DocumentStoreOperationException("Error in translating Index payload to make it ES compliant.", e); + } logger.debug("\ncreateTable(), Sending 'PUT' request to URL : " + conn.getURL()); logger.debug("Request content: " + sb.toString()); @@ -407,12 +413,17 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { conn.setRequestMethod("PUT"); + conn.setRequestProperty("Content-Type", "application/json"); } catch (ProtocolException e) { shutdownConnection(conn); throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e); } - attachContent(conn, settingsAndMappings); + try { + attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(settingsAndMappings)); + } catch(IOException e) { + throw new DocumentStoreOperationException("Error in translating DynamicIndex payload to make it ES compliant.", e); + } handleResponse(conn, result); // Generate a metrics log so we can track how long the operation took. @@ -557,10 +568,15 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { private void attachDocument(HttpURLConnection conn, DocumentStoreDataEntity doc) throws DocumentStoreOperationException { - conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); +// conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + conn.setRequestProperty("Content-Type", "application/json"); conn.setRequestProperty("Connection", "Close"); - - attachContent(conn, doc.getContentInJson()); + + try { + attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(doc.getContentInJson())); + } catch(IOException e) { + throw new DocumentStoreOperationException("Error in translating Document payload to make it ES compliant.", e); + } } private DocumentOperationResult checkDocumentExistence(String indexName, @@ -650,6 +666,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { conn.setRequestMethod("PUT"); + conn.setRequestProperty("Content-Type", "application/json"); } catch (ProtocolException e) { shutdownConnection(conn); throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e); @@ -823,6 +840,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); } catch (ProtocolException e) { shutdownConnection(conn); throw new DocumentStoreOperationException("Failed to set HTTP request method to POST.", e); @@ -872,6 +890,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { try { conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); } catch (ProtocolException e) { shutdownConnection(conn); throw new DocumentStoreOperationException("Failed to set HTTP request method to POST.", e); diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java index 8c79022..3dc03f9 100644 --- a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java @@ -23,15 +23,23 @@ package org.onap.aai.sa.searchdbabstraction.util; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.commons.io.IOUtils; import org.onap.aai.sa.rest.DocumentFieldSchema; import org.onap.aai.sa.rest.DocumentSchema; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; public class DocumentSchemaUtil { + private static String dynamicCustomMapping = null; + private static final String DYNAMIC_CUSTOM_TEMPALTE_FILE = System.getProperty("CONFIG_HOME") + File.separator + + "dynamic-custom-template.json"; + public static String generateDocumentMappings(String documentSchema) throws JsonParseException, JsonMappingException, IOException { @@ -42,11 +50,23 @@ public class DocumentSchemaUtil { return generateDocumentMappings(schema); } - public static String generateDocumentMappings(DocumentSchema schema) { + public static String generateDocumentMappings(DocumentSchema schema) throws IOException { + + // Adding dynamic template to add fielddata=true to dynamic fields of type "string" + // in order to avoid aggregation queries breaking in ESv6.1.2 + if(dynamicCustomMapping == null) { + try { + dynamicCustomMapping = IOUtils.toString(new FileInputStream(DYNAMIC_CUSTOM_TEMPALTE_FILE), "UTF-8").replaceAll("\\s+", ""); + } catch (IOException e) { + throw new IOException("Dynamic Custom template configuration went wrong! Please check for the correct template file.", e); + } + } // Now, generate the Elastic Search mapping json and return it. StringBuilder sb = new StringBuilder(); sb.append("{"); + // Adding custom mapping which adds fielddata=true to dynamic fields of type "string" + sb.append(dynamicCustomMapping != null ? dynamicCustomMapping : ""); sb.append("\"properties\": {"); generateFieldMappings(schema.getFields(), sb); diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java new file mode 100644 index 0000000..2e97103 --- /dev/null +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java @@ -0,0 +1,83 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017-2018 Amdocs + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.sa.searchdbabstraction.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs; + + +/** + * This class as the name suggests is to translate the payload of PUT & POST requests + * to ElasticSearch (ES) to its compatible syntax, specially compatible with ES v6 or above. + * + * For example, data type such as "string" is now replaced by "text" or "keyword". + * + * So this class will make those translations reading off from a json configuration file, therefore + * the configuration can be updated with new translations as and when required without touching the code. + * + * @author EDWINL + * + */ +public class ElasticSearchPayloadTranslator { + + private static Logger logger = LoggerFactory.getInstance().getLogger(ElasticSearchPayloadTranslator.class.getName()); + private static final String CONFIG_DIRECTORY = System.getProperty("CONFIG_HOME"); + private static final String ES_PAYLOAD_TRANSLATION_FILE = "es-payload-translation.json"; + + + /** + * Using String replacement to translate the payload to ES compatible version + * + * @param source + * @return translated payload in String + * @throws IOException + */ + public static String translateESPayload(String source) throws IOException { + logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "translateESPayload, method-params[ source=" + source + "]"); + String pathToTranslationFile = CONFIG_DIRECTORY + File.separator + ES_PAYLOAD_TRANSLATION_FILE; + + String translatedPayload = source.replaceAll("\\s+", ""); // removing whitespaces + JSONObject translationConfigPayload = new JSONObject(IOUtils.toString( + new FileInputStream(new File(pathToTranslationFile)), "UTF-8")); + + JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations"); + + for(Object obj : attrTranslations) { + JSONObject jsonObj = ((JSONObject)obj); + String from = jsonObj.get("from").toString(); + String to = jsonObj.get("to").toString(); + if(translatedPayload.indexOf(from) > 0) { + translatedPayload = translatedPayload.replaceAll(from, to); + } + } + + logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+translatedPayload); + return translatedPayload; + } +} diff --git a/src/test/java/org/onap/aai/sa/rest/IndexApiTest.java b/src/test/java/org/onap/aai/sa/rest/IndexApiTest.java index 8f3bfac..60e798a 100644 --- a/src/test/java/org/onap/aai/sa/rest/IndexApiTest.java +++ b/src/test/java/org/onap/aai/sa/rest/IndexApiTest.java @@ -21,6 +21,7 @@ package org.onap.aai.sa.rest; +import org.junit.Before; // import org.glassfish.jersey.server.ResourceConfig; // import org.glassfish.jersey.test.JerseyTest; import org.junit.Test; @@ -75,6 +76,11 @@ public class IndexApiTest { // } // // + + @Before + public void setup() throws Exception { + System.setProperty("CONFIG_HOME", System.getProperty("user.dir")+ File.separator + "src/test/resources/json"); + } /** * Tests the dynamic shcema creation flow that send the request @@ -177,7 +183,8 @@ public class IndexApiTest { + "\"tokenizer\": \"whitespace\"," + "\"filter\": [\"lowercase\",\"asciifolding\"]}}}}"; String EXPECTED_MAPPINGS = - "{\"properties\": {" + "{\"dynamic_templates\":[{\"strings\":{\"match_mapping_type\":\"string\",\"match\":\"*\",\"mapping\":{\"type\":\"text\",\"fielddata\":true}}}]" + + ",\"properties\": {" + "\"serverName\": {" + "\"type\": \"string\", " + "\"index\": \"analyzed\", " diff --git a/src/test/java/org/onap/aai/sa/rest/StubEsController.java b/src/test/java/org/onap/aai/sa/rest/StubEsController.java index fe72090..8cd25a6 100644 --- a/src/test/java/org/onap/aai/sa/rest/StubEsController.java +++ b/src/test/java/org/onap/aai/sa/rest/StubEsController.java @@ -31,6 +31,7 @@ import org.onap.aai.sa.searchdbabstraction.util.DocumentSchemaUtil; import org.onap.aai.sa.searchdbabstraction.entity.Document; import org.onap.aai.sa.rest.DocumentSchema; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -67,8 +68,12 @@ public class StubEsController implements DocumentStoreInterface { OperationResult opResult = new OperationResult(); opResult.setResultCode(200); - opResult.setResult(index + "@" + analysisConfig.getEsIndexSettings() + "@" - + DocumentSchemaUtil.generateDocumentMappings(documentSchema)); + try { + opResult.setResult(index + "@" + analysisConfig.getEsIndexSettings() + "@" + + DocumentSchemaUtil.generateDocumentMappings(documentSchema)); + } catch (IOException e) { + e.printStackTrace(); + } return opResult; } diff --git a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java new file mode 100644 index 0000000..11b2acb --- /dev/null +++ b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java @@ -0,0 +1,50 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017-2018 Amdocs + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.sa.searchdbabstraction.util; + +import static org.junit.Assert.assertTrue; + +import java.io.File; + +import org.junit.Before; +import org.junit.Test; +import org.onap.aai.sa.rest.TestUtils; + +public class ElasticSearchPayloadTranslatorTest { + + private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/simpleDocument.json"; + + @Before + public void setup() throws Exception { + System.setProperty("CONFIG_HOME", System.getProperty("user.dir")+ File.separator + "src/test/resources/json"); + } + + @Test + public void testPayloadTranslation_FromStringToText() throws Exception { + File schemaFile = new File(SIMPLE_DOC_SCHEMA_JSON); + String documentJson = TestUtils.readFileToString(schemaFile); + assertTrue(documentJson.contains("\"data-type\":\"string\"")); + assertTrue(documentJson.contains("\"searchable\":true")); + String translatedPayload = ElasticSearchPayloadTranslator.translateESPayload(documentJson); + assertTrue(translatedPayload.contains("\"data-type\":\"text\"")); + assertTrue(translatedPayload.contains("\"index\":true")); + } +} diff --git a/src/test/resources/json/dynamic-custom-template.json b/src/test/resources/json/dynamic-custom-template.json new file mode 100644 index 0000000..a7bd5ae --- /dev/null +++ b/src/test/resources/json/dynamic-custom-template.json @@ -0,0 +1,12 @@ +"dynamic_templates":[ + { + "strings":{ + "match_mapping_type":"string", + "match": "*", + "mapping":{ + "type":"text", + "fielddata":true + } + } + } +], \ No newline at end of file diff --git a/src/test/resources/json/es-payload-translation.json b/src/test/resources/json/es-payload-translation.json new file mode 100644 index 0000000..e5290b0 --- /dev/null +++ b/src/test/resources/json/es-payload-translation.json @@ -0,0 +1,24 @@ +{ + "attr-translations": [ + { + "from": "\"data-type\":\"string\"", + "to": "\"data-type\":\"text\",\"fielddata\":true" + }, + { + "from": "\"type\":\"string\",\"index\":\"analyzed\"", + "to": "\"type\":\"text\",\"index\":\"true\",\"fielddata\":true" + }, + { + "from": "\"type\":\"string\",\"index\":\"not_analyzed\"", + "to": "\"type\":\"keyword\",\"index\":\"true\"" + }, + { + "from": "\"type\":\"string\"", + "to": "\"type\":\"text\",\"fielddata\":true" + }, + { + "from": "searchable", + "to": "index" + } + ] +} \ No newline at end of file