import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs;
import org.onap.aai.sa.searchdbabstraction.util.AggregationParsingUtil;
import org.onap.aai.sa.searchdbabstraction.util.DocumentSchemaUtil;
+import org.onap.aai.sa.searchdbabstraction.util.ElasticSearchPayloadTranslator;
import org.onap.aai.sa.searchdbabstraction.util.SearchDbConstants;
import org.onap.aai.cl.api.LogFields;
import org.onap.aai.cl.api.LogLine;
//result.setResult("{\"index\": \"" + index + ", \"type\": \"" + DEFAULT_TYPE + "\"}");
}
- } catch (DocumentStoreOperationException e) {
+ } catch (DocumentStoreOperationException | IOException e) {
result.setFailureCause("Document store operation failure. Cause: " + e.getMessage());
}
try {
conn.setRequestMethod("PUT");
+ conn.setRequestProperty("Content-Type", "application/json");
} catch (ProtocolException e) {
shutdownConnection(conn);
throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e);
sb.append(indexMappings);
sb.append("}}");
- attachContent(conn, sb.toString());
+ try {
+ attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(sb.toString()));
+ } catch(IOException e) {
+ throw new DocumentStoreOperationException("Error in translating Index payload to make it ES compliant.", e);
+ }
logger.debug("\ncreateTable(), Sending 'PUT' request to URL : " + conn.getURL());
logger.debug("Request content: " + sb.toString());
try {
conn.setRequestMethod("PUT");
+ conn.setRequestProperty("Content-Type", "application/json");
} catch (ProtocolException e) {
shutdownConnection(conn);
throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e);
}
- attachContent(conn, settingsAndMappings);
+ try {
+ attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(settingsAndMappings));
+ } catch(IOException e) {
+ throw new DocumentStoreOperationException("Error in translating DynamicIndex payload to make it ES compliant.", e);
+ }
handleResponse(conn, result);
// Generate a metrics log so we can track how long the operation took.
private void attachDocument(HttpURLConnection conn, DocumentStoreDataEntity doc)
throws DocumentStoreOperationException {
- conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+// conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+ conn.setRequestProperty("Content-Type", "application/json");
conn.setRequestProperty("Connection", "Close");
-
- attachContent(conn, doc.getContentInJson());
+
+ try {
+ attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(doc.getContentInJson()));
+ } catch(IOException e) {
+ throw new DocumentStoreOperationException("Error in translating Document payload to make it ES compliant.", e);
+ }
}
private DocumentOperationResult checkDocumentExistence(String indexName,
try {
conn.setRequestMethod("PUT");
+ conn.setRequestProperty("Content-Type", "application/json");
} catch (ProtocolException e) {
shutdownConnection(conn);
throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e);
try {
conn.setRequestMethod("POST");
+ conn.setRequestProperty("Content-Type", "application/json");
} catch (ProtocolException e) {
shutdownConnection(conn);
throw new DocumentStoreOperationException("Failed to set HTTP request method to POST.", e);
try {
conn.setRequestMethod("POST");
+ conn.setRequestProperty("Content-Type", "application/json");
} catch (ProtocolException e) {
shutdownConnection(conn);
throw new DocumentStoreOperationException("Failed to set HTTP request method to POST.", e);
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.commons.io.IOUtils;
import org.onap.aai.sa.rest.DocumentFieldSchema;
import org.onap.aai.sa.rest.DocumentSchema;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class DocumentSchemaUtil {
+ private static String dynamicCustomMapping = null;
+ private static final String DYNAMIC_CUSTOM_TEMPALTE_FILE = System.getProperty("CONFIG_HOME") + File.separator
+ + "dynamic-custom-template.json";
+
public static String generateDocumentMappings(String documentSchema)
throws JsonParseException, JsonMappingException, IOException {
return generateDocumentMappings(schema);
}
- public static String generateDocumentMappings(DocumentSchema schema) {
+ public static String generateDocumentMappings(DocumentSchema schema) throws IOException {
+
+ // Adding dynamic template to add fielddata=true to dynamic fields of type "string"
+ // in order to avoid aggregation queries breaking in ESv6.1.2
+ if(dynamicCustomMapping == null) {
+ try {
+ dynamicCustomMapping = IOUtils.toString(new FileInputStream(DYNAMIC_CUSTOM_TEMPALTE_FILE), "UTF-8").replaceAll("\\s+", "");
+ } catch (IOException e) {
+ throw new IOException("Dynamic Custom template configuration went wrong! Please check for the correct template file.", e);
+ }
+ }
// Now, generate the Elastic Search mapping json and return it.
StringBuilder sb = new StringBuilder();
sb.append("{");
+ // Adding custom mapping which adds fielddata=true to dynamic fields of type "string"
+ sb.append(dynamicCustomMapping != null ? dynamicCustomMapping : "");
sb.append("\"properties\": {");
generateFieldMappings(schema.getFields(), sb);
--- /dev/null
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.sa.searchdbabstraction.util;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.onap.aai.cl.api.Logger;
+import org.onap.aai.cl.eelf.LoggerFactory;
+import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs;
+
+
+/**
+ * This class as the name suggests is to translate the payload of PUT & POST requests
+ * to ElasticSearch (ES) to its compatible syntax, specially compatible with ES v6 or above.
+ *
+ * For example, data type such as "string" is now replaced by "text" or "keyword".
+ *
+ * So this class will make those translations reading off from a json configuration file, therefore
+ * the configuration can be updated with new translations as and when required without touching the code.
+ *
+ * @author EDWINL
+ *
+ */
+public class ElasticSearchPayloadTranslator {
+
+ private static Logger logger = LoggerFactory.getInstance().getLogger(ElasticSearchPayloadTranslator.class.getName());
+ private static final String CONFIG_DIRECTORY = System.getProperty("CONFIG_HOME");
+ private static final String ES_PAYLOAD_TRANSLATION_FILE = "es-payload-translation.json";
+
+
+ /**
+ * Using String replacement to translate the payload to ES compatible version
+ *
+ * @param source
+ * @return translated payload in String
+ * @throws IOException
+ */
+ public static String translateESPayload(String source) throws IOException {
+ logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "translateESPayload, method-params[ source=" + source + "]");
+ String pathToTranslationFile = CONFIG_DIRECTORY + File.separator + ES_PAYLOAD_TRANSLATION_FILE;
+
+ String translatedPayload = source.replaceAll("\\s+", ""); // removing whitespaces
+ JSONObject translationConfigPayload = new JSONObject(IOUtils.toString(
+ new FileInputStream(new File(pathToTranslationFile)), "UTF-8"));
+
+ JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations");
+
+ for(Object obj : attrTranslations) {
+ JSONObject jsonObj = ((JSONObject)obj);
+ String from = jsonObj.get("from").toString();
+ String to = jsonObj.get("to").toString();
+ if(translatedPayload.indexOf(from) > 0) {
+ translatedPayload = translatedPayload.replaceAll(from, to);
+ }
+ }
+
+ logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+translatedPayload);
+ return translatedPayload;
+ }
+}
package org.onap.aai.sa.rest;
+import org.junit.Before;
// import org.glassfish.jersey.server.ResourceConfig;
// import org.glassfish.jersey.test.JerseyTest;
import org.junit.Test;
// }
//
//
+
+ @Before
+ public void setup() throws Exception {
+ System.setProperty("CONFIG_HOME", System.getProperty("user.dir")+ File.separator + "src/test/resources/json");
+ }
/**
* Tests the dynamic shcema creation flow that send the request
+ "\"tokenizer\": \"whitespace\","
+ "\"filter\": [\"lowercase\",\"asciifolding\"]}}}}";
String EXPECTED_MAPPINGS =
- "{\"properties\": {"
+ "{\"dynamic_templates\":[{\"strings\":{\"match_mapping_type\":\"string\",\"match\":\"*\",\"mapping\":{\"type\":\"text\",\"fielddata\":true}}}]"
+ + ",\"properties\": {"
+ "\"serverName\": {"
+ "\"type\": \"string\", "
+ "\"index\": \"analyzed\", "
import org.onap.aai.sa.searchdbabstraction.entity.Document;
import org.onap.aai.sa.rest.DocumentSchema;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
OperationResult opResult = new OperationResult();
opResult.setResultCode(200);
- opResult.setResult(index + "@" + analysisConfig.getEsIndexSettings() + "@"
- + DocumentSchemaUtil.generateDocumentMappings(documentSchema));
+ try {
+ opResult.setResult(index + "@" + analysisConfig.getEsIndexSettings() + "@"
+ + DocumentSchemaUtil.generateDocumentMappings(documentSchema));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
return opResult;
}
--- /dev/null
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.sa.searchdbabstraction.util;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.onap.aai.sa.rest.TestUtils;
+
+public class ElasticSearchPayloadTranslatorTest {
+
+ private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/simpleDocument.json";
+
+ @Before
+ public void setup() throws Exception {
+ System.setProperty("CONFIG_HOME", System.getProperty("user.dir")+ File.separator + "src/test/resources/json");
+ }
+
+ @Test
+ public void testPayloadTranslation_FromStringToText() throws Exception {
+ File schemaFile = new File(SIMPLE_DOC_SCHEMA_JSON);
+ String documentJson = TestUtils.readFileToString(schemaFile);
+ assertTrue(documentJson.contains("\"data-type\":\"string\""));
+ assertTrue(documentJson.contains("\"searchable\":true"));
+ String translatedPayload = ElasticSearchPayloadTranslator.translateESPayload(documentJson);
+ assertTrue(translatedPayload.contains("\"data-type\":\"text\""));
+ assertTrue(translatedPayload.contains("\"index\":true"));
+ }
+}
--- /dev/null
+"dynamic_templates":[
+ {
+ "strings":{
+ "match_mapping_type":"string",
+ "match": "*",
+ "mapping":{
+ "type":"text",
+ "fielddata":true
+ }
+ }
+ }
+],
\ No newline at end of file
--- /dev/null
+{
+ "attr-translations": [
+ {
+ "from": "\"data-type\":\"string\"",
+ "to": "\"data-type\":\"text\",\"fielddata\":true"
+ },
+ {
+ "from": "\"type\":\"string\",\"index\":\"analyzed\"",
+ "to": "\"type\":\"text\",\"index\":\"true\",\"fielddata\":true"
+ },
+ {
+ "from": "\"type\":\"string\",\"index\":\"not_analyzed\"",
+ "to": "\"type\":\"keyword\",\"index\":\"true\""
+ },
+ {
+ "from": "\"type\":\"string\"",
+ "to": "\"type\":\"text\",\"fielddata\":true"
+ },
+ {
+ "from": "searchable",
+ "to": "index"
+ }
+ ]
+}
\ No newline at end of file
Code Review / aai / search-data-service.git / commitdiff