From 5ea47abeeb1713b56006fd69b1564cbd4c4220c7 Mon Sep 17 00:00:00 2001 From: sblimkie Date: Mon, 25 Feb 2019 15:24:30 -0500 Subject: [PATCH] Search service configurable index settings When deploying the search service, an optional settings-confg.json file can be used to define the index settings if the deployer does not want to use the default settings. Change-Id: I67902684ac4a432b5a010a7177e9980a0c7592d1 Issue-ID: AAI-2191 Signed-off-by: sblimkie --- .../org/onap/aai/sa/rest/SettingConfiguration.java | 84 ++++++++++++++++++++++ .../dao/ElasticSearchHttpController.java | 20 +++--- .../util/SearchDbConstants.java | 1 + .../onap/aai/sa/rest/SettingConfigurationTest.java | 57 +++++++++++++++ .../dao/ElasticSearchHttpControllerTest.java | 9 ++- src/test/resources/json/settings-config.json | 9 +++ 6 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/onap/aai/sa/rest/SettingConfiguration.java create mode 100644 src/test/java/org/onap/aai/sa/rest/SettingConfigurationTest.java create mode 100644 src/test/resources/json/settings-config.json diff --git a/src/main/java/org/onap/aai/sa/rest/SettingConfiguration.java b/src/main/java/org/onap/aai/sa/rest/SettingConfiguration.java new file mode 100644 index 0000000..74e46f4 --- /dev/null +++ b/src/main/java/org/onap/aai/sa/rest/SettingConfiguration.java @@ -0,0 +1,84 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2019 AT&T Intellectual Property. All rights reserved. + * Copyright © 2019 Amdocs + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.sa.rest; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicBoolean; +import org.onap.aai.sa.searchdbabstraction.util.SearchDbConstants; + + +public class SettingConfiguration { + + /** + * Indicates whether or not we have imported the filter and analyzer configurations. + */ + private AtomicBoolean configured = new AtomicBoolean(false); + + /** + * A json format string which is readable by Elastic Search and defines all of the custom filters and analyzers that + * we need Elastic Search to know about. + */ + private String settings; + + public void init(String settingConfigFile) { + + if (configured.compareAndSet(false, true)) { + try { + Path path = Paths.get(settingConfigFile); + settings = new String(Files.readAllBytes(path)); + + // Remove the enclosing brackets from the json blob. + settings = settings.replaceFirst("\\{", ""); + settings = settings.substring(0, settings.lastIndexOf("}")); + } catch (IOException e) { + // It is valid not to have a settings file. + settings = ""; + } + } + } + + + /** + * Returns the set of pre-configured settings. + * + * @return - settings. + */ + public String getSettings() { + init(SearchDbConstants.SDB_SETTINGS_CONFIG_FILE); + return settings; + } + + public String getSettingsWithAnalysis(AnalysisConfiguration analysisConfig) { + String ac = analysisConfig.getEsIndexSettings(); + StringBuilder sb = new StringBuilder(); + sb.append(ac.substring(0, ac.lastIndexOf("}"))); + + if (!getSettings().trim().isEmpty()) { + sb.append(", " + getSettings()); + } + + sb.append(" }"); + return sb.toString(); + } +} diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java index c4a52b4..a4af160 100644 --- a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java @@ -22,7 +22,6 @@ package org.onap.aai.sa.searchdbabstraction.elasticsearch.dao; import static javax.ws.rs.core.HttpHeaders.CONTENT_TYPE; -import static javax.ws.rs.core.MediaType.APPLICATION_FORM_URLENCODED; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; import com.fasterxml.jackson.annotation.JsonInclude.Include; @@ -72,6 +71,7 @@ import org.onap.aai.sa.rest.ApiUtils; import org.onap.aai.sa.rest.BulkRequest; import org.onap.aai.sa.rest.BulkRequest.OperationType; import org.onap.aai.sa.rest.DocumentSchema; +import org.onap.aai.sa.rest.SettingConfiguration; import org.onap.aai.sa.searchdbabstraction.elasticsearch.config.ElasticSearchConfig; import org.onap.aai.sa.searchdbabstraction.elasticsearch.exception.DocumentStoreOperationException; import org.onap.aai.sa.searchdbabstraction.elasticsearch.exception.DocumentStoreOperationException.ErrorMessage; @@ -133,10 +133,12 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { private final ElasticSearchConfig config; protected AnalysisConfiguration analysisConfig; + protected SettingConfiguration settingConfig; public ElasticSearchHttpController(ElasticSearchConfig config) { this.config = config; analysisConfig = new AnalysisConfiguration(); + settingConfig = new SettingConfiguration(); String rootUrl = null; try { @@ -184,8 +186,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { public OperationResult createIndex(String index, DocumentSchema documentSchema) { try { // Submit the request to ElasticSearch to create the index using a default document type. - OperationResult result = createTable(index, DEFAULT_TYPE, analysisConfig.getEsIndexSettings(), - DocumentSchemaUtil.generateDocumentMappings(documentSchema)); + OperationResult result = createTable(index, DEFAULT_TYPE, analysisConfig, + DocumentSchemaUtil.generateDocumentMappings(documentSchema), settingConfig); // ElasticSearch will return us a 200 code on success when we // want to report a 201, so translate the result here. @@ -236,10 +238,10 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { } // @Override - protected OperationResult createTable(String indexName, String typeName, String indexSettings, String indexMappings) - throws DocumentStoreOperationException { - if (indexSettings == null) { - logger.debug("No settings provided."); + protected OperationResult createTable(String indexName, String typeName, AnalysisConfiguration ac, + String indexMappings, SettingConfiguration sc) throws DocumentStoreOperationException { + if (ac.getEsIndexSettings() == null) { + logger.debug("No analysis settings provided."); } if (indexMappings == null) { @@ -249,10 +251,10 @@ public class ElasticSearchHttpController implements DocumentStoreInterface { MdcOverride override = getStartTime(new MdcOverride()); HttpURLConnection conn = createConnection(buildUrl(createUriBuilder(indexName)), HttpMethod.PUT); - + StringBuilder sb = new StringBuilder(128); sb.append("{ \"settings\" : "); - sb.append(indexSettings); + sb.append(sc.getSettingsWithAnalysis(ac)); sb.append(","); sb.append("\"mappings\" : {"); diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/SearchDbConstants.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/SearchDbConstants.java index 3ffcefc..fd63741 100644 --- a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/SearchDbConstants.java +++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/SearchDbConstants.java @@ -42,6 +42,7 @@ public class SearchDbConstants { public static final String SDB_AUTH_CONFIG_FILENAME = SDB_AUTH + "search_policy.json"; public static final String SDB_FILTER_CONFIG_FILE = SDB_SPECIFIC_CONFIG + "filter-config.json"; public static final String SDB_ANALYSIS_CONFIG_FILE = SDB_SPECIFIC_CONFIG + "analysis-config.json"; + public static final String SDB_SETTINGS_CONFIG_FILE = SDB_SPECIFIC_CONFIG + "settings-config.json"; // Logging related public static final String SDB_SERVICE_NAME = "SearchDataService"; diff --git a/src/test/java/org/onap/aai/sa/rest/SettingConfigurationTest.java b/src/test/java/org/onap/aai/sa/rest/SettingConfigurationTest.java new file mode 100644 index 0000000..9cc4e3b --- /dev/null +++ b/src/test/java/org/onap/aai/sa/rest/SettingConfigurationTest.java @@ -0,0 +1,57 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2019 AT&T Intellectual Property. All rights reserved. + * Copyright © 2019 Amdocs + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.sa.rest; + +import org.junit.Assert; +import org.junit.Test; + + +public class SettingConfigurationTest { + + @Test + public void settingConfigTest() throws Exception { + SettingConfiguration config = new SettingConfiguration(); + config.init("src/test/resources/json/settings-config.json"); + String settings = config.getSettings(); + System.out.println("SettingsConfig:\n" + settings); + Assert.assertTrue(settings.contains("number_of_shards")); + } + + @Test + public void settingConfigAnalysisTest() throws Exception { + AnalysisConfiguration ac = new AnalysisConfiguration(); + ac.init("src/test/resources/json/filter-config.json", "src/test/resources/json/analysis-config.json"); + System.out.println("AnalysisConfig:\n" + ac.buildEsIndexSettings()); + + SettingConfiguration config = new SettingConfiguration(); + config.init("src/test/resources/json/settings-config.json"); + String settings = config.getSettingsWithAnalysis(ac); + System.out.println("SettingsAnalysisConfig:\n" + settings); + Assert.assertTrue(settings.contains("number_of_shards")); + Assert.assertTrue(settings.contains("nGram_analyzer")); + + config = new SettingConfiguration(); + config.init("src/test/resources/json/missing-file.json"); + settings = config.getSettingsWithAnalysis(ac); + System.out.println("SettingsAnalysisConfigMissing:\n" + settings); + Assert.assertFalse(ac.getEsIndexSettings().isEmpty()); + } +} diff --git a/src/test/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpControllerTest.java b/src/test/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpControllerTest.java index 23d0ea2..007e3fc 100644 --- a/src/test/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpControllerTest.java +++ b/src/test/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpControllerTest.java @@ -34,7 +34,9 @@ import org.json.JSONObject; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.onap.aai.sa.rest.AnalysisConfiguration; import org.onap.aai.sa.rest.DocumentSchema; +import org.onap.aai.sa.rest.SettingConfiguration; import org.onap.aai.sa.searchdbabstraction.elasticsearch.config.ElasticSearchConfig; import org.onap.aai.sa.searchdbabstraction.entity.OperationResult; @@ -114,8 +116,13 @@ public class ElasticSearchHttpControllerTest { @Test public void testCreateTable() throws Exception { + AnalysisConfiguration ac = new AnalysisConfiguration(); + ac.init("src/test/resources/json/filter-config.json", "src/test/resources/json/analysis-config.json"); + SettingConfiguration sc = new SettingConfiguration(); + sc.init("src/test/resources/json/settings-config.json"); + OperationResult result = - elasticSearch.createTable(TEST_INDEX_NAME, "aai-entities", indexSettings, indexMappings); + elasticSearch.createTable(TEST_INDEX_NAME, "aai-entities", ac, indexMappings, sc); assertThat(result.getResult(), containsString("\"index\":\"test\"}")); assertThat(result.getResultCode(), either(is(200)).or(is(400))); } diff --git a/src/test/resources/json/settings-config.json b/src/test/resources/json/settings-config.json new file mode 100644 index 0000000..5ebdbcc --- /dev/null +++ b/src/test/resources/json/settings-config.json @@ -0,0 +1,9 @@ +{ + "number_of_shards": "5", + "replication": "TLV_DATACENTER:2", + "drop_on_delete_index": true, + "version": { + "created": "6020399" + } +} + -- 2.16.6