sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/database/ElasticsearchDataMigrationProvider.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * ONAP : ccsdk features
   4  * ================================================================================
   5  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
   6  * All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *     http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  *
  21  */
  22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database;
  23
  24 import java.io.File;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.nio.charset.StandardCharsets;
  28 import java.nio.file.Files;
  29 import java.text.ParseException;
  30 import java.util.ArrayList;
  31 import java.util.Arrays;
  32 import java.util.List;
  33 import java.util.Set;
  34 import org.json.JSONObject;
  35 import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
  36 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit;
  37 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult;
  38 import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo;
  39 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry;
  40 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList;
  41 import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion;
  42 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry;
  43 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
  44 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateAliasRequest;
  45 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateIndexRequest;
  46 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteAliasRequest;
  47 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteIndexRequest;
  48 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.AcknowledgedResponse;
  49 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.GetInfoResponse;
  50 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListAliasesResponse;
  51 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListIndicesResponse;
  52 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType;
  53 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.DataMigrationProviderService;
  54 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
  55 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData;
  56 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
  57 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer;
  58 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport;
  59 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
  60 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup;
  61 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 public class ElasticsearchDataMigrationProvider implements DataMigrationProviderService {
  66
  67
  68     private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDataMigrationProvider.class);
  69     private final HtDatabaseClient dbClient;
  70
  71     public ElasticsearchDataMigrationProvider(String url, String username, String password, boolean trustAll,
  72             long timeoutms) throws Exception {
  73         dbClient = HtDatabaseClient.getClient(new HostInfo[] {HostInfo.parse(url)}, username, password, trustAll,
  74                 true, timeoutms);
  75     }
  76
  77     @Override
  78     public DataMigrationReport importData(String filename, boolean dryrun) throws Exception {
  79         return this.importData(filename, dryrun, Release.CURRENT_RELEASE);
  80     }
  81
  82     @Override
  83     public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception {
  84         DataMigrationReport report = new DataMigrationReport();
  85         File file = new File(filename);
  86         if (!file.exists()) {
  87             if (dryrun) {
  88                 report.error("file %s not found", filename);
  89                 return report;
  90             }
  91             throw new FileNotFoundException(filename);
  92         }
  93         DataContainer container = null;
  94         try {
  95             container = DataContainer.load(file);
  96         } catch (Exception e) {
  97             if (dryrun) {
  98                 report.error("problem loading file %s: %s", filename, e.getMessage());
  99                 return report;
 100             }
 101             throw new Exception("problem loading file " + filename, e);
 102         }
 103         ReleaseInformation ri = ReleaseInformation.getInstance(forRelease);
 104         SearchHitConverter converter;
 105         Set<ComponentName> components = ri.getComponents();
 106         //for all db components of dest architecture
 107         for (ComponentName component : components) {
 108             //convert to ComponentData for current release with existing ComponentData of the container
 109             converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component);
 110             if (converter == null) {
 111                 continue;
 112             }
 113             ComponentData data = converter.convert(container);
 114             if (data != null) {
 115                 String indexName = ri.getAlias(component);
 116                 String dataTypeName = ri.getDataType(component);
 117                 if (dryrun) {
 118                     report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName);
 119                 } else {
 120                     LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName);
 121                 }
 122                 for (SearchHit item : data) {
 123                     if (!dryrun) {
 124                         String id = this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(),
 125                                 item.getSourceAsString(), true);
 126                         if (!item.getId().equals(id)) {
 127                             LOG.warn("entry for {} with original id {} was written with another id {}",
 128                                     component.getValue(), item.getId(), id);
 129                         }
 130                     }
 131                 }
 132             } else {
 133                 if (dryrun) {
 134                     report.error("unable to convert data for " + component.getValue() + " from version "
 135                             + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n");
 136                 } else {
 137                     LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(),
 138                             container.getRelease().getValue(), forRelease.getValue());
 139                 }
 140             }
 141         }
 142         LOG.info("import of {} completed", filename);
 143         if (dryrun) {
 144             report.log("import of %s completed", filename);
 145         }
 146         report.setCompleted(true);
 147         return report;
 148     }
 149
 150
 151     /**
 152      * export data if file exists .1 (.n) will be created
 153      *
 154      */
 155     @Override
 156     public DataMigrationReport exportData(String filename) {
 157         DataMigrationReport report = new DataMigrationReport();
 158
 159         DataContainer container = new DataContainer();
 160
 161         filename = this.checkFilenameForWrite(filename);
 162         LOG.info("output will be written to {}", filename);
 163         //autodetect version
 164         Release dbRelease = this.autoDetectRelease();
 165         if (dbRelease == null) {
 166             report.error("unbable to detect db release. is database initialized?");
 167             return report;
 168         }
 169         ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease);
 170         boolean componentsSucceeded = true;
 171         for (ComponentName c : ri.getComponents()) {
 172             ComponentData data = new ComponentData(c);
 173             SearchResult<SearchHit> result = this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false);
 174             data.addAll(result.getHits());
 175             container.addComponent(c, data);
 176         }
 177         try {
 178             Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8);
 179             report.setCompleted(componentsSucceeded);
 180         } catch (IOException e) {
 181             LOG.warn("problem writing data to {}: {}", filename, e);
 182         }
 183         return report;
 184     }
 185
 186     private String checkFilenameForWrite(String filename) {
 187         File f = new File(filename);
 188         if (!f.exists()) {
 189             return filename;
 190         }
 191         return this.checkFilenameForWrite(filename, 0);
 192     }
 193
 194     private String checkFilenameForWrite(String filename, int apdx) {
 195         File f = new File(String.format("$s.$d", filename, apdx));
 196         if (!f.exists()) {
 197             return filename;
 198         }
 199         return this.checkFilenameForWrite(filename, apdx + 1);
 200     }
 201
 202     @Override
 203     public Release getCurrentVersion() {
 204         return Release.CURRENT_RELEASE;
 205     }
 206
 207
 208     @Override
 209     public Release autoDetectRelease() {
 210         DatabaseVersion dbVersion = this.readActualVersion();
 211         AliasesEntryList aliases = this.readAliases();
 212         IndicesEntryList indices = this.readIndices();
 213         if (indices == null) {
 214             return null;
 215         }
 216         List<Release> foundReleases = new ArrayList<>();
 217         //if there are active aliases reduce indices to the active ones
 218         if (aliases != null && aliases.size() > 0) {
 219             indices = indices.subList(aliases.getLinkedIndices());
 220         }
 221         for (Release r : Release.values()) {
 222             if (r.isDbInRange(dbVersion, SdnrDbType.ELASTICSEARCH)) {
 223                 ReleaseInformation ri = ReleaseInformation.getInstance(r);
 224                 if (ri != null && ri.containsIndices(indices)) {
 225                     foundReleases.add(r);
 226                 }
 227             }
 228         }
 229         if (foundReleases.size() == 1) {
 230             return foundReleases.get(0);
 231         }
 232         LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(),
 233                 foundReleases);
 234         return null;
 235     }
 236
 237     private DatabaseVersion readActualVersion() {
 238         try {
 239             GetInfoResponse response = this.dbClient.getInfo();
 240             return response.getVersion();
 241         } catch (Exception e) {
 242             LOG.warn(e.getMessage());
 243         }
 244         return null;
 245     }
 246
 247     private AliasesEntryList readAliases() {
 248         AliasesEntryList entries = null;
 249         try {
 250             ListAliasesResponse response = this.dbClient.getAliases();
 251             entries = response.getEntries();
 252         } catch (ParseException | IOException e) {
 253             LOG.error(e.getMessage());
 254         }
 255         return entries;
 256     }
 257
 258     private IndicesEntryList readIndices() {
 259         IndicesEntryList entries = null;
 260         try {
 261             ListIndicesResponse response = this.dbClient.getIndices();
 262             entries = response.getEntries();
 263         } catch (ParseException | IOException e) {
 264             LOG.error(e.getMessage());
 265         }
 266         return entries;
 267     }
 268
 269
 270     @Override
 271     public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate,
 272             long timeoutms) {
 273         if (timeoutms > 0) {
 274             this.dbClient.waitForYellowStatus(timeoutms);
 275         }
 276         DatabaseVersion dbVersion = this.readActualVersion();
 277         if (dbVersion == null) {
 278             return false;
 279         }
 280         LOG.info("detected database version {}", dbVersion);
 281         if (release == null) {
 282             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.ELASTICSEARCH);
 283             if (release == null) {
 284                 LOG.warn("unable to autodetect release for this database version for release {}",
 285                         ReleaseGroup.CURRENT_RELEASE.name());
 286                 return false;
 287             }
 288             LOG.info("autodetect release {}", release);
 289         }
 290         if (!release.isDbInRange(dbVersion, SdnrDbType.ELASTICSEARCH)) {
 291             LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release);
 292             return false;
 293         }
 294         if (forceRecreate) {
 295             this.clearDatabase(release, dbPrefix, 0);
 296         }
 297         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 298         AliasesEntryList aliases = this.readAliases();
 299         IndicesEntryList indices = this.readIndices();
 300         if (aliases == null || indices == null) {
 301             return false;
 302         }
 303         AcknowledgedResponse response = null;
 304         if (!ri.runPreInitCommands(this.dbClient)) {
 305             return false;
 306         }
 307         for (ComponentName component : ri.getComponents()) {
 308             try {
 309                 if (ri.hasOwnDbIndex(component)) {
 310                     //check if index already exists
 311                     String indexName = ri.getIndex(component, dbPrefix);
 312                     String aliasName = ri.getAlias(component, dbPrefix);
 313                     if (indices.findByIndex(indexName) == null) {
 314                         LOG.info("creating index for {}", component);
 315                         CreateIndexRequest request = new CreateIndexRequest(ri.getIndex(component, dbPrefix));
 316                         request.mappings(new JSONObject(ri.getDatabaseMapping(component)));
 317                         request.settings(new JSONObject(ri.getDatabaseSettings(component, numShards, numReplicas)));
 318                         response = this.dbClient.createIndex(request);
 319                         LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 320                     } else {
 321                         LOG.info("index {} for {} already exists", indexName, component);
 322                     }
 323                     //check if alias already exists
 324                     if (aliases.findByAlias(aliasName) == null) {
 325                         LOG.info("creating alias for {}", component);
 326                         response = this.dbClient.createAlias(new CreateAliasRequest(indexName, aliasName));
 327                         LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 328                     } else {
 329                         LOG.info("alias {} for index {} for {} already exists", aliasName, indexName, component);
 330                     }
 331                 }
 332             } catch (IOException e) {
 333                 LOG.error(e.getMessage());
 334                 return false;
 335             }
 336         }
 337         if (!ri.runPostInitCommands(this.dbClient)) {
 338             return false;
 339         }
 340         return true;
 341     }
 342
 343     @Override
 344     public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) {
 345
 346         if (timeoutms > 0) {
 347             this.dbClient.waitForYellowStatus(timeoutms);
 348         }
 349         //check aliases
 350         AliasesEntryList entries = this.readAliases();
 351         IndicesEntryList entries2 = this.readIndices();
 352         if (entries == null) {
 353             return false;
 354         }
 355         if (release == null) {
 356             DatabaseVersion dbVersion = this.readActualVersion();
 357             if (dbVersion == null) {
 358                 return false;
 359             }
 360             LOG.info("detected database version {}", dbVersion);
 361             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.ELASTICSEARCH);
 362             if (release == null) {
 363                 LOG.warn("unable to autodetect release for this database version for release {}",
 364                         ReleaseGroup.CURRENT_RELEASE.name());
 365                 return false;
 366             }
 367             LOG.info("autodetect release {}", release);
 368         }
 369         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 370         AcknowledgedResponse response;
 371         if (entries.size() <= 0) {
 372             LOG.info("no aliases to clear");
 373         } else {
 374             //check for every component of release if alias exists
 375             for (ComponentName component : ri.getComponents()) {
 376                 String aliasToDelete = ri.getAlias(component, dbPrefix);
 377                 AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
 378                 if (entryToDelete != null) {
 379                     try {
 380                         LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex());
 381                         response = this.dbClient.deleteAlias(
 382                                 new DeleteAliasRequest(entryToDelete.getIndex(), entryToDelete.getAlias()));
 383                         LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 384                     } catch (IOException e) {
 385                         LOG.error(e.getMessage());
 386                         return false;
 387                     }
 388                 } else {
 389                     //try to find malformed typed index with alias name
 390                     IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete);
 391                     if (entry2ToDelete != null) {
 392                         try {
 393                             LOG.info("deleting index {}", entry2ToDelete.getName());
 394                             response = this.dbClient.deleteIndex(new DeleteIndexRequest(entry2ToDelete.getName()));
 395                             LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 396                         } catch (IOException e) {
 397                             LOG.error(e.getMessage());
 398                             return false;
 399                         }
 400                     }
 401                 }
 402             }
 403         }
 404         if (entries2 == null) {
 405             return false;
 406         }
 407         if (entries2.size() <= 0) {
 408             LOG.info("no indices to clear");
 409         } else {
 410             //check for every component of release if index exists
 411             for (ComponentName component : ri.getComponents()) {
 412                 String indexToDelete = ri.getIndex(component, dbPrefix);
 413                 IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
 414                 if (entryToDelete != null) {
 415                     try {
 416                         LOG.info("deleting index {}", entryToDelete.getName());
 417                         response = this.dbClient.deleteIndex(new DeleteIndexRequest(entryToDelete.getName()));
 418                         LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 419                     } catch (IOException e) {
 420                         LOG.error(e.getMessage());
 421                         return false;
 422                     }
 423                 }
 424             }
 425         }
 426
 427         return true;
 428     }
 429
 430     /**
 431      * @param timeoutms
 432      * @return
 433      */
 434     public boolean clearCompleteDatabase(long timeoutms) {
 435         if (timeoutms > 0) {
 436             this.dbClient.waitForYellowStatus(timeoutms);
 437         }
 438         //check aliases and indices
 439         AliasesEntryList aliases = this.readAliases();
 440         IndicesEntryList indices = this.readIndices();
 441         if (aliases == null || indices == null) {
 442             return false;
 443         }
 444         for (AliasesEntry alias : aliases) {
 445             try {
 446                 LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex());
 447                 this.dbClient.deleteAlias(new DeleteAliasRequest(alias.getIndex(), alias.getAlias()));
 448             } catch (IOException e) {
 449                 LOG.error("problem deleting alias {}: {}", alias.getAlias(), e);
 450                 return false;
 451             }
 452         }
 453         for (IndicesEntry index : indices) {
 454             try {
 455                 LOG.info("deleting index {}", index.getName());
 456                 this.dbClient.deleteIndex(new DeleteIndexRequest(index.getName()));
 457             } catch (IOException e) {
 458                 LOG.error("problem deleting index {}: {}", index.getName(), e);
 459                 return false;
 460             }
 461         }
 462         return true;
 463     }
 464
 465 }