gerrit.onap Code Review - ccsdk/features.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * ONAP : ccsdk features
   4  * ================================================================================
   5  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
   6  * All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *     http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  *
  21  */
  22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database;
  23
  24 import java.io.File;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.nio.charset.StandardCharsets;
  28 import java.nio.file.Files;
  29 import java.text.ParseException;
  30 import java.util.ArrayList;
  31 import java.util.Arrays;
  32 import java.util.List;
  33 import java.util.Set;
  34 import org.json.JSONObject;
  35 import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
  36 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit;
  37 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult;
  38 import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo;
  39 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry;
  40 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList;
  41 import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion;
  42 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry;
  43 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
  44 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateAliasRequest;
  45 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateIndexRequest;
  46 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteAliasRequest;
  47 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteIndexRequest;
  48 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.AcknowledgedResponse;
  49 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.GetInfoResponse;
  50 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListAliasesResponse;
  51 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListIndicesResponse;
  52 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType;
  53 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.DataMigrationProviderService;
  54 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
  55 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData;
  56 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
  57 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer;
  58 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport;
  59 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
  60 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup;
  61 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 @Deprecated
  66 public class ElasticsearchDataMigrationProvider implements DataMigrationProviderService {
  67
  68
  69     private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDataMigrationProvider.class);
  70     private static final String LOG_DELETING_INDEX = "deleting index {}";
  71     private final HtDatabaseClient dbClient;
  72
  73     public ElasticsearchDataMigrationProvider(String url, String username, String password, boolean trustAll,
  74             long timeoutms) throws Exception {
  75         dbClient = HtDatabaseClient.getClient(new HostInfo[] {HostInfo.parse(url)}, username, password, trustAll, true,
  76                 timeoutms);
  77     }
  78
  79     @Override
  80     public DataMigrationReport importData(String filename, boolean dryrun) throws Exception {
  81         return this.importData(filename, dryrun, Release.CURRENT_RELEASE);
  82     }
  83
  84     @Override
  85     public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception {
  86         DataMigrationReport report = new DataMigrationReport();
  87         File file = new File(filename);
  88         if (!file.exists()) {
  89             if (dryrun) {
  90                 report.error("file %s not found", filename);
  91                 return report;
  92             }
  93             throw new FileNotFoundException(filename);
  94         }
  95         DataContainer container = null;
  96         try {
  97             container = DataContainer.load(file);
  98         } catch (Exception e) {
  99             if (dryrun) {
 100                 report.error("problem loading file %s: %s", filename, e.getMessage());
 101                 return report;
 102             }
 103             throw new Exception("problem loading file " + filename, e);
 104         }
 105         ReleaseInformation ri = ReleaseInformation.getInstance(forRelease);
 106         SearchHitConverter converter;
 107         Set<ComponentName> components = ri.getComponents();
 108         //for all db components of dest architecture
 109         for (ComponentName component : components) {
 110             //convert to ComponentData for current release with existing ComponentData of the container
 111             converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component);
 112             if (converter == null) {
 113                 continue;
 114             }
 115             ComponentData data = converter.convert(container);
 116             if (data != null) {
 117                 String indexName = ri.getAlias(component);
 118                 String dataTypeName = ri.getDataType(component);
 119                 if (dryrun) {
 120                     report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName);
 121                 } else {
 122                     LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName);
 123                 }
 124                 for (SearchHit item : data) {
 125                     if (!dryrun) {
 126                         String id = this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(),
 127                                 item.getSourceAsString(), true);
 128                         if (!item.getId().equals(id)) {
 129                             LOG.warn("entry for {} with original id {} was written with another id {}",
 130                                     component.getValue(), item.getId(), id);
 131                         }
 132                     }
 133                 }
 134             } else {
 135                 if (dryrun) {
 136                     report.error("unable to convert data for " + component.getValue() + " from version "
 137                             + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n");
 138                 } else {
 139                     LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(),
 140                             container.getRelease().getValue(), forRelease.getValue());
 141                 }
 142             }
 143         }
 144         LOG.info("import of {} completed", filename);
 145         if (dryrun) {
 146             report.log("import of %s completed", filename);
 147         }
 148         report.setCompleted(true);
 149         return report;
 150     }
 151
 152
 153     /**
 154      * export data if file exists .1 (.n) will be created
 155      *
 156      */
 157     @Override
 158     public DataMigrationReport exportData(String filename) {
 159         DataMigrationReport report = new DataMigrationReport();
 160
 161         DataContainer container = new DataContainer();
 162
 163         filename = this.checkFilenameForWrite(filename);
 164         LOG.info("output will be written to {}", filename);
 165         //autodetect version
 166         Release dbRelease = this.autoDetectRelease();
 167         if (dbRelease == null) {
 168             report.error("unbable to detect db release. is database initialized?");
 169             return report;
 170         }
 171         ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease);
 172         boolean componentsSucceeded = true;
 173         for (ComponentName c : ri.getComponents()) {
 174             ComponentData data = new ComponentData(c);
 175             SearchResult<SearchHit> result = this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false);
 176             data.addAll(result.getHits());
 177             container.addComponent(c, data);
 178         }
 179         try {
 180             Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8);
 181             report.setCompleted(componentsSucceeded);
 182         } catch (IOException e) {
 183             LOG.warn("problem writing data to {}: {}", filename, e);
 184         }
 185         return report;
 186     }
 187
 188     private String checkFilenameForWrite(String filename) {
 189         File f = new File(filename);
 190         if (!f.exists()) {
 191             return filename;
 192         }
 193         return this.checkFilenameForWrite(filename, 0);
 194     }
 195
 196     private String checkFilenameForWrite(String filename, int apdx) {
 197         File f = new File(String.format("$s.$d", filename, apdx));
 198         if (!f.exists()) {
 199             return filename;
 200         }
 201         return this.checkFilenameForWrite(filename, apdx + 1);
 202     }
 203
 204     @Override
 205     public Release getCurrentVersion() {
 206         return Release.CURRENT_RELEASE;
 207     }
 208
 209
 210     @Override
 211     public Release autoDetectRelease() {
 212         DatabaseVersion dbVersion = this.readActualVersion();
 213         AliasesEntryList aliases = this.readAliases();
 214         IndicesEntryList indices = this.readIndices();
 215         if (indices == null) {
 216             return null;
 217         }
 218         List<Release> foundReleases = new ArrayList<>();
 219         //if there are active aliases reduce indices to the active ones
 220         if (aliases != null && !aliases.isEmpty()) {
 221             indices = indices.subList(aliases.getLinkedIndices());
 222         }
 223         for (Release r : Release.values()) {
 224             if (r.isDbInRange(dbVersion, SdnrDbType.ELASTICSEARCH)) {
 225                 ReleaseInformation ri = ReleaseInformation.getInstance(r);
 226                 if (ri != null && ri.containsIndices(indices)) {
 227                     foundReleases.add(r);
 228                 }
 229             }
 230         }
 231         if (foundReleases.size() == 1) {
 232             return foundReleases.get(0);
 233         }
 234         LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(),
 235                 foundReleases);
 236         return null;
 237     }
 238
 239     private DatabaseVersion readActualVersion() {
 240         try {
 241             GetInfoResponse response = this.dbClient.getInfo();
 242             return response.getVersion();
 243         } catch (Exception e) {
 244             LOG.warn(e.getMessage());
 245         }
 246         return null;
 247     }
 248
 249     private AliasesEntryList readAliases() {
 250         AliasesEntryList entries = null;
 251         try {
 252             ListAliasesResponse response = this.dbClient.getAliases();
 253             entries = response.getEntries();
 254         } catch (ParseException | IOException e) {
 255             LOG.error(e.getMessage());
 256         }
 257         return entries;
 258     }
 259
 260     private IndicesEntryList readIndices() {
 261         IndicesEntryList entries = null;
 262         try {
 263             ListIndicesResponse response = this.dbClient.getIndices();
 264             entries = response.getEntries();
 265         } catch (ParseException | IOException e) {
 266             LOG.error(e.getMessage());
 267         }
 268         return entries;
 269     }
 270
 271
 272     @Override
 273     public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate,
 274             long timeoutms) {
 275         if (timeoutms > 0) {
 276             this.dbClient.waitForYellowStatus(timeoutms);
 277         }
 278         DatabaseVersion dbVersion = this.readActualVersion();
 279         if (dbVersion == null) {
 280             return false;
 281         }
 282         LOG.info("detected database version {}", dbVersion);
 283         if (release == null) {
 284             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.ELASTICSEARCH);
 285             if (release == null) {
 286                 LOG.warn("unable to autodetect release for this database version for release {}",
 287                         ReleaseGroup.CURRENT_RELEASE.name());
 288                 return false;
 289             }
 290             LOG.info("autodetect release {}", release);
 291         }
 292         if (!release.isDbInRange(dbVersion, SdnrDbType.ELASTICSEARCH)) {
 293             LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release);
 294             return false;
 295         }
 296         if (forceRecreate) {
 297             this.clearDatabase(release, dbPrefix, 0);
 298         }
 299         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 300         AliasesEntryList aliases = this.readAliases();
 301         IndicesEntryList indices = this.readIndices();
 302         if (aliases == null || indices == null) {
 303             return false;
 304         }
 305         AcknowledgedResponse response = null;
 306         if (!ri.runPreInitCommands(this.dbClient)) {
 307             return false;
 308         }
 309         for (ComponentName component : ri.getComponents()) {
 310             try {
 311                 if (ri.hasOwnDbIndex(component)) {
 312                     //check if index already exists
 313                     String indexName = ri.getIndex(component, dbPrefix);
 314                     String aliasName = ri.getAlias(component, dbPrefix);
 315                     if (indices.findByIndex(indexName) == null) {
 316                         LOG.info("creating index for {}", component);
 317                         CreateIndexRequest request = new CreateIndexRequest(ri.getIndex(component, dbPrefix));
 318                         request.mappings(new JSONObject(ri.getDatabaseMapping(component)));
 319                         request.settings(new JSONObject(ri.getDatabaseSettings(component, numShards, numReplicas)));
 320                         response = this.dbClient.createIndex(request);
 321                         LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 322                     } else {
 323                         LOG.info("index {} for {} already exists", indexName, component);
 324                     }
 325                     //check if alias already exists
 326                     if (aliases.findByAlias(aliasName) == null) {
 327                         LOG.info("creating alias for {}", component);
 328                         response = this.dbClient.createAlias(new CreateAliasRequest(indexName, aliasName));
 329                         LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 330                     } else {
 331                         LOG.info("alias {} for index {} for {} already exists", aliasName, indexName, component);
 332                     }
 333                 }
 334             } catch (IOException e) {
 335                 LOG.error(e.getMessage());
 336                 return false;
 337             }
 338         }
 339         if (!ri.runPostInitCommands(this.dbClient)) {
 340             return false;
 341         }
 342         return true;
 343     }
 344
 345     @Override
 346     public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) {
 347
 348         if (timeoutms > 0) {
 349             this.dbClient.waitForYellowStatus(timeoutms);
 350         }
 351         //check aliases
 352         AliasesEntryList entries = this.readAliases();
 353         IndicesEntryList entries2 = this.readIndices();
 354         if (entries == null) {
 355             return false;
 356         }
 357         if (release == null) {
 358             DatabaseVersion dbVersion = this.readActualVersion();
 359             if (dbVersion == null) {
 360                 return false;
 361             }
 362             LOG.info("detected database version {}", dbVersion);
 363             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.ELASTICSEARCH);
 364             if (release == null) {
 365                 LOG.warn("unable to autodetect release for this database version for release {}",
 366                         ReleaseGroup.CURRENT_RELEASE.name());
 367                 return false;
 368             }
 369             LOG.info("autodetect release {}", release);
 370         }
 371         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 372         AcknowledgedResponse response;
 373         if (entries.isEmpty()) {
 374             LOG.info("no aliases to clear");
 375         } else {
 376             //check for every component of release if alias exists
 377             for (ComponentName component : ri.getComponents()) {
 378                 String aliasToDelete = ri.getAlias(component, dbPrefix);
 379                 AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
 380                 if (entryToDelete != null) {
 381                     try {
 382                         LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex());
 383                         response = this.dbClient.deleteAlias(
 384                                 new DeleteAliasRequest(entryToDelete.getIndex(), entryToDelete.getAlias()));
 385                         LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 386                     } catch (IOException e) {
 387                         LOG.error(e.getMessage());
 388                         return false;
 389                     }
 390                 } else {
 391                     //try to find malformed typed index with alias name
 392                     IndicesEntry entry2ToDelete = entries2 == null ? null : entries2.findByIndex(aliasToDelete);
 393                     if (entry2ToDelete != null) {
 394                         try {
 395                             LOG.info(LOG_DELETING_INDEX, entry2ToDelete.getName());
 396                             response = this.dbClient.deleteIndex(new DeleteIndexRequest(entry2ToDelete.getName()));
 397                             LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 398                         } catch (IOException e) {
 399                             LOG.error(e.getMessage());
 400                             return false;
 401                         }
 402                     }
 403                 }
 404             }
 405         }
 406         if (entries2 == null) {
 407             return false;
 408         }
 409         if (entries2.isEmpty()) {
 410             LOG.info("no indices to clear");
 411         } else {
 412             //check for every component of release if index exists
 413             for (ComponentName component : ri.getComponents()) {
 414                 String indexToDelete = ri.getIndex(component, dbPrefix);
 415                 IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
 416                 if (entryToDelete != null) {
 417                     try {
 418                         LOG.info(LOG_DELETING_INDEX, entryToDelete.getName());
 419                         response = this.dbClient.deleteIndex(new DeleteIndexRequest(entryToDelete.getName()));
 420                         LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 421                     } catch (IOException e) {
 422                         LOG.error(e.getMessage());
 423                         return false;
 424                     }
 425                 }
 426             }
 427         }
 428
 429         return true;
 430     }
 431
 432     /**
 433      * @param timeoutms
 434      * @return
 435      */
 436     public boolean clearCompleteDatabase(long timeoutms) {
 437         if (timeoutms > 0) {
 438             this.dbClient.waitForYellowStatus(timeoutms);
 439         }
 440         //check aliases and indices
 441         AliasesEntryList aliases = this.readAliases();
 442         IndicesEntryList indices = this.readIndices();
 443         if (aliases == null || indices == null) {
 444             return false;
 445         }
 446         for (AliasesEntry alias : aliases) {
 447             try {
 448                 LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex());
 449                 this.dbClient.deleteAlias(new DeleteAliasRequest(alias.getIndex(), alias.getAlias()));
 450             } catch (IOException e) {
 451                 LOG.error("problem deleting alias {}: {}", alias.getAlias(), e);
 452                 return false;
 453             }
 454         }
 455         for (IndicesEntry index : indices) {
 456             try {
 457                 LOG.info(LOG_DELETING_INDEX, index.getName());
 458                 this.dbClient.deleteIndex(new DeleteIndexRequest(index.getName()));
 459             } catch (IOException e) {
 460                 LOG.error("problem deleting index {}: {}", index.getName(), e);
 461                 return false;
 462             }
 463         }
 464         return true;
 465     }
 466
 467 }