gerrit.onap Code Review - ccsdk/features.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * ONAP : ccsdk features
   4  * ================================================================================
   5  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
   6  * All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *     http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  *
  21  */
  22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup;
  23
  24 import java.io.File;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.nio.charset.StandardCharsets;
  28 import java.nio.file.Files;
  29 import java.text.ParseException;
  30 import java.util.ArrayList;
  31 import java.util.Arrays;
  32 import java.util.List;
  33 import java.util.Set;
  34 import org.json.JSONObject;
  35 import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
  36 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit;
  37 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult;
  38 import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo;
  39 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry;
  40 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList;
  41 import org.onap.ccsdk.features.sdnr.wt.common.database.data.EsVersion;
  42 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry;
  43 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
  44 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateAliasRequest;
  45 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateIndexRequest;
  46 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteAliasRequest;
  47 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteIndexRequest;
  48 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.AcknowledgedResponse;
  49 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.GetInfoResponse;
  50 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListAliasesResponse;
  51 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListIndicesResponse;
  52 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData;
  53 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
  54 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer;
  55 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport;
  56 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
  57 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup;
  58 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 public class DataMigrationProviderImpl implements DataMigrationProviderService {
  63
  64
  65     private static final Logger LOG = LoggerFactory.getLogger(DataMigrationProviderImpl.class);
  66
  67     private final HtDatabaseClient dbClient;
  68
  69     public DataMigrationProviderImpl(HostInfo[] hosts, String username, String password, boolean trustAll,
  70             long timeoutms) throws Exception {
  71
  72         dbClient = HtDatabaseClient.getClient(hosts, username, password, trustAll, timeoutms);
  73     }
  74
  75     @Override
  76     public DataMigrationReport importData(String filename, boolean dryrun) throws Exception {
  77         return this.importData(filename, dryrun, Release.CURRENT_RELEASE);
  78     }
  79
  80     public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception {
  81         DataMigrationReport report = new DataMigrationReport();
  82         File file = new File(filename);
  83         if (!file.exists()) {
  84             if (dryrun) {
  85                 report.error("file %s not found", filename);
  86                 return report;
  87             }
  88             throw new FileNotFoundException(filename);
  89         }
  90         DataContainer container = null;
  91         try {
  92             container = DataContainer.load(file);
  93         } catch (Exception e) {
  94             if (dryrun) {
  95                 report.error("problem loading file %s: %s", filename, e.getMessage());
  96                 return report;
  97             }
  98             throw new Exception("problem loading file " + filename, e);
  99         }
 100         ReleaseInformation ri = ReleaseInformation.getInstance(forRelease);
 101         SearchHitConverter converter;
 102         Set<ComponentName> components = ri.getComponents();
 103         //for all db components of dest architecture
 104         for (ComponentName component : components) {
 105             //convert to ComponentData for current release with existing ComponentData of the container
 106             converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component);
 107             if (converter == null) {
 108                 continue;
 109             }
 110             ComponentData data = converter.convert(container);
 111             if (data != null) {
 112                 String indexName = ri.getAlias(component);
 113                 String dataTypeName = ri.getDataType(component);
 114                 if (dryrun) {
 115                     report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName);
 116                 } else {
 117                     LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName);
 118                 }
 119                 for (SearchHit item : data) {
 120                     if (!dryrun) {
 121                         String id = this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(),
 122                                 item.getSourceAsString(), true);
 123                         if (!item.getId().equals(id)) {
 124                             LOG.warn("entry for {} with original id {} was written with another id {}",
 125                                     component.getValue(), item.getId(), id);
 126                         }
 127                     }
 128                 }
 129             } else {
 130                 if (dryrun) {
 131                     report.error("unable to convert data for " + component.getValue() + " from version "
 132                             + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n");
 133                 } else {
 134                     LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(),
 135                             container.getRelease().getValue(), forRelease.getValue());
 136                 }
 137             }
 138         }
 139         LOG.info("import of {} completed", filename);
 140         if (dryrun) {
 141             report.log("import of %s completed", filename);
 142         }
 143         report.setCompleted(true);
 144         return report;
 145     }
 146
 147
 148     /**
 149      * export data if file exists .1 (.n) will be created
 150      *
 151      */
 152     @Override
 153     public DataMigrationReport exportData(String filename) {
 154         DataMigrationReport report = new DataMigrationReport();
 155
 156         DataContainer container = new DataContainer();
 157
 158         filename = this.checkFilenameForWrite(filename);
 159         LOG.info("output will be written to {}", filename);
 160         //autodetect version
 161         Release dbRelease = this.autoDetectRelease();
 162         if (dbRelease == null) {
 163             report.error("unbable to detect db release. is database initialized?");
 164             return report;
 165         }
 166         ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease);
 167         boolean componentsSucceeded = true;
 168         for (ComponentName c : ri.getComponents()) {
 169             ComponentData data = new ComponentData(c);
 170             SearchResult<SearchHit> result = this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false);
 171             data.addAll(result.getHits());
 172             container.addComponent(c, data);
 173         }
 174         try {
 175             Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8);
 176             report.setCompleted(componentsSucceeded);
 177         } catch (IOException e) {
 178             LOG.warn("problem writing data to {}: {}", filename, e);
 179         }
 180         return report;
 181     }
 182
 183     private String checkFilenameForWrite(String filename) {
 184         File f = new File(filename);
 185         if (!f.exists()) {
 186             return filename;
 187         }
 188         return this.checkFilenameForWrite(filename, 0);
 189     }
 190
 191     private String checkFilenameForWrite(String filename, int apdx) {
 192         File f = new File(String.format("$s.$d", filename, apdx));
 193         if (!f.exists()) {
 194             return filename;
 195         }
 196         return this.checkFilenameForWrite(filename, apdx + 1);
 197     }
 198
 199     @Override
 200     public Release getCurrentVersion() {
 201         return Release.CURRENT_RELEASE;
 202     }
 203
 204
 205     public Release autoDetectRelease() {
 206         EsVersion dbVersion = this.readActualVersion();
 207         AliasesEntryList aliases = this.readAliases();
 208         IndicesEntryList indices = this.readIndices();
 209         if (indices == null) {
 210             return null;
 211         }
 212         List<Release> foundReleases = new ArrayList<>();
 213         //if there are active aliases reduce indices to the active ones
 214         if (aliases != null && aliases.size() > 0) {
 215             indices = indices.subList(aliases.getLinkedIndices());
 216         }
 217         for (Release r : Release.values()) {
 218             if (r.isDbInRange(dbVersion)) {
 219                 ReleaseInformation ri = ReleaseInformation.getInstance(r);
 220                 if (ri != null && ri.containsIndices(indices)) {
 221                     foundReleases.add(r);
 222                 }
 223             }
 224         }
 225         if (foundReleases.size() == 1) {
 226             return foundReleases.get(0);
 227         }
 228         LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(),
 229                 foundReleases);
 230         return null;
 231     }
 232
 233     private EsVersion readActualVersion() {
 234         try {
 235             GetInfoResponse response = this.dbClient.getInfo();
 236             return response.getVersion();
 237         } catch (Exception e) {
 238             LOG.warn(e.getMessage());
 239         }
 240         return null;
 241     }
 242
 243     private AliasesEntryList readAliases() {
 244         AliasesEntryList entries = null;
 245         try {
 246             ListAliasesResponse response = this.dbClient.getAliases();
 247             entries = response.getEntries();
 248         } catch (ParseException | IOException e) {
 249             LOG.error(e.getMessage());
 250         }
 251         return entries;
 252     }
 253
 254     private IndicesEntryList readIndices() {
 255         IndicesEntryList entries = null;
 256         try {
 257             ListIndicesResponse response = this.dbClient.getIndices();
 258             entries = response.getEntries();
 259         } catch (ParseException | IOException e) {
 260             LOG.error(e.getMessage());
 261         }
 262         return entries;
 263     }
 264
 265
 266     @Override
 267     public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate,
 268             long timeoutms) {
 269         if (timeoutms > 0) {
 270             this.dbClient.waitForYellowStatus(timeoutms);
 271         }
 272         EsVersion dbVersion = this.readActualVersion();
 273         if (dbVersion == null) {
 274             return false;
 275         }
 276         LOG.info("detected database version {}", dbVersion);
 277         if (release == null) {
 278             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion);
 279             if (release == null) {
 280                 LOG.warn("unable to autodetect release for this database version for release {}",
 281                         ReleaseGroup.CURRENT_RELEASE.name());
 282                 return false;
 283             }
 284             LOG.info("autodetect release {}", release);
 285         }
 286         if (!release.isDbInRange(dbVersion)) {
 287             LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release);
 288             return false;
 289         }
 290         if (forceRecreate) {
 291             this.clearDatabase(release, dbPrefix, 0);
 292         }
 293         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 294         AliasesEntryList aliases = this.readAliases();
 295         IndicesEntryList indices = this.readIndices();
 296         if (aliases == null || indices == null) {
 297             return false;
 298         }
 299         AcknowledgedResponse response = null;
 300         if (!ri.runPreInitCommands(this.dbClient)) {
 301             return false;
 302         }
 303         for (ComponentName component : ri.getComponents()) {
 304             try {
 305                 if (ri.hasOwnDbIndex(component)) {
 306                     //check if index already exists
 307                     String indexName = ri.getIndex(component, dbPrefix);
 308                     String aliasName = ri.getAlias(component, dbPrefix);
 309                     if (indices.findByIndex(indexName) == null) {
 310                         LOG.info("creating index for {}", component);
 311                         CreateIndexRequest request = new CreateIndexRequest(ri.getIndex(component, dbPrefix));
 312                         request.mappings(new JSONObject(ri.getDatabaseMapping(component)));
 313                         request.settings(new JSONObject(ri.getDatabaseSettings(component, numShards, numReplicas)));
 314                         response = this.dbClient.createIndex(request);
 315                         LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 316                     } else {
 317                         LOG.info("index {} for {} already exists", indexName, component);
 318                     }
 319                     //check if alias already exists
 320                     if (aliases.findByAlias(aliasName) == null) {
 321                         LOG.info("creating alias for {}", component);
 322                         response = this.dbClient.createAlias(new CreateAliasRequest(indexName, aliasName));
 323                         LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 324                     } else {
 325                         LOG.info("alias {} for index {} for {} already exists", aliasName, indexName, component);
 326                     }
 327                 }
 328             } catch (IOException e) {
 329                 LOG.error(e.getMessage());
 330                 return false;
 331             }
 332         }
 333         if (!ri.runPostInitCommands(this.dbClient)) {
 334             return false;
 335         }
 336         return true;
 337     }
 338
 339     @Override
 340     public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) {
 341
 342         if (timeoutms > 0) {
 343             this.dbClient.waitForYellowStatus(timeoutms);
 344         }
 345         //check aliases
 346         AliasesEntryList entries = this.readAliases();
 347         IndicesEntryList entries2 = this.readIndices();
 348         if (entries == null) {
 349             return false;
 350         }
 351         if (release == null) {
 352             EsVersion dbVersion = this.readActualVersion();
 353             if (dbVersion == null) {
 354                 return false;
 355             }
 356             LOG.info("detected database version {}", dbVersion);
 357             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion);
 358             if (release == null) {
 359                 LOG.warn("unable to autodetect release for this database version for release {}",
 360                         ReleaseGroup.CURRENT_RELEASE.name());
 361                 return false;
 362             }
 363             LOG.info("autodetect release {}", release);
 364         }
 365         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 366         AcknowledgedResponse response;
 367         if (entries.size() <= 0) {
 368             LOG.info("no aliases to clear");
 369         } else {
 370             //check for every component of release if alias exists
 371             for (ComponentName component : ri.getComponents()) {
 372                 String aliasToDelete = ri.getAlias(component, dbPrefix);
 373                 AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
 374                 if (entryToDelete != null) {
 375                     try {
 376                         LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex());
 377                         response = this.dbClient.deleteAlias(
 378                                 new DeleteAliasRequest(entryToDelete.getIndex(), entryToDelete.getAlias()));
 379                         LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 380                     } catch (IOException e) {
 381                         LOG.error(e.getMessage());
 382                         return false;
 383                     }
 384                 }
 385                 else {
 386                     //try to find malformed typed index with alias name
 387                     IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete);
 388                     if (entry2ToDelete != null) {
 389                         try {
 390                             LOG.info("deleting index {}", entry2ToDelete.getName());
 391                             response = this.dbClient.deleteIndex(new DeleteIndexRequest(entry2ToDelete.getName()));
 392                             LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 393                         } catch (IOException e) {
 394                             LOG.error(e.getMessage());
 395                             return false;
 396                         }
 397                     }
 398                 }
 399             }
 400         }
 401         if (entries2 == null) {
 402             return false;
 403         }
 404         if (entries2.size() <= 0) {
 405             LOG.info("no indices to clear");
 406         } else {
 407             //check for every component of release if index exists
 408             for (ComponentName component : ri.getComponents()) {
 409                 String indexToDelete = ri.getIndex(component, dbPrefix);
 410                 IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
 411                 if (entryToDelete != null) {
 412                     try {
 413                         LOG.info("deleting index {}", entryToDelete.getName());
 414                         response = this.dbClient.deleteIndex(new DeleteIndexRequest(entryToDelete.getName()));
 415                         LOG.info(response.isResponseSucceeded() ? "succeeded" : "failed");
 416                     } catch (IOException e) {
 417                         LOG.error(e.getMessage());
 418                         return false;
 419                     }
 420                 }
 421             }
 422         }
 423
 424         return true;
 425     }
 426
 427     /**
 428      * @param timeoutms
 429      * @return
 430      */
 431     public boolean clearCompleteDatabase(long timeoutms) {
 432         if (timeoutms > 0) {
 433             this.dbClient.waitForYellowStatus(timeoutms);
 434         }
 435         //check aliases and indices
 436         AliasesEntryList aliases = this.readAliases();
 437         IndicesEntryList indices = this.readIndices();
 438         if (aliases == null || indices == null) {
 439             return false;
 440         }
 441         for (AliasesEntry alias : aliases) {
 442             try {
 443                 LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex());
 444                 this.dbClient.deleteAlias(new DeleteAliasRequest(alias.getIndex(), alias.getAlias()));
 445             } catch (IOException e) {
 446                 LOG.error("problem deleting alias {}: {}", alias.getAlias(), e);
 447                 return false;
 448             }
 449         }
 450         for (IndicesEntry index : indices) {
 451             try {
 452                 LOG.info("deleting index {}", index.getName());
 453                 this.dbClient.deleteIndex(new DeleteIndexRequest(index.getName()));
 454             } catch (IOException e) {
 455                 LOG.error("problem deleting index {}: {}", index.getName(), e);
 456                 return false;
 457             }
 458         }
 459         return true;
 460     }
 461
 462 }