gerrit.onap Code Review - ccsdk/features.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * ONAP : ccsdk features
   4  * ================================================================================
   5  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
   6  * All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *     http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  *
  21  */
  22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database;
  23
  24 import java.io.File;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.nio.charset.StandardCharsets;
  28 import java.nio.file.Files;
  29 import java.sql.SQLException;
  30 import java.text.ParseException;
  31 import java.util.ArrayList;
  32 import java.util.Arrays;
  33 import java.util.List;
  34 import java.util.Set;
  35 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit;
  36 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult;
  37 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry;
  38 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList;
  39 import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion;
  40 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry;
  41 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
  42 import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient;
  43 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType;
  44 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.DataMigrationProviderService;
  45 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
  46 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData;
  47 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
  48 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer;
  49 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport;
  50 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
  51 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup;
  52 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 public class MariaDbDataMigrationProvider implements DataMigrationProviderService {
  57
  58
  59     private static final Logger LOG = LoggerFactory.getLogger(MariaDbDataMigrationProvider.class);
  60     private static final SdnrDbType DBTYPE = SdnrDbType.MARIADB;
  61     private static final String LOG_DELETING_INDEX = "deleting index {}";
  62     private final SqlDBClient dbClient;
  63
  64     public MariaDbDataMigrationProvider(String url, String username, String password, boolean trustAll,
  65             long timeoutms) throws Exception {
  66         dbClient = new SqlDBClient(url, username, password);
  67     }
  68
  69     @Override
  70     public DataMigrationReport importData(String filename, boolean dryrun) throws Exception {
  71         return this.importData(filename, dryrun, Release.CURRENT_RELEASE);
  72     }
  73
  74     @Override
  75     public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception {
  76         DataMigrationReport report = new DataMigrationReport();
  77         File file = new File(filename);
  78         if (!file.exists()) {
  79             if (dryrun) {
  80                 report.error("file %s not found", filename);
  81                 return report;
  82             }
  83             throw new FileNotFoundException(filename);
  84         }
  85         DataContainer container = null;
  86         try {
  87             container = DataContainer.load(file);
  88         } catch (Exception e) {
  89             if (dryrun) {
  90                 report.error("problem loading file %s: %s", filename, e.getMessage());
  91                 return report;
  92             }
  93             throw new Exception("problem loading file " + filename, e);
  94         }
  95         ReleaseInformation ri = ReleaseInformation.getInstance(forRelease);
  96         SearchHitConverter converter;
  97         Set<ComponentName> components = ri.getComponents();
  98         //for all db components of dest architecture
  99         for (ComponentName component : components) {
 100             //convert to ComponentData for current release with existing ComponentData of the container
 101             converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component);
 102             if (converter == null) {
 103                 continue;
 104             }
 105             ComponentData data = converter.convert(container);
 106             if (data != null) {
 107                 String indexName = ri.getAlias(component);
 108                 String dataTypeName = ri.getDataType(component);
 109                 if (dryrun) {
 110                     report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName);
 111                 } else {
 112                     LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName);
 113                 }
 114                 for (SearchHit item : data) {
 115                     if (!dryrun) {
 116                         String id = null;//this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(), item.getSourceAsString(), true);
 117                         if (!item.getId().equals(id)) {
 118                             LOG.warn("entry for {} with original id {} was written with another id {}",
 119                                     component.getValue(), item.getId(), id);
 120                         }
 121                     }
 122                 }
 123             } else {
 124                 if (dryrun) {
 125                     report.error("unable to convert data for " + component.getValue() + " from version "
 126                             + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n");
 127                 } else {
 128                     LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(),
 129                             container.getRelease().getValue(), forRelease.getValue());
 130                 }
 131             }
 132         }
 133         LOG.info("import of {} completed", filename);
 134         if (dryrun) {
 135             report.log("import of %s completed", filename);
 136         }
 137         report.setCompleted(true);
 138         return report;
 139     }
 140
 141
 142     /**
 143      * export data if file exists .1 (.n) will be created
 144      *
 145      */
 146     @Override
 147     public DataMigrationReport exportData(String filename) {
 148         DataMigrationReport report = new DataMigrationReport();
 149
 150         DataContainer container = new DataContainer();
 151
 152         filename = this.checkFilenameForWrite(filename);
 153         LOG.info("output will be written to {}", filename);
 154         //autodetect version
 155         Release dbRelease = this.autoDetectRelease();
 156         if (dbRelease == null) {
 157             report.error("unbable to detect db release. is database initialized?");
 158             return report;
 159         }
 160         ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease);
 161         boolean componentsSucceeded = true;
 162         for (ComponentName c : ri.getComponents()) {
 163             ComponentData data = new ComponentData(c);
 164             // TODO : check why doReadAllJsonData are comment
 165             //SearchResult<SearchHit> result = null;//this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false);
 166             //data.addAll(result.getHits());
 167             container.addComponent(c, data);
 168         }
 169         try {
 170             Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8);
 171             report.setCompleted(componentsSucceeded);
 172         } catch (IOException e) {
 173             LOG.warn("problem writing data to {}: {}", filename, e);
 174         }
 175         return report;
 176     }
 177
 178     private String checkFilenameForWrite(String filename) {
 179         File f = new File(filename);
 180         if (!f.exists()) {
 181             return filename;
 182         }
 183         return this.checkFilenameForWrite(filename, 0);
 184     }
 185
 186     private String checkFilenameForWrite(String filename, int apdx) {
 187         File f = new File(String.format("$s.$d", filename, apdx));
 188         if (!f.exists()) {
 189             return filename;
 190         }
 191         return this.checkFilenameForWrite(filename, apdx + 1);
 192     }
 193
 194     @Override
 195     public Release getCurrentVersion() {
 196         return Release.CURRENT_RELEASE;
 197     }
 198
 199
 200     @Override
 201     public Release autoDetectRelease() {
 202         DatabaseVersion dbVersion;
 203         try {
 204             dbVersion = this.dbClient.readActualVersion();
 205         } catch (SQLException | ParseException e) {
 206             LOG.error("unable to detect db version", e);
 207             return null;
 208         }
 209         AliasesEntryList aliases = this.dbClient.readViews();
 210         IndicesEntryList indices = this.dbClient.readTables();
 211         if (indices == null) {
 212             return null;
 213         }
 214         List<Release> foundReleases = new ArrayList<>();
 215         //if there are active aliases reduce indices to the active ones
 216         if (aliases != null && !aliases.isEmpty()) {
 217             indices = indices.subList(aliases.getLinkedIndices());
 218         }
 219         for (Release r : Release.values()) {
 220             if (r.isDbInRange(dbVersion, SdnrDbType.MARIADB)) {
 221                 ReleaseInformation ri = ReleaseInformation.getInstance(r);
 222                 if (ri != null && ri.containsIndices(indices)) {
 223                     foundReleases.add(r);
 224                 }
 225             }
 226         }
 227         if (foundReleases.size() == 1) {
 228             return foundReleases.get(0);
 229         }
 230         LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(),
 231                 foundReleases);
 232         return null;
 233     }
 234
 235     @Override
 236     public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate,
 237             long timeoutms) {
 238         if (timeoutms > 0) {
 239             this.dbClient.waitForYellowStatus(timeoutms);
 240         }
 241         DatabaseVersion dbVersion;
 242         try {
 243             dbVersion = this.dbClient.readActualVersion();
 244         } catch (SQLException | ParseException e1) {
 245             LOG.error("unable to detect db version", e1);
 246             return false;
 247         }
 248         if (dbVersion == null) {
 249             return false;
 250         }
 251         LOG.info("detected database version {}", dbVersion);
 252         if (release == null) {
 253             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.MARIADB);
 254             if (release == null) {
 255                 LOG.warn("unable to autodetect release for this database version for release {}",
 256                         ReleaseGroup.CURRENT_RELEASE.name());
 257                 return false;
 258             }
 259             LOG.info("autodetect release {}", release);
 260         }
 261         if (!release.isDbInRange(dbVersion, SdnrDbType.MARIADB)) {
 262             LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release);
 263             return false;
 264         }
 265         if (forceRecreate) {
 266             this.clearDatabase(release, dbPrefix, 0);
 267         }
 268         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 269         AliasesEntryList views = this.dbClient.readViews();
 270         IndicesEntryList indices = this.dbClient.readTables();
 271         if (views == null || indices == null) {
 272             return false;
 273         }
 274         boolean response = false;
 275         if (!ri.runPreInitCommands(this.dbClient)) {
 276             return false;
 277         }
 278         for (ComponentName component : ri.getComponents()) {
 279             try {
 280                 if (ri.hasOwnDbIndex(component)) {
 281                     //check if index already exists
 282                     String tableName = ri.getIndex(component, dbPrefix);
 283                     String viewName = ri.getAlias(component, dbPrefix);
 284                     if (indices.findByIndex(tableName) == null) {
 285                         LOG.info("creating index for {}", component);
 286                         response = this.dbClient.createTable(ri.getIndex(component, dbPrefix), ri.getDatabaseMapping(component, DBTYPE));
 287                         LOG.info(response ? "succeeded" : "failed");
 288                     } else {
 289                         LOG.info("index {} for {} already exists", tableName, component);
 290                     }
 291                     //check if alias already exists
 292                     if (views.findByAlias(viewName) == null) {
 293                         LOG.info("creating alias for {}", component);
 294                         response = this.dbClient.createView(tableName, viewName);
 295                         LOG.info(response ? "succeeded" : "failed");
 296                     } else {
 297                         LOG.info("view {} for table {} for {} already exists", viewName, tableName, component);
 298                     }
 299                 }
 300             } catch (SQLException e) {
 301                 LOG.error(e.getMessage());
 302                 return false;
 303             }
 304         }
 305         if (!ri.runPostInitCommands(this.dbClient)) {
 306             return false;
 307         }
 308         return true;
 309     }
 310
 311     @Override
 312     public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) {
 313
 314         if (timeoutms > 0) {
 315             this.dbClient.waitForYellowStatus(timeoutms);
 316         }
 317         //check aliases
 318         AliasesEntryList entries = this.dbClient.readViews();
 319         IndicesEntryList entries2 = this.dbClient.readTables();
 320         if (entries == null) {
 321             return false;
 322         }
 323         if (release == null) {
 324             DatabaseVersion dbVersion;
 325             try {
 326                 dbVersion = this.dbClient.readActualVersion();
 327             } catch (SQLException | ParseException e) {
 328                 LOG.error("unable to detect db version", e);
 329                 return false;
 330             }
 331             LOG.info("detected database version {}", dbVersion);
 332             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.MARIADB);
 333             if (release == null) {
 334                 LOG.warn("unable to autodetect release for this database version for release {}",
 335                         ReleaseGroup.CURRENT_RELEASE.name());
 336                 return false;
 337             }
 338             LOG.info("autodetect release {}", release);
 339         }
 340         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 341         boolean response;
 342         if (entries.isEmpty()) {
 343             LOG.info("no aliases to clear");
 344         } else {
 345             //check for every component of release if alias exists
 346             for (ComponentName component : ri.getComponents()) {
 347                 String aliasToDelete = ri.getAlias(component, dbPrefix);
 348                 AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
 349                 if (entryToDelete != null) {
 350                     try {
 351                         LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex());
 352                         response = this.dbClient.deleteView(entryToDelete.getAlias());
 353                         LOG.info(response ? "succeeded" : "failed");
 354                     } catch (SQLException e) {
 355                         LOG.error(e.getMessage());
 356                         return false;
 357                     }
 358                 } else {
 359                     //try to find malformed typed index with alias name
 360                     IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete);
 361                     if (entry2ToDelete != null) {
 362                         try {
 363                             LOG.info(LOG_DELETING_INDEX, entry2ToDelete.getName());
 364                             response = this.dbClient.deleteTable(entry2ToDelete.getName());
 365                             LOG.info(response ? "succeeded" : "failed");
 366                         } catch (SQLException e) {
 367                             LOG.error(e.getMessage());
 368                             return false;
 369                         }
 370                     }
 371                 }
 372             }
 373         }
 374         if (entries2 == null) {
 375             return false;
 376         }
 377         if (entries2.isEmpty()) {
 378             LOG.info("no indices to clear");
 379         } else {
 380             //check for every component of release if index exists
 381             for (ComponentName component : ri.getComponents()) {
 382                 String indexToDelete = ri.getIndex(component, dbPrefix);
 383                 IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
 384                 if (entryToDelete != null) {
 385                     try {
 386                         LOG.info(LOG_DELETING_INDEX, entryToDelete.getName());
 387                         response = this.dbClient.deleteTable(entryToDelete.getName());
 388                         LOG.info(response ? "succeeded" : "failed");
 389                     } catch (SQLException e) {
 390                         LOG.error(e.getMessage());
 391                         return false;
 392                     }
 393                 }
 394             }
 395         }
 396
 397         return true;
 398     }
 399
 400     /**
 401      * @param timeoutms
 402      * @return
 403      */
 404     public boolean clearCompleteDatabase(long timeoutms) {
 405         if (timeoutms > 0) {
 406             this.dbClient.waitForYellowStatus(timeoutms);
 407         }
 408         //check aliases and indices
 409         AliasesEntryList aliases = this.dbClient.readViews();
 410         IndicesEntryList indices = this.dbClient.readTables();
 411         if (aliases == null || indices == null) {
 412             return false;
 413         }
 414         for (AliasesEntry alias : aliases) {
 415             try {
 416                 LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex());
 417                 this.dbClient.deleteView(alias.getAlias());
 418             } catch (SQLException e) {
 419                 LOG.error("problem deleting alias {}: {}", alias.getAlias(), e);
 420                 return false;
 421             }
 422         }
 423         for (IndicesEntry index : indices) {
 424             try {
 425                 LOG.info(LOG_DELETING_INDEX, index.getName());
 426                 this.dbClient.deleteTable(index.getName());
 427             } catch (SQLException e) {
 428                 LOG.error("problem deleting index {}: {}", index.getName(), e);
 429                 return false;
 430             }
 431         }
 432         return true;
 433     }
 434
 435 }