gerrit.onap Code Review - ccsdk/features.git/blob

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * ONAP : ccsdk features
   4  * ================================================================================
   5  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
   6  * All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *     http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  *
  21  */
  22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.database;
  23
  24 import java.io.File;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.nio.charset.StandardCharsets;
  28 import java.nio.file.Files;
  29 import java.sql.SQLException;
  30 import java.text.ParseException;
  31 import java.util.ArrayList;
  32 import java.util.Arrays;
  33 import java.util.List;
  34 import java.util.Set;
  35 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit;
  36 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult;
  37 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry;
  38 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList;
  39 import org.onap.ccsdk.features.sdnr.wt.common.database.data.DatabaseVersion;
  40 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry;
  41 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
  42 import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.sqldb.SqlDBClient;
  43 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.SdnrDbType;
  44 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.DataMigrationProviderService;
  45 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.ReleaseInformation;
  46 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData;
  47 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
  48 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer;
  49 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport;
  50 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
  51 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ReleaseGroup;
  52 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 public class MariaDbDataMigrationProvider implements DataMigrationProviderService {
  57
  58
  59     private static final Logger LOG = LoggerFactory.getLogger(MariaDbDataMigrationProvider.class);
  60     private static final SdnrDbType DBTYPE = SdnrDbType.MARIADB;
  61     private final SqlDBClient dbClient;
  62
  63     public MariaDbDataMigrationProvider(String url, String username, String password, boolean trustAll,
  64             long timeoutms) throws Exception {
  65         dbClient = new SqlDBClient(url, username, password);
  66     }
  67
  68     @Override
  69     public DataMigrationReport importData(String filename, boolean dryrun) throws Exception {
  70         return this.importData(filename, dryrun, Release.CURRENT_RELEASE);
  71     }
  72
  73     @Override
  74     public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception {
  75         DataMigrationReport report = new DataMigrationReport();
  76         File file = new File(filename);
  77         if (!file.exists()) {
  78             if (dryrun) {
  79                 report.error("file %s not found", filename);
  80                 return report;
  81             }
  82             throw new FileNotFoundException(filename);
  83         }
  84         DataContainer container = null;
  85         try {
  86             container = DataContainer.load(file);
  87         } catch (Exception e) {
  88             if (dryrun) {
  89                 report.error("problem loading file %s: %s", filename, e.getMessage());
  90                 return report;
  91             }
  92             throw new Exception("problem loading file " + filename, e);
  93         }
  94         ReleaseInformation ri = ReleaseInformation.getInstance(forRelease);
  95         SearchHitConverter converter;
  96         Set<ComponentName> components = ri.getComponents();
  97         //for all db components of dest architecture
  98         for (ComponentName component : components) {
  99             //convert to ComponentData for current release with existing ComponentData of the container
 100             converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component);
 101             if (converter == null) {
 102                 continue;
 103             }
 104             ComponentData data = converter.convert(container);
 105             if (data != null) {
 106                 String indexName = ri.getAlias(component);
 107                 String dataTypeName = ri.getDataType(component);
 108                 if (dryrun) {
 109                     report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName);
 110                 } else {
 111                     LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName);
 112                 }
 113                 for (SearchHit item : data) {
 114                     if (!dryrun) {
 115                         String id = null;//this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(), item.getSourceAsString(), true);
 116                         if (!item.getId().equals(id)) {
 117                             LOG.warn("entry for {} with original id {} was written with another id {}",
 118                                     component.getValue(), item.getId(), id);
 119                         }
 120                     }
 121                 }
 122             } else {
 123                 if (dryrun) {
 124                     report.error("unable to convert data for " + component.getValue() + " from version "
 125                             + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n");
 126                 } else {
 127                     LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(),
 128                             container.getRelease().getValue(), forRelease.getValue());
 129                 }
 130             }
 131         }
 132         LOG.info("import of {} completed", filename);
 133         if (dryrun) {
 134             report.log("import of %s completed", filename);
 135         }
 136         report.setCompleted(true);
 137         return report;
 138     }
 139
 140
 141     /**
 142      * export data if file exists .1 (.n) will be created
 143      *
 144      */
 145     @Override
 146     public DataMigrationReport exportData(String filename) {
 147         DataMigrationReport report = new DataMigrationReport();
 148
 149         DataContainer container = new DataContainer();
 150
 151         filename = this.checkFilenameForWrite(filename);
 152         LOG.info("output will be written to {}", filename);
 153         //autodetect version
 154         Release dbRelease = this.autoDetectRelease();
 155         if (dbRelease == null) {
 156             report.error("unbable to detect db release. is database initialized?");
 157             return report;
 158         }
 159         ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease);
 160         boolean componentsSucceeded = true;
 161         for (ComponentName c : ri.getComponents()) {
 162             ComponentData data = new ComponentData(c);
 163             SearchResult<SearchHit> result = null;//this.dbClient.doReadAllJsonData(ri.getAlias(c), ri.getDataType(c), false);
 164             data.addAll(result.getHits());
 165             container.addComponent(c, data);
 166         }
 167         try {
 168             Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8);
 169             report.setCompleted(componentsSucceeded);
 170         } catch (IOException e) {
 171             LOG.warn("problem writing data to {}: {}", filename, e);
 172         }
 173         return report;
 174     }
 175
 176     private String checkFilenameForWrite(String filename) {
 177         File f = new File(filename);
 178         if (!f.exists()) {
 179             return filename;
 180         }
 181         return this.checkFilenameForWrite(filename, 0);
 182     }
 183
 184     private String checkFilenameForWrite(String filename, int apdx) {
 185         File f = new File(String.format("$s.$d", filename, apdx));
 186         if (!f.exists()) {
 187             return filename;
 188         }
 189         return this.checkFilenameForWrite(filename, apdx + 1);
 190     }
 191
 192     @Override
 193     public Release getCurrentVersion() {
 194         return Release.CURRENT_RELEASE;
 195     }
 196
 197
 198     @Override
 199     public Release autoDetectRelease() {
 200         DatabaseVersion dbVersion;
 201         try {
 202             dbVersion = this.dbClient.readActualVersion();
 203         } catch (SQLException | ParseException e) {
 204             LOG.error("unable to detect db version", e);
 205             return null;
 206         }
 207         AliasesEntryList aliases = this.dbClient.readViews();
 208         IndicesEntryList indices = this.dbClient.readTables();
 209         if (indices == null) {
 210             return null;
 211         }
 212         List<Release> foundReleases = new ArrayList<>();
 213         //if there are active aliases reduce indices to the active ones
 214         if (aliases != null && aliases.size() > 0) {
 215             indices = indices.subList(aliases.getLinkedIndices());
 216         }
 217         for (Release r : Release.values()) {
 218             if (r.isDbInRange(dbVersion, SdnrDbType.MARIADB)) {
 219                 ReleaseInformation ri = ReleaseInformation.getInstance(r);
 220                 if (ri != null && ri.containsIndices(indices)) {
 221                     foundReleases.add(r);
 222                 }
 223             }
 224         }
 225         if (foundReleases.size() == 1) {
 226             return foundReleases.get(0);
 227         }
 228         LOG.error("detect {} releases: {}. unable to detect for which one to do sth.", foundReleases.size(),
 229                 foundReleases);
 230         return null;
 231     }
 232
 233     @Override
 234     public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix, boolean forceRecreate,
 235             long timeoutms) {
 236         if (timeoutms > 0) {
 237             this.dbClient.waitForYellowStatus(timeoutms);
 238         }
 239         DatabaseVersion dbVersion;
 240         try {
 241             dbVersion = this.dbClient.readActualVersion();
 242         } catch (SQLException | ParseException e1) {
 243             LOG.error("unable to detect db version", e1);
 244             return false;
 245         }
 246         if (dbVersion == null) {
 247             return false;
 248         }
 249         LOG.info("detected database version {}", dbVersion);
 250         if (release == null) {
 251             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.MARIADB);
 252             if (release == null) {
 253                 LOG.warn("unable to autodetect release for this database version for release {}",
 254                         ReleaseGroup.CURRENT_RELEASE.name());
 255                 return false;
 256             }
 257             LOG.info("autodetect release {}", release);
 258         }
 259         if (!release.isDbInRange(dbVersion, SdnrDbType.MARIADB)) {
 260             LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release);
 261             return false;
 262         }
 263         if (forceRecreate) {
 264             this.clearDatabase(release, dbPrefix, 0);
 265         }
 266         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 267         AliasesEntryList views = this.dbClient.readViews();
 268         IndicesEntryList indices = this.dbClient.readTables();
 269         if (views == null || indices == null) {
 270             return false;
 271         }
 272         boolean response = false;
 273         if (!ri.runPreInitCommands(this.dbClient)) {
 274             return false;
 275         }
 276         for (ComponentName component : ri.getComponents()) {
 277             try {
 278                 if (ri.hasOwnDbIndex(component)) {
 279                     //check if index already exists
 280                     String tableName = ri.getIndex(component, dbPrefix);
 281                     String viewName = ri.getAlias(component, dbPrefix);
 282                     if (indices.findByIndex(tableName) == null) {
 283                         LOG.info("creating index for {}", component);
 284                         response = this.dbClient.createTable(ri.getIndex(component, dbPrefix), ri.getDatabaseMapping(component, DBTYPE));
 285                         LOG.info(response ? "succeeded" : "failed");
 286                     } else {
 287                         LOG.info("index {} for {} already exists", tableName, component);
 288                     }
 289                     //check if alias already exists
 290                     if (views.findByAlias(viewName) == null) {
 291                         LOG.info("creating alias for {}", component);
 292                         response = this.dbClient.createView(tableName, viewName);
 293                         LOG.info(response ? "succeeded" : "failed");
 294                     } else {
 295                         LOG.info("view {} for table {} for {} already exists", viewName, tableName, component);
 296                     }
 297                 }
 298             } catch (SQLException e) {
 299                 LOG.error(e.getMessage());
 300                 return false;
 301             }
 302         }
 303         if (!ri.runPostInitCommands(this.dbClient)) {
 304             return false;
 305         }
 306         return true;
 307     }
 308
 309     @Override
 310     public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) {
 311
 312         if (timeoutms > 0) {
 313             this.dbClient.waitForYellowStatus(timeoutms);
 314         }
 315         //check aliases
 316         AliasesEntryList entries = this.dbClient.readViews();
 317         IndicesEntryList entries2 = this.dbClient.readTables();
 318         if (entries == null) {
 319             return false;
 320         }
 321         if (release == null) {
 322             DatabaseVersion dbVersion;
 323             try {
 324                 dbVersion = this.dbClient.readActualVersion();
 325             } catch (SQLException | ParseException e) {
 326                 LOG.error("unable to detect db version", e);
 327                 return false;
 328             }
 329             LOG.info("detected database version {}", dbVersion);
 330             release = ReleaseGroup.CURRENT_RELEASE.getLatestCompatibleRelease(dbVersion, SdnrDbType.MARIADB);
 331             if (release == null) {
 332                 LOG.warn("unable to autodetect release for this database version for release {}",
 333                         ReleaseGroup.CURRENT_RELEASE.name());
 334                 return false;
 335             }
 336             LOG.info("autodetect release {}", release);
 337         }
 338         ReleaseInformation ri = ReleaseInformation.getInstance(release);
 339         boolean response;
 340         if (entries.size() <= 0) {
 341             LOG.info("no aliases to clear");
 342         } else {
 343             //check for every component of release if alias exists
 344             for (ComponentName component : ri.getComponents()) {
 345                 String aliasToDelete = ri.getAlias(component, dbPrefix);
 346                 AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
 347                 if (entryToDelete != null) {
 348                     try {
 349                         LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex());
 350                         response = this.dbClient.deleteView(entryToDelete.getAlias());
 351                         LOG.info(response ? "succeeded" : "failed");
 352                     } catch (SQLException e) {
 353                         LOG.error(e.getMessage());
 354                         return false;
 355                     }
 356                 } else {
 357                     //try to find malformed typed index with alias name
 358                     IndicesEntry entry2ToDelete = entries2.findByIndex(aliasToDelete);
 359                     if (entry2ToDelete != null) {
 360                         try {
 361                             LOG.info("deleting index {}", entry2ToDelete.getName());
 362                             response = this.dbClient.deleteTable(entry2ToDelete.getName());
 363                             LOG.info(response ? "succeeded" : "failed");
 364                         } catch (SQLException e) {
 365                             LOG.error(e.getMessage());
 366                             return false;
 367                         }
 368                     }
 369                 }
 370             }
 371         }
 372         if (entries2 == null) {
 373             return false;
 374         }
 375         if (entries2.size() <= 0) {
 376             LOG.info("no indices to clear");
 377         } else {
 378             //check for every component of release if index exists
 379             for (ComponentName component : ri.getComponents()) {
 380                 String indexToDelete = ri.getIndex(component, dbPrefix);
 381                 IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
 382                 if (entryToDelete != null) {
 383                     try {
 384                         LOG.info("deleting index {}", entryToDelete.getName());
 385                         response = this.dbClient.deleteTable(entryToDelete.getName());
 386                         LOG.info(response ? "succeeded" : "failed");
 387                     } catch (SQLException e) {
 388                         LOG.error(e.getMessage());
 389                         return false;
 390                     }
 391                 }
 392             }
 393         }
 394
 395         return true;
 396     }
 397
 398     /**
 399      * @param timeoutms
 400      * @return
 401      */
 402     public boolean clearCompleteDatabase(long timeoutms) {
 403         if (timeoutms > 0) {
 404             this.dbClient.waitForYellowStatus(timeoutms);
 405         }
 406         //check aliases and indices
 407         AliasesEntryList aliases = this.dbClient.readViews();
 408         IndicesEntryList indices = this.dbClient.readTables();
 409         if (aliases == null || indices == null) {
 410             return false;
 411         }
 412         for (AliasesEntry alias : aliases) {
 413             try {
 414                 LOG.info("deleting alias {} for index {}", alias.getAlias(), alias.getIndex());
 415                 this.dbClient.deleteView(alias.getAlias());
 416             } catch (SQLException e) {
 417                 LOG.error("problem deleting alias {}: {}", alias.getAlias(), e);
 418                 return false;
 419             }
 420         }
 421         for (IndicesEntry index : indices) {
 422             try {
 423                 LOG.info("deleting index {}", index.getName());
 424                 this.dbClient.deleteTable(index.getName());
 425             } catch (SQLException e) {
 426                 LOG.error("problem deleting index {}: {}", index.getName(), e);
 427                 return false;
 428             }
 429         }
 430         return true;
 431     }
 432
 433 }