sdnr/wt/data-provider/setup/src/main/java/org/onap/ccsdk/features/sdnr/wt/dataprovider/setup/DataMigrationProviderImpl.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * ONAP : ccsdk features
   4  * ================================================================================
   5  * Copyright (C) 2020 highstreet technologies GmbH Intellectual Property.
   6  * All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *     http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  *
  21  */
  22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.setup;
  23
  24 import java.io.File;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.nio.charset.StandardCharsets;
  28 import java.nio.file.Files;
  29 import java.text.ParseException;
  30 import java.util.ArrayList;
  31 import java.util.Arrays;
  32 import java.util.List;
  33 import java.util.Set;
  34
  35 import org.json.JSONObject;
  36 import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
  37 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchHit;
  38 import org.onap.ccsdk.features.sdnr.wt.common.database.SearchResult;
  39 import org.onap.ccsdk.features.sdnr.wt.common.database.config.HostInfo;
  40 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntry;
  41 import org.onap.ccsdk.features.sdnr.wt.common.database.data.AliasesEntryList;
  42 import org.onap.ccsdk.features.sdnr.wt.common.database.data.EsVersion;
  43 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntry;
  44 import org.onap.ccsdk.features.sdnr.wt.common.database.data.IndicesEntryList;
  45 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateAliasRequest;
  46 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.CreateIndexRequest;
  47 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteAliasRequest;
  48 import org.onap.ccsdk.features.sdnr.wt.common.database.requests.DeleteIndexRequest;
  49 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.AcknowledgedResponse;
  50 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.GetInfoResponse;
  51 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListAliasesResponse;
  52 import org.onap.ccsdk.features.sdnr.wt.common.database.responses.ListIndicesResponse;
  53 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentData;
  54 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.ComponentName;
  55 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataMigrationReport;
  56 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.DataContainer;
  57 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.Release;
  58 import org.onap.ccsdk.features.sdnr.wt.dataprovider.setup.data.SearchHitConverter;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 public class DataMigrationProviderImpl implements DataMigrationProviderService {
  63
  64         private final Logger LOG = LoggerFactory.getLogger(DataMigrationProviderImpl.class);
  65
  66         private final HtDatabaseClient dbClient;
  67
  68         public DataMigrationProviderImpl(HostInfo[] hosts, String username, String password, boolean trustAll) {
  69                 this.dbClient = new HtDatabaseClient(hosts, username, password, trustAll);
  70         }
  71
  72         @Override
  73         public DataMigrationReport importData(String filename, boolean dryrun) throws Exception {
  74                 return this.importData(filename, dryrun, Release.CURRENT_RELEASE);
  75         }
  76
  77         public DataMigrationReport importData(String filename, boolean dryrun, Release forRelease) throws Exception {
  78                 DataMigrationReport report = new DataMigrationReport();
  79                 File file = new File(filename);
  80                 if (!file.exists()) {
  81                         if (dryrun) {
  82                                 report.error("file %s not found", filename);
  83                                 return report;
  84                         }
  85                         throw new FileNotFoundException(filename);
  86                 }
  87                 DataContainer container = null;
  88                 try {
  89                         container = DataContainer.load(file);
  90                 } catch (Exception e) {
  91                         if (dryrun) {
  92                                 report.error("problem loading file %s: %s", filename, e.getMessage());
  93                                 return report;
  94                         }
  95                         throw new Exception("problem loading file " + filename, e);
  96                 }
  97                 ReleaseInformation ri = ReleaseInformation.getInstance(forRelease);
  98                 SearchHitConverter converter;
  99                 Set<ComponentName> components = ri.getComponents();
 100                 //for all db components of dest architecture
 101                 for (ComponentName component : components) {
 102                         //convert to ComponentData for current release with existing ComponentData of the container
 103                         converter = SearchHitConverter.Factory.getInstance(container.getRelease(), forRelease, component);
 104                         if (converter == null) {
 105                                 continue;
 106                         }
 107                         ComponentData data = converter.convert(container);
 108                         if (data != null) {
 109                                 String indexName = ri.getAlias(component);
 110                                 String dataTypeName = ri.getDataType(component);
 111                                 if (dryrun) {
 112                                         report.log("write %d entries into %s/%s", data.size(), indexName, dataTypeName);
 113                                 } else {
 114                                         LOG.debug("write {} entries into {}/{}", data.size(), indexName, dataTypeName);
 115                                 }
 116                                 for (SearchHit item : data) {
 117                                         if (!dryrun) {
 118                                                 String id = this.dbClient.doWriteRaw(indexName, dataTypeName, item.getId(),
 119                                                                 item.getSourceAsString());
 120                                                 if (!item.getId().equals(id)) {
 121                                                         LOG.warn("entry for {} with original id {} was written with another id {}",
 122                                                                         component.getValue(), item.getId(), id);
 123                                                 }
 124                                         }
 125                                 }
 126                         } else {
 127                                 if (dryrun) {
 128                                         report.error("unable to convert data for " + component.getValue() + " from version "
 129                                                         + container.getRelease().getValue() + " to " + forRelease.getValue() + "\n");
 130                                 } else {
 131                                         LOG.warn("unable to convert data for {} from version {} to {}", component.getValue(),
 132                                                         container.getRelease().getValue(), forRelease.getValue());
 133                                 }
 134                         }
 135                 }
 136                 LOG.info("import of {} completed", filename);
 137                 if (dryrun) {
 138                         report.log("import of %s completed", filename);
 139                 }
 140                 report.setCompleted(true);
 141                 return report;
 142         }
 143
 144
 145         /**
 146          * export data
 147          * if file exists .1 (.n) will be created
 148          *
 149          */
 150         @Override
 151         public DataMigrationReport exportData(String filename) {
 152                 DataMigrationReport report = new DataMigrationReport();
 153
 154                 DataContainer container = new DataContainer();
 155
 156                 filename = this.checkFilenameForWrite(filename);
 157                 LOG.info("output will be written to {}", filename);
 158                 //autodetect version
 159                 Release dbRelease = this.autoDetectRelease();
 160                 if(dbRelease==null) {
 161                         report.error("unbable to detect db release. is database initialized?");
 162                         return report;
 163                 }
 164                 ReleaseInformation ri = ReleaseInformation.getInstance(dbRelease);
 165                 boolean componentsSucceeded = true;
 166                 for(ComponentName c: ri.getComponents()) {
 167                         ComponentData data = new ComponentData(c);
 168                         SearchResult<SearchHit> result = this.dbClient.doReadAllJsonData(ri.getAlias(c),ri.getDataType(c),false);
 169                         data.addAll(result.getHits());
 170                         container.addComponent(c, data );
 171                 }
 172                 try {
 173                         Files.write(new File(filename).toPath(), Arrays.asList(container.toJSON()), StandardCharsets.UTF_8);
 174                         report.setCompleted(componentsSucceeded);
 175                 } catch (IOException e) {
 176                         LOG.warn("problem writing data to {}: {}", filename, e);
 177                 }
 178                 return report;
 179         }
 180
 181         private String checkFilenameForWrite(String filename) {
 182                 File f = new File(filename);
 183                 if (!f.exists()) {
 184                         return filename;
 185                 }
 186                 return this.checkFilenameForWrite(filename, 0);
 187         }
 188
 189         private String checkFilenameForWrite(String filename, int apdx) {
 190                 File f = new File(String.format("$s.$d",filename,apdx));
 191                 if (!f.exists()) {
 192                         return filename;
 193                 }
 194                 return this.checkFilenameForWrite(filename, apdx + 1);
 195         }
 196
 197         @Override
 198         public Release getCurrentVersion() {
 199                 return Release.CURRENT_RELEASE;
 200         }
 201
 202
 203         public Release autoDetectRelease() {
 204                 EsVersion dbVersion = this.readActualVersion();
 205                 AliasesEntryList aliases = this.readAliases();
 206                 IndicesEntryList indices = this.readIndices();
 207                 if(indices==null) {
 208                         return null;
 209                 }
 210                 List<Release> foundReleases = new ArrayList<Release>();
 211                 //if there are active aliases reduce indices to the active ones
 212                 if(aliases!=null && aliases.size()>0) {
 213                         indices = indices.subList(aliases.getLinkedIndices());
 214                 }
 215                 for(Release r:Release.values()) {
 216                         if(r.isDbInRange(dbVersion)) {
 217                                 ReleaseInformation ri = ReleaseInformation.getInstance(r);
 218                                 if(ri!=null && ri.containsIndices(indices)) {
 219                                         foundReleases.add(r);
 220                                 }
 221                         }
 222                 }
 223                 if (foundReleases.size() == 1) {
 224                         return foundReleases.get(0);
 225                 }
 226                 LOG.error("detect {} releases: {}. unable to detect for which one to do sth.",foundReleases.size(), foundReleases);
 227                 return null;
 228         }
 229         private EsVersion readActualVersion() {
 230                 try {
 231                         GetInfoResponse response = this.dbClient.getInfo();
 232                         return response.getVersion();
 233                 } catch (Exception e) {
 234                         LOG.warn(e.getMessage());
 235                 }
 236                 return null;
 237         }
 238
 239         private AliasesEntryList readAliases() {
 240                 AliasesEntryList entries = null;
 241                 try {
 242                         ListAliasesResponse response = this.dbClient.getAliases();
 243                         entries = response.getEntries();
 244                 } catch (ParseException | IOException e) {
 245                         LOG.error(e.getMessage());
 246                 }
 247                 return entries;
 248         }
 249
 250         private IndicesEntryList readIndices() {
 251                 IndicesEntryList entries = null;
 252                 try {
 253                         ListIndicesResponse response = this.dbClient.getIndices();
 254                         entries = response.getEntries();
 255                 } catch (ParseException | IOException e) {
 256                         LOG.error(e.getMessage());
 257                 }
 258                 return entries;
 259         }
 260
 261         @Override
 262         public boolean initDatabase(Release release, int numShards, int numReplicas, String dbPrefix,
 263                         boolean forceRecreate,long timeoutms) {
 264                 if(timeoutms>0) {
 265                         this.dbClient.waitForYellowStatus(timeoutms);
 266                 }
 267                 EsVersion dbVersion = this.readActualVersion();
 268                 if (dbVersion == null) {
 269                         return false;
 270                 }
 271                 if (!release.isDbInRange(dbVersion)) {
 272                         LOG.warn("db version {} maybe not compatible with release {}", dbVersion, release);
 273                         return false;
 274                 }
 275                 if (forceRecreate) {
 276                         this.clearDatabase(release, dbPrefix,0);
 277                 }
 278                 ReleaseInformation ri = ReleaseInformation.getInstance(release);
 279                 AliasesEntryList aliases = this.readAliases();
 280                 IndicesEntryList indices = this.readIndices();
 281                 if (aliases == null || indices == null) {
 282                         return false;
 283                 }
 284                 AcknowledgedResponse response = null;
 285                 if(!ri.runPreInitCommands(this.dbClient)) {
 286                         return false;
 287                 }
 288                 for (ComponentName component : ri.getComponents()) {
 289                         try {
 290                                 if (ri.hasOwnDbIndex(component)) {
 291                                         //check if index already exists
 292                                         String indexName = ri.getIndex(component, dbPrefix);
 293                                         String aliasName = ri.getAlias(component, dbPrefix);
 294                                         if (indices.findByIndex(indexName) == null) {
 295                                                 LOG.info("creating index for {}", component);
 296                                                 CreateIndexRequest request = new CreateIndexRequest(ri.getIndex(component, dbPrefix));
 297                                                 request.mappings(new JSONObject(ri.getDatabaseMapping(component)));
 298                                                 request.settings(new JSONObject(ri.getDatabaseSettings(component, numShards, numReplicas)));
 299                                                 response = this.dbClient.createIndex(request);
 300                                                 LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 301                                         } else {
 302                                                 LOG.info("index {} for {} already exists", indexName, component);
 303                                         }
 304                                         //check if alias already exists
 305                                         if (aliases.findByAlias(aliasName) == null) {
 306                                                 LOG.info("creating alias for {}", component);
 307                                                 response = this.dbClient.createAlias(new CreateAliasRequest(indexName, aliasName));
 308                                                 LOG.info(response.isAcknowledged() ? "succeeded" : "failed");
 309                                         } else {
 310                                                 LOG.info("alias {} for index {} for {} already exists", aliasName, indexName, component);
 311                                         }
 312                                 }
 313                         } catch (IOException e) {
 314                                 LOG.error(e.getMessage());
 315                                 return false;
 316                         }
 317                 }
 318                 if(!ri.runPostInitCommands(this.dbClient)) {
 319                         return false;
 320                 }
 321                 return true;
 322         }
 323
 324         @Override
 325         public boolean clearDatabase(Release release, String dbPrefix, long timeoutms) {
 326
 327                 if(timeoutms>0) {
 328                         this.dbClient.waitForYellowStatus(timeoutms);
 329                 }
 330                 //check aliases
 331                 AliasesEntryList entries = this.readAliases();
 332                 if (entries == null) {
 333                         return false;
 334                 }
 335                 ReleaseInformation ri = ReleaseInformation.getInstance(release);
 336                 AcknowledgedResponse response;
 337                 if (entries.size() <= 0) {
 338                         LOG.info("no aliases to clear");
 339                 } else {
 340                         //check for every component of release if alias exists
 341                         for (ComponentName component : ri.getComponents()) {
 342                                 String aliasToDelete = ri.getAlias(component, dbPrefix);
 343                                 AliasesEntry entryToDelete = entries.findByAlias(aliasToDelete);
 344                                 if (entryToDelete != null) {
 345                                         try {
 346                                                 LOG.info("deleting alias {} for index {}", entryToDelete.getAlias(), entryToDelete.getIndex());
 347                                                 response=this.dbClient.deleteAlias(
 348                                                                 new DeleteAliasRequest(entryToDelete.getIndex(), entryToDelete.getAlias()));
 349                                                 LOG.info(response.isResponseSucceeded()?"succeeded":"failed");
 350                                         } catch (IOException e) {
 351                                                 LOG.error(e.getMessage());
 352                                                 return false;
 353                                         }
 354                                 }
 355                         }
 356                 }
 357                 IndicesEntryList entries2 = this.readIndices();
 358                 if (entries2 == null) {
 359                         return false;
 360                 }
 361                 if (entries2.size() <= 0) {
 362                         LOG.info("no indices to clear");
 363                 } else {
 364                         //check for every component of release if index exists
 365                         for (ComponentName component : ri.getComponents()) {
 366                                 String indexToDelete = ri.getIndex(component, dbPrefix);
 367                                 IndicesEntry entryToDelete = entries2.findByIndex(indexToDelete);
 368                                 if (entryToDelete != null) {
 369                                         try {
 370                                                 LOG.info("deleting index {}", entryToDelete.getName());
 371                                                 response=this.dbClient.deleteIndex(new DeleteIndexRequest(entryToDelete.getName()));
 372                                                 LOG.info(response.isResponseSucceeded()?"succeeded":"failed");
 373                                         } catch (IOException e) {
 374                                                 LOG.error(e.getMessage());
 375                                                 return false;
 376                                         }
 377                                 }
 378                         }
 379                 }
 380
 381                 return true;
 382         }
 383
 384 }