src/main/java/org/onap/aai/migration/MigrationControllerInternal.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.aai.migration;
  22
  23 import com.att.eelf.configuration.Configuration;
  24 import com.att.eelf.configuration.EELFLogger;
  25 import com.att.eelf.configuration.EELFManager;
  26 import com.beust.jcommander.JCommander;
  27 import com.beust.jcommander.Parameter;
  28 import org.apache.commons.configuration.ConfigurationException;
  29 import org.apache.commons.configuration.PropertiesConfiguration;
  30 import org.apache.commons.lang.exception.ExceptionUtils;
  31 import org.apache.tinkerpop.gremlin.structure.Graph;
  32 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
  33 import org.onap.aai.db.props.AAIProperties;
  34 import org.onap.aai.dbmap.AAIGraph;
  35 import org.onap.aai.dbmap.DBConnectionType;
  36 import org.onap.aai.edges.EdgeIngestor;
  37 import org.onap.aai.exceptions.AAIException;
  38 import org.onap.aai.introspection.Loader;
  39 import org.onap.aai.introspection.LoaderFactory;
  40 import org.onap.aai.introspection.ModelType;
  41 import org.onap.aai.logging.LoggingContext;
  42 import org.onap.aai.logging.LoggingContext.StatusCode;
  43 import org.onap.aai.serialization.db.EdgeSerializer;
  44 import org.onap.aai.serialization.engines.JanusGraphDBEngine;
  45 import org.onap.aai.serialization.engines.QueryStyle;
  46 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
  47 import org.onap.aai.setup.SchemaVersion;
  48 import org.onap.aai.setup.SchemaVersions;
  49 import org.onap.aai.util.AAIConstants;
  50 import org.onap.aai.util.FormatDate;
  51 import org.reflections.Reflections;
  52 import org.slf4j.MDC;
  53
  54 import java.io.File;
  55 import java.io.IOException;
  56 import java.lang.reflect.InvocationTargetException;
  57 import java.nio.file.Files;
  58 import java.nio.file.Path;
  59 import java.nio.file.Paths;
  60 import java.util.ArrayList;
  61 import java.util.Collections;
  62 import java.util.List;
  63 import java.util.Properties;
  64 import java.util.Set;
  65 import java.util.stream.Collectors;
  66
  67
  68 /**
  69  * Runs a series of migrations from a defined directory based on the presence of
  70  * the {@link org.onap.aai.migration.Enabled Enabled} annotation
  71  *
  72  * It will also write a record of the migrations run to the database.
  73  */
  74 public class MigrationControllerInternal {
  75
  76     private EELFLogger logger;
  77     private final int DANGER_ZONE = 10;
  78     public static final String VERTEX_TYPE = "migration-list-1707";
  79     private final List<String> resultsSummary = new ArrayList<>();
  80     private final List<NotificationHelper> notifications = new ArrayList<>();
  81     private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots";
  82
  83     private LoaderFactory loaderFactory;
  84     private EdgeIngestor edgeIngestor;
  85     private EdgeSerializer edgeSerializer;
  86     private final SchemaVersions schemaVersions;
  87
  88     public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
  89         this.loaderFactory = loaderFactory;
  90         this.edgeIngestor = edgeIngestor;
  91         this.edgeSerializer = edgeSerializer;
  92         this.schemaVersions = schemaVersions;
  93     }
  94
  95     /**
  96      * The main method.
  97      *
  98      * @param args
  99      *            the arguments
 100      */
 101     public void run(String[] args) {
 102         // Set the logging file properties to be used by EELFManager
 103         System.setProperty("aai.service.name", MigrationController.class.getSimpleName());
 104         Properties props = System.getProperties();
 105         props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
 106         props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
 107
 108         logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
 109         MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
 110
 111         boolean loadSnapshot = false;
 112
 113         CommandLineArgs cArgs = new CommandLineArgs();
 114
 115         JCommander jCommander = new JCommander(cArgs, args);
 116         jCommander.setProgramName(MigrationController.class.getSimpleName());
 117
 118         // Set flag to load from snapshot based on the presence of snapshot and
 119         // graph storage backend of inmemory
 120         if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) {
 121             try {
 122                 PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
 123                 if (config.getString("storage.backend").equals("inmemory")) {
 124                     loadSnapshot = true;
 125                     System.setProperty("load.snapshot.file", "true");
 126                     System.setProperty("snapshot.location", cArgs.dataSnapshot);
 127                 }
 128             } catch (ConfigurationException e) {
 129                 LoggingContext.statusCode(StatusCode.ERROR);
 130                 LoggingContext.responseCode(LoggingContext.DATA_ERROR);
 131                 logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
 132                 return;
 133             }
 134         }
 135         System.setProperty("realtime.db.config", cArgs.config);
 136         logAndPrint("\n\n---------- Connecting to Graph ----------");
 137         AAIGraph.getInstance();
 138
 139         logAndPrint("---------- Connection Established ----------");
 140         SchemaVersion version = schemaVersions.getDefaultVersion();
 141         QueryStyle queryStyle = QueryStyle.TRAVERSAL;
 142         ModelType introspectorFactoryType = ModelType.MOXY;
 143         Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
 144         TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
 145
 146         if (cArgs.help) {
 147             jCommander.usage();
 148             engine.rollback();
 149             return;
 150         }
 151
 152         Reflections reflections = new Reflections("org.onap.aai.migration");
 153         List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
 154         //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
 155         if (cArgs.list) {
 156             listMigrationWithStatus(cArgs, migratorClasses, engine);
 157             return;
 158         }
 159
 160         logAndPrint("---------- Looking for migration scripts to be executed. ----------");
 161         //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
 162         if (!cArgs.excludeClasses.isEmpty()) {
 163             migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
 164             listMigrationWithStatus(cArgs, migratorClasses, engine);
 165         }
 166         List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
 167
 168         sortList(migratorClassesToRun);
 169
 170         if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
 171             LoggingContext.statusCode(StatusCode.ERROR);
 172             LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
 173             logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
 174             logAndPrint("---------- Done ----------");
 175             LoggingContext.successStatusFields();
 176         }
 177
 178         logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
 179         logAndPrint("---------- Executing Migration Scripts ----------");
 180
 181
 182         if (!cArgs.skipPreMigrationSnapShot) {
 183             takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
 184         }
 185
 186         for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
 187             String name = migratorClass.getSimpleName();
 188             Migrator migrator;
 189             if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) {//Check either of enabled annotation or runDisabled flag
 190
 191                 try {
 192                     engine.startTransaction();
 193                     if (!cArgs.forced && hasAlreadyRun(name, engine)) {
 194                         logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution");
 195                         continue;
 196                     }
 197                     migrator = migratorClass
 198                         .getConstructor(
 199                             TransactionalGraphEngine.class,
 200                             LoaderFactory.class,
 201                             EdgeIngestor.class,
 202                             EdgeSerializer.class,
 203                             SchemaVersions.class
 204                         ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
 205                 } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
 206                     LoggingContext.statusCode(StatusCode.ERROR);
 207                     LoggingContext.responseCode(LoggingContext.DATA_ERROR);
 208                     logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
 209                     LoggingContext.successStatusFields();
 210                     engine.rollback();
 211                     continue;
 212                 }
 213                 logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script.");
 214                 logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs.");
 215                 MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName());
 216
 217                 migrator.run();
 218
 219                 commitChanges(engine, migrator, cArgs);
 220             } else {
 221                 logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled.");
 222             }
 223         }
 224         MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
 225         for (NotificationHelper notificationHelper : notifications) {
 226             try {
 227                 notificationHelper.triggerEvents();
 228             } catch (AAIException e) {
 229                 LoggingContext.statusCode(StatusCode.ERROR);
 230                 LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
 231                 logAndPrint("\tcould not event");
 232                 logger.error("could not event", e);
 233                 LoggingContext.successStatusFields();
 234             }
 235         }
 236         logAndPrint("---------- Done ----------");
 237
 238         // Save post migration snapshot if snapshot was loaded
 239         if (!cArgs.skipPostMigrationSnapShot) {
 240             generateSnapshot(engine, "post");
 241         }
 242
 243         outputResultsSummary();
 244     }
 245
 246     /**
 247      * This method is used to remove excluded classes from migration from the
 248      * script command.
 249      *
 250      * @param excludeClasses
 251      *            : Classes to be removed from Migration
 252      * @param migratorClasses
 253      *            : Classes to execute migration.
 254      * @return
 255      */
 256     private List<Class<? extends Migrator>> filterMigrationClasses(
 257             List<String> excludeClasses,
 258             List<Class<? extends Migrator>> migratorClasses) {
 259
 260         List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
 261                 .stream()
 262                 .filter(migratorClass -> !excludeClasses.contains(migratorClass
 263                         .getSimpleName())).collect(Collectors.toList());
 264
 265         return filteredMigratorClasses;
 266     }
 267
 268     private void listMigrationWithStatus(CommandLineArgs cArgs,
 269             List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
 270             sortList(migratorClasses);
 271             engine.startTransaction();
 272             System.out.println("---------- List of all migrations ----------");
 273             migratorClasses.forEach(migratorClass -> {
 274                 boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class);
 275                 String enabled = enabledAnnotation ? "Enabled" : "Disabled";
 276                 StringBuilder sb = new StringBuilder();
 277                 sb.append(migratorClass.getSimpleName());
 278                 sb.append(" in package ");
 279                 sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1));
 280                 sb.append(" is ");
 281                 sb.append(enabled);
 282                 sb.append(" ");
 283                 sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]");
 284                 System.out.println(sb.toString());
 285             });
 286             engine.rollback();
 287             System.out.println("---------- Done ----------");
 288         }
 289
 290     private String getDbStatus(String name, TransactionalGraphEngine engine) {
 291         if (hasAlreadyRun(name, engine)) {
 292             return "Already executed in this env";
 293         }
 294         return "Will be run on next execution if Enabled";
 295     }
 296
 297     private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) {
 298         return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
 299     }
 300     private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
 301         Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class);
 302         /*
 303          * TODO- Change this to make sure only classes in the specific $release are added in the runList
 304          * Or add a annotation like exclude which folks again need to remember to add ??
 305          */
 306
 307         migratorClasses.remove(PropertyMigrator.class);
 308         migratorClasses.remove(EdgeMigrator.class);
 309         return migratorClasses;
 310     }
 311
 312
 313     private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
 314
 315         /*int sum = 0;
 316         for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
 317             if (migratorClass.isAnnotationPresent(Enabled.class)) {
 318                 sum += migratorClass.getAnnotation(MigrationPriority.class).value();
 319             }
 320         }
 321
 322         if (sum >= DANGER_ZONE) {
 323
 324             logAndPrint("Entered Danger Zone. Taking snapshot.");
 325         }*/
 326
 327         //always take snapshot for now
 328
 329         generateSnapshot(engine, "pre");
 330
 331     }
 332
 333
 334     private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs,
 335             List<Class<? extends Migrator>> migratorClasses) {
 336         List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
 337         if (cArgs.scripts.isEmpty() && cArgs.runDisabled.isEmpty()) {
 338             return migratorClasses;
 339
 340         }
 341         for (Class<? extends Migrator> migratorClass : migratorClasses) {
 342             if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) {
 343                 migratorClassesToRun.add(migratorClass);
 344             }
 345         }
 346         return migratorClassesToRun;
 347     }
 348     private boolean migratorExplicitlySpecified(CommandLineArgs cArgs, String migratorName){
 349         return !cArgs.scripts.isEmpty() && cArgs.scripts.contains(migratorName);
 350     }
 351     private boolean migratorToRunWhenDisabled(CommandLineArgs cArgs, String migratorName){
 352         return !cArgs.runDisabled.isEmpty() && cArgs.runDisabled.contains(migratorName);
 353     }
 354
 355     private void sortList(List<Class<? extends Migrator>> migratorClasses) {
 356         Collections.sort(migratorClasses, (m1, m2) -> {
 357             try {
 358                 if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
 359                     return 1;
 360                 } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) {
 361                     return -1;
 362                 } else {
 363                     return m1.getSimpleName().compareTo(m2.getSimpleName());
 364                 }
 365             } catch (Exception e) {
 366                 return 0;
 367             }
 368         });
 369     }
 370
 371
 372     private void generateSnapshot(TransactionalGraphEngine engine, String phase) {
 373
 374         FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
 375         String dateStr= fd.getDateTime();
 376         String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
 377         logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
 378         Graph transaction = null;
 379         try {
 380
 381             Path pathToFile = Paths.get(fileName);
 382             if (!pathToFile.toFile().exists()) {
 383                 Files.createDirectories(pathToFile.getParent());
 384             }
 385             transaction = engine.startTransaction();
 386             transaction.io(IoCore.graphson()).writeGraph(fileName);
 387             engine.rollback();
 388         } catch (IOException e) {
 389             LoggingContext.statusCode(StatusCode.ERROR);
 390             LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
 391             logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
 392             LoggingContext.successStatusFields();
 393             engine.rollback();
 394         }
 395
 396         logAndPrint( phase + " migration snapshot saved to " + fileName);
 397     }
 398     /**
 399      * Log and print.
 400      *
 401      * @param msg
 402      *            the msg
 403      */
 404     protected void logAndPrint(String msg) {
 405         System.out.println(msg);
 406         logger.info(msg);
 407     }
 408
 409     /**
 410      * Commit changes.
 411      *
 412      * @param engine
 413      *            the graph transaction
 414      * @param migrator
 415      *            the migrator
 416      * @param cArgs
 417      */
 418     protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) {
 419
 420         String simpleName = migrator.getClass().getSimpleName();
 421         String message;
 422         if (migrator.getStatus().equals(Status.FAILURE)) {
 423             message = "Migration " + simpleName + " Failed. Rolling back.";
 424             LoggingContext.statusCode(StatusCode.ERROR);
 425             LoggingContext.responseCode(LoggingContext.DATA_ERROR);
 426             logAndPrint("\t" + message);
 427             LoggingContext.successStatusFields();
 428             migrator.rollback();
 429         } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
 430             message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
 431             LoggingContext.statusCode(StatusCode.ERROR);
 432             LoggingContext.responseCode(LoggingContext.DATA_ERROR);
 433             logAndPrint("\t" + message);
 434             LoggingContext.successStatusFields();
 435             migrator.rollback();
 436         } else {
 437             MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
 438
 439             if (cArgs.commit) {
 440                 if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
 441                     engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
 442                 }
 443                 engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
 444                 .property(simpleName, true).iterate();
 445                 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
 446                 notifications.add(migrator.getNotificationHelper());
 447                 migrator.commit();
 448                 message = "Migration " + simpleName + " Succeeded. Changes Committed.";
 449                 logAndPrint("\t"+ message +"\t");
 450             } else {
 451                 message = "--commit not specified. Not committing changes for " + simpleName + " to database.";
 452                 logAndPrint("\t" + message);
 453                 migrator.rollback();
 454             }
 455
 456         }
 457
 458         resultsSummary.add(message);
 459
 460     }
 461
 462     private void outputResultsSummary() {
 463         logAndPrint("---------------------------------");
 464         logAndPrint("-------------Summary-------------");
 465         for (String result : resultsSummary) {
 466             logAndPrint(result);
 467         }
 468         logAndPrint("---------------------------------");
 469         logAndPrint("---------------------------------");
 470     }
 471
 472 }
 473
 474 class CommandLineArgs {
 475
 476     @Parameter(names = "--help", help = true)
 477     public boolean help;
 478
 479     @Parameter(names = "-c", description = "location of configuration file")
 480     public String config;
 481
 482     @Parameter(names = "-m", description = "names of migration scripts")
 483     public List<String> scripts = new ArrayList<>();
 484
 485     @Parameter(names = "-l", description = "list the status of migrations")
 486     public boolean list = false;
 487
 488     @Parameter(names = "-d", description = "location of data snapshot", hidden = true)
 489     public String dataSnapshot;
 490
 491     @Parameter(names = "-f", description = "force migrations to be rerun")
 492     public boolean forced = false;
 493
 494     @Parameter(names = "--commit", description = "commit changes to graph")
 495     public boolean commit = false;
 496
 497     @Parameter(names = "-e", description = "exclude list of migrator classes")
 498     public List<String> excludeClasses = new ArrayList<>();
 499
 500     @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot")
 501     public boolean skipPreMigrationSnapShot = false;
 502
 503     @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot")
 504     public boolean skipPostMigrationSnapShot = false;
 505
 506     @Parameter(names = "--runDisabled", description = "List of migrators which are to be run even when disabled")
 507     public List<String> runDisabled = new ArrayList<>();
 508
 509 }