2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.onap.aai.migration;
24 import java.io.IOException;
25 import java.lang.reflect.InvocationTargetException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.List;
32 import java.util.Properties;
34 import java.util.stream.Collectors;
36 import org.apache.commons.configuration.ConfigurationException;
37 import org.apache.commons.configuration.PropertiesConfiguration;
38 import org.apache.commons.lang.exception.ExceptionUtils;
39 import org.apache.tinkerpop.gremlin.structure.Graph;
40 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
41 import org.onap.aai.datasnapshot.DataSnapshot;
42 import org.onap.aai.db.props.AAIProperties;
43 import org.onap.aai.dbmap.AAIGraph;
44 import org.onap.aai.dbmap.DBConnectionType;
45 import org.onap.aai.edges.EdgeIngestor;
46 import org.onap.aai.exceptions.AAIException;
47 import org.onap.aai.introspection.Loader;
48 import org.onap.aai.introspection.LoaderFactory;
49 import org.onap.aai.introspection.ModelType;
50 import org.onap.aai.serialization.db.EdgeSerializer;
51 import org.onap.aai.setup.SchemaVersions;
52 import org.onap.aai.setup.SchemaVersion;
53 import org.onap.aai.logging.LoggingContext;
54 import org.onap.aai.logging.LoggingContext.StatusCode;
55 import org.onap.aai.serialization.engines.QueryStyle;
56 import org.onap.aai.serialization.engines.JanusGraphDBEngine;
57 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
58 import org.onap.aai.util.AAIConstants;
59 import org.onap.aai.util.FormatDate;
60 import org.reflections.Reflections;
63 import com.att.eelf.configuration.Configuration;
64 import com.att.eelf.configuration.EELFLogger;
65 import com.att.eelf.configuration.EELFManager;
66 import com.beust.jcommander.JCommander;
67 import com.beust.jcommander.Parameter;
70 * Runs a series of migrations from a defined directory based on the presence of
71 * the {@link org.onap.aai.migration.Enabled Enabled} annotation
73 * It will also write a record of the migrations run to the database.
75 public class MigrationControllerInternal {
77 private EELFLogger logger;
78 private final int DANGER_ZONE = 10;
79 public static final String VERTEX_TYPE = "migration-list-1707";
80 private final List<String> resultsSummary = new ArrayList<>();
81 private final List<NotificationHelper> notifications = new ArrayList<>();
82 private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots";
84 private LoaderFactory loaderFactory;
85 private EdgeIngestor edgeIngestor;
86 private EdgeSerializer edgeSerializer;
87 private final SchemaVersions schemaVersions;
89 public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
90 this.loaderFactory = loaderFactory;
91 this.edgeIngestor = edgeIngestor;
92 this.edgeSerializer = edgeSerializer;
93 this.schemaVersions = schemaVersions;
103 public void run(String[] args) {
104 // Set the logging file properties to be used by EELFManager
105 System.setProperty("aai.service.name", MigrationController.class.getSimpleName());
106 Properties props = System.getProperties();
107 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
108 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
110 logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
111 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
113 boolean loadSnapshot = false;
115 CommandLineArgs cArgs = new CommandLineArgs();
117 JCommander jCommander = new JCommander(cArgs, args);
118 jCommander.setProgramName(MigrationController.class.getSimpleName());
120 // Set flag to load from snapshot based on the presence of snapshot and
121 // graph storage backend of inmemory
122 if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) {
124 PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
125 if (config.getString("storage.backend").equals("inmemory")) {
127 // System.setProperty("load.snapshot.file", "true");
128 System.setProperty("snapshot.location", cArgs.dataSnapshot);
129 String snapshotLocation =cArgs.dataSnapshot;
132 int index = snapshotLocation.lastIndexOf("\\");
134 //Use default directory path
135 snapshotDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "snapshots";
136 snapshotFile = snapshotLocation;
138 snapshotDir = snapshotLocation.substring(0, index+1);
139 snapshotFile = snapshotLocation.substring(index+1, snapshotLocation.length()) ;
141 String [] dataSnapShotArgs = {"-c","MULTITHREAD_RELOAD","-f", snapshotFile, "-oldFileDir",snapshotDir, "-caller","migration"};
142 DataSnapshot dataSnapshot = new DataSnapshot();
143 dataSnapshot.executeCommand(dataSnapShotArgs, true, false, null, "MULTITHREAD_RELOAD", snapshotFile);
145 } catch (ConfigurationException e) {
146 LoggingContext.statusCode(StatusCode.ERROR);
147 LoggingContext.responseCode(LoggingContext.DATA_ERROR);
148 logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
153 System.setProperty("realtime.db.config", cArgs.config);
154 logAndPrint("\n\n---------- Connecting to Graph ----------");
155 AAIGraph.getInstance();
158 logAndPrint("---------- Connection Established ----------");
159 SchemaVersion version = schemaVersions.getDefaultVersion();
160 QueryStyle queryStyle = QueryStyle.TRAVERSAL;
161 ModelType introspectorFactoryType = ModelType.MOXY;
162 Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
163 TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
171 Reflections reflections = new Reflections("org.onap.aai.migration");
172 List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
173 //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
175 listMigrationWithStatus(cArgs, migratorClasses, engine);
179 logAndPrint("---------- Looking for migration scripts to be executed. ----------");
180 //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
181 if (!cArgs.excludeClasses.isEmpty()) {
182 migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
183 listMigrationWithStatus(cArgs, migratorClasses, engine);
185 List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
187 sortList(migratorClassesToRun);
189 if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
190 LoggingContext.statusCode(StatusCode.ERROR);
191 LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
192 logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
193 logAndPrint("---------- Done ----------");
194 LoggingContext.successStatusFields();
197 logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
198 logAndPrint("---------- Executing Migration Scripts ----------");
201 if (!cArgs.skipPreMigrationSnapShot) {
202 takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
205 for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
206 String name = migratorClass.getSimpleName();
208 if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) {
211 engine.startTransaction();
212 if (!cArgs.forced && hasAlreadyRun(name, engine)) {
213 logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution");
216 migrator = migratorClass
218 TransactionalGraphEngine.class,
221 EdgeSerializer.class,
223 ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
224 } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
225 LoggingContext.statusCode(StatusCode.ERROR);
226 LoggingContext.responseCode(LoggingContext.DATA_ERROR);
227 logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
228 LoggingContext.successStatusFields();
232 logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script.");
233 logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs.");
234 MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName());
238 commitChanges(engine, migrator, cArgs);
240 logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled.");
243 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
244 for (NotificationHelper notificationHelper : notifications) {
246 notificationHelper.triggerEvents();
247 } catch (AAIException e) {
248 LoggingContext.statusCode(StatusCode.ERROR);
249 LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
250 logAndPrint("\tcould not event");
251 logger.error("could not event", e);
252 LoggingContext.successStatusFields();
255 logAndPrint("---------- Done ----------");
257 // Save post migration snapshot if snapshot was loaded
258 if (!cArgs.skipPostMigrationSnapShot) {
259 generateSnapshot(engine, "post");
262 outputResultsSummary();
266 * This method is used to remove excluded classes from migration from the
269 * @param excludeClasses
270 * : Classes to be removed from Migration
271 * @param migratorClasses
272 * : Classes to execute migration.
275 private List<Class<? extends Migrator>> filterMigrationClasses(
276 List<String> excludeClasses,
277 List<Class<? extends Migrator>> migratorClasses) {
279 List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
281 .filter(migratorClass -> !excludeClasses.contains(migratorClass
282 .getSimpleName())).collect(Collectors.toList());
284 return filteredMigratorClasses;
287 private void listMigrationWithStatus(CommandLineArgs cArgs,
288 List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
289 sortList(migratorClasses);
290 engine.startTransaction();
291 System.out.println("---------- List of all migrations ----------");
292 migratorClasses.forEach(migratorClass -> {
293 boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class);
294 String enabled = enabledAnnotation ? "Enabled" : "Disabled";
295 StringBuilder sb = new StringBuilder();
296 sb.append(migratorClass.getSimpleName());
297 sb.append(" in package ");
298 sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1));
302 sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]");
303 System.out.println(sb.toString());
306 System.out.println("---------- Done ----------");
309 private String getDbStatus(String name, TransactionalGraphEngine engine) {
310 if (hasAlreadyRun(name, engine)) {
311 return "Already executed in this env";
313 return "Will be run on next execution if Enabled";
316 private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) {
317 return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
319 private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
320 Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class).stream()
321 .filter(clazz -> clazz.isAnnotationPresent(MigrationPriority.class))
322 .collect(Collectors.toSet());
324 * TODO- Change this to make sure only classes in the specific $release are added in the runList
325 * Or add a annotation like exclude which folks again need to remember to add ??
328 migratorClasses.remove(PropertyMigrator.class);
329 migratorClasses.remove(EdgeMigrator.class);
330 return migratorClasses;
334 private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
337 for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
338 if (migratorClass.isAnnotationPresent(Enabled.class)) {
339 sum += migratorClass.getAnnotation(MigrationPriority.class).value();
343 if (sum >= DANGER_ZONE) {
345 logAndPrint("Entered Danger Zone. Taking snapshot.");
348 //always take snapshot for now
350 generateSnapshot(engine, "pre");
355 private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs,
356 List<Class<? extends Migrator>> migratorClasses) {
357 List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
358 if (cArgs.scripts.isEmpty()) {
359 return migratorClasses;
362 for (Class<? extends Migrator> migratorClass : migratorClasses) {
363 if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName())
364 || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) {
365 migratorClassesToRun.add(migratorClass);
369 return migratorClassesToRun;
372 private boolean migratorExplicitlySpecified(CommandLineArgs cArgs, String migratorName){
373 return !cArgs.scripts.isEmpty() && cArgs.scripts.contains(migratorName);
375 private boolean migratorToRunWhenDisabled(CommandLineArgs cArgs, String migratorName){
376 return !cArgs.runDisabled.isEmpty() && cArgs.runDisabled.contains(migratorName);
379 private void sortList(List<Class<? extends Migrator>> migratorClasses) {
380 Collections.sort(migratorClasses, (m1, m2) -> {
382 if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
384 } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) {
387 return m1.getSimpleName().compareTo(m2.getSimpleName());
389 } catch (Exception e) {
396 private void generateSnapshot(TransactionalGraphEngine engine, String phase) {
398 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
399 String dateStr= fd.getDateTime();
400 String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
401 logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
402 Graph transaction = null;
405 Path pathToFile = Paths.get(fileName);
406 if (!pathToFile.toFile().exists()) {
407 Files.createDirectories(pathToFile.getParent());
409 String [] dataSnapshotArgs = {"-c","THREADED_SNAPSHOT", "-fileName",fileName, "-caller","migration"};
410 DataSnapshot dataSnapshot = new DataSnapshot();
411 dataSnapshot.executeCommand(dataSnapshotArgs, true, false, null, "THREADED_SNAPSHOT", null);
412 // transaction = engine.startTransaction();
413 // transaction.io(IoCore.graphson()).writeGraph(fileName);
414 // engine.rollback();
415 } catch (IOException e) {
416 LoggingContext.statusCode(StatusCode.ERROR);
417 LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
418 logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
419 LoggingContext.successStatusFields();
423 logAndPrint( phase + " migration snapshot saved to " + fileName);
431 protected void logAndPrint(String msg) {
432 System.out.println(msg);
440 * the graph transaction
445 protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) {
447 String simpleName = migrator.getClass().getSimpleName();
449 if (migrator.getStatus().equals(Status.FAILURE)) {
450 message = "Migration " + simpleName + " Failed. Rolling back.";
451 LoggingContext.statusCode(StatusCode.ERROR);
452 LoggingContext.responseCode(LoggingContext.DATA_ERROR);
453 logAndPrint("\t" + message);
454 LoggingContext.successStatusFields();
456 } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
457 message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
458 LoggingContext.statusCode(StatusCode.ERROR);
459 LoggingContext.responseCode(LoggingContext.DATA_ERROR);
460 logAndPrint("\t" + message);
461 LoggingContext.successStatusFields();
464 MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
467 if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
468 engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
470 engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
471 .property(simpleName, true).iterate();
472 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
473 notifications.add(migrator.getNotificationHelper());
475 message = "Migration " + simpleName + " Succeeded. Changes Committed.";
476 logAndPrint("\t"+ message +"\t");
478 message = "--commit not specified. Not committing changes for " + simpleName + " to database.";
479 logAndPrint("\t" + message);
485 resultsSummary.add(message);
489 private void outputResultsSummary() {
490 logAndPrint("---------------------------------");
491 logAndPrint("-------------Summary-------------");
492 for (String result : resultsSummary) {
495 logAndPrint("---------------------------------");
496 logAndPrint("---------------------------------");
501 class CommandLineArgs {
503 @Parameter(names = "--help", help = true)
506 @Parameter(names = "-c", description = "location of configuration file")
507 public String config;
509 @Parameter(names = "-m", description = "names of migration scripts")
510 public List<String> scripts = new ArrayList<>();
512 @Parameter(names = "-l", description = "list the status of migrations")
513 public boolean list = false;
515 @Parameter(names = "-d", description = "location of data snapshot", hidden = true)
516 public String dataSnapshot;
518 @Parameter(names = "-f", description = "force migrations to be rerun")
519 public boolean forced = false;
521 @Parameter(names = "--commit", description = "commit changes to graph")
522 public boolean commit = false;
524 @Parameter(names = "-e", description = "exclude list of migrator classes")
525 public List<String> excludeClasses = new ArrayList<>();
527 @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot")
528 public boolean skipPreMigrationSnapShot = false;
530 @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot")
531 public boolean skipPostMigrationSnapShot = false;
533 @Parameter(names = "--runDisabled", description = "List of migrators which are to be run even when disabled")
534 public List<String> runDisabled = new ArrayList<>();