fixed issue with --rundisabled flag
[aai/graphadmin.git] / src / main / java / org / onap / aai / migration / MigrationControllerInternal.java
1 /**
2  * ============LICENSE_START=======================================================
3  * org.onap.aai
4  * ================================================================================
5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *    http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20
21 package org.onap.aai.migration;
22
23 import java.io.File;
24 import java.io.IOException;
25 import java.lang.reflect.InvocationTargetException;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.Paths;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.List;
32 import java.util.Properties;
33 import java.util.Set;
34 import java.util.stream.Collectors;
35
36 import org.apache.commons.configuration.ConfigurationException;
37 import org.apache.commons.configuration.PropertiesConfiguration;
38 import org.apache.commons.lang.exception.ExceptionUtils;
39 import org.apache.tinkerpop.gremlin.structure.Graph;
40 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
41 import org.onap.aai.datasnapshot.DataSnapshot;
42 import org.onap.aai.db.props.AAIProperties;
43 import org.onap.aai.dbmap.AAIGraph;
44 import org.onap.aai.dbmap.DBConnectionType;
45 import org.onap.aai.edges.EdgeIngestor;
46 import org.onap.aai.exceptions.AAIException;
47 import org.onap.aai.introspection.Loader;
48 import org.onap.aai.introspection.LoaderFactory;
49 import org.onap.aai.introspection.ModelType;
50 import org.onap.aai.serialization.db.EdgeSerializer;
51 import org.onap.aai.setup.SchemaVersions;
52 import org.onap.aai.setup.SchemaVersion;
53 import org.onap.aai.logging.LoggingContext;
54 import org.onap.aai.logging.LoggingContext.StatusCode;
55 import org.onap.aai.serialization.engines.QueryStyle;
56 import org.onap.aai.serialization.engines.JanusGraphDBEngine;
57 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
58 import org.onap.aai.util.AAIConstants;
59 import org.onap.aai.util.FormatDate;
60 import org.reflections.Reflections;
61 import org.slf4j.MDC;
62
63 import com.att.eelf.configuration.Configuration;
64 import com.att.eelf.configuration.EELFLogger;
65 import com.att.eelf.configuration.EELFManager;
66 import com.beust.jcommander.JCommander;
67 import com.beust.jcommander.Parameter;
68
69 /**
70  * Runs a series of migrations from a defined directory based on the presence of
71  * the {@link org.onap.aai.migration.Enabled Enabled} annotation
72  *
73  * It will also write a record of the migrations run to the database.
74  */
75 public class MigrationControllerInternal {
76
77         private EELFLogger logger;
78         private final int DANGER_ZONE = 10;
79         public static final String VERTEX_TYPE = "migration-list-1707";
80         private final List<String> resultsSummary = new ArrayList<>();
81         private final List<NotificationHelper> notifications = new ArrayList<>();
82         private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots";
83
84         private LoaderFactory loaderFactory;
85         private EdgeIngestor edgeIngestor;
86         private EdgeSerializer edgeSerializer;
87         private final SchemaVersions schemaVersions;
88
89         public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
90             this.loaderFactory = loaderFactory;
91                 this.edgeIngestor = edgeIngestor;
92                 this.edgeSerializer = edgeSerializer;
93                 this.schemaVersions = schemaVersions;
94                 
95         }
96
97         /**
98          * The main method.
99          *
100          * @param args
101          *            the arguments
102          */
103         public void run(String[] args) {
104                 // Set the logging file properties to be used by EELFManager
105                 System.setProperty("aai.service.name", MigrationController.class.getSimpleName());
106                 Properties props = System.getProperties();
107                 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
108                 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
109
110                 logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
111                 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
112
113                 boolean loadSnapshot = false;
114
115                 CommandLineArgs cArgs = new CommandLineArgs();
116
117                 JCommander jCommander = new JCommander(cArgs, args);
118                 jCommander.setProgramName(MigrationController.class.getSimpleName());
119
120                 // Set flag to load from snapshot based on the presence of snapshot and
121                 // graph storage backend of inmemory
122                 if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) {
123                         try {
124                                 PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
125                                 if (config.getString("storage.backend").equals("inmemory")) {
126                                         loadSnapshot = true;
127 //                                      System.setProperty("load.snapshot.file", "true");
128                                         System.setProperty("snapshot.location", cArgs.dataSnapshot);
129                                         String snapshotLocation =cArgs.dataSnapshot;
130                                         String snapshotDir;
131                                         String snapshotFile;
132                                         int index = snapshotLocation.lastIndexOf("\\");
133                                         if (index == -1){
134                                                 //Use default directory path
135                                                 snapshotDir =  AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "snapshots";
136                                                 snapshotFile = snapshotLocation;
137                                         } else {
138                                                 snapshotDir = snapshotLocation.substring(0, index+1);
139                                                 snapshotFile = snapshotLocation.substring(index+1, snapshotLocation.length()) ;
140                                         }
141                                         String [] dataSnapShotArgs = {"-c","MULTITHREAD_RELOAD","-f", snapshotFile, "-oldFileDir",snapshotDir, "-caller","migration"};
142                                         DataSnapshot dataSnapshot = new DataSnapshot();
143                                         dataSnapshot.executeCommand(dataSnapShotArgs, true, false, null, "MULTITHREAD_RELOAD", snapshotFile);
144                                 }
145                         } catch (ConfigurationException e) {
146                                 LoggingContext.statusCode(StatusCode.ERROR);
147                                 LoggingContext.responseCode(LoggingContext.DATA_ERROR);
148                                 logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
149                                 return;
150                         }
151                 }
152                 else {
153                         System.setProperty("realtime.db.config", cArgs.config);
154                         logAndPrint("\n\n---------- Connecting to Graph ----------");
155                         AAIGraph.getInstance();
156                 }
157
158                 logAndPrint("---------- Connection Established ----------");
159                 SchemaVersion version = schemaVersions.getDefaultVersion();
160                 QueryStyle queryStyle = QueryStyle.TRAVERSAL;
161                 ModelType introspectorFactoryType = ModelType.MOXY;
162                 Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
163                 TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
164
165                 if (cArgs.help) {
166                         jCommander.usage();
167                         engine.rollback();
168                         return;
169                 }
170
171                 Reflections reflections = new Reflections("org.onap.aai.migration");
172                 List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
173                 //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
174                 if (cArgs.list) {
175                         listMigrationWithStatus(cArgs, migratorClasses, engine);
176                         return;
177                 }
178
179                 logAndPrint("---------- Looking for migration scripts to be executed. ----------");
180                 //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
181                 if (!cArgs.excludeClasses.isEmpty()) {
182                         migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
183                         listMigrationWithStatus(cArgs, migratorClasses, engine);
184                 }
185                 List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
186
187                 sortList(migratorClassesToRun);
188
189                 if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
190                         LoggingContext.statusCode(StatusCode.ERROR);
191                         LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
192                         logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
193                         logAndPrint("---------- Done ----------");
194                         LoggingContext.successStatusFields();
195                 }
196
197                 logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
198                 logAndPrint("---------- Executing Migration Scripts ----------");
199
200
201                 if (!cArgs.skipPreMigrationSnapShot) {
202                         takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
203                 }
204
205                 for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
206                         String name = migratorClass.getSimpleName();
207                         Migrator migrator;
208                         if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) {
209
210                                 try {
211                                         engine.startTransaction();
212                                         if (!cArgs.forced && hasAlreadyRun(name, engine)) {
213                                                 logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution");
214                                                 continue;
215                                         }
216                                         migrator = migratorClass
217                                                 .getConstructor(
218                                                         TransactionalGraphEngine.class,
219                                                         LoaderFactory.class,
220                                                         EdgeIngestor.class,
221                                                         EdgeSerializer.class,
222                                                         SchemaVersions.class
223                                                 ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
224                                 } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
225                                         LoggingContext.statusCode(StatusCode.ERROR);
226                                         LoggingContext.responseCode(LoggingContext.DATA_ERROR);
227                                         logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
228                                         LoggingContext.successStatusFields();
229                                         engine.rollback();
230                                         continue;
231                                 }
232                                 logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script.");
233                                 logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs.");
234                                 MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName());
235
236                                 migrator.run();
237
238                                 commitChanges(engine, migrator, cArgs);
239                         } else {
240                                 logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled.");
241                         }
242                 }
243                 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
244                 for (NotificationHelper notificationHelper : notifications) {
245                         try {
246                                 notificationHelper.triggerEvents();
247                         } catch (AAIException e) {
248                                 LoggingContext.statusCode(StatusCode.ERROR);
249                                 LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
250                                 logAndPrint("\tcould not event");
251                                 logger.error("could not event", e);
252                                 LoggingContext.successStatusFields();
253                         }
254                 }
255                 logAndPrint("---------- Done ----------");
256
257                 // Save post migration snapshot if snapshot was loaded
258                 if (!cArgs.skipPostMigrationSnapShot) {
259                         generateSnapshot(engine, "post");
260                 }
261
262                 outputResultsSummary();
263         }
264
265         /**
266          * This method is used to remove excluded classes from migration from the
267          * script command.
268          *
269          * @param excludeClasses
270          *            : Classes to be removed from Migration
271          * @param migratorClasses
272          *            : Classes to execute migration.
273          * @return
274          */
275         private List<Class<? extends Migrator>> filterMigrationClasses(
276                         List<String> excludeClasses,
277                         List<Class<? extends Migrator>> migratorClasses) {
278
279                 List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
280                                 .stream()
281                                 .filter(migratorClass -> !excludeClasses.contains(migratorClass
282                                                 .getSimpleName())).collect(Collectors.toList());
283
284                 return filteredMigratorClasses;
285         }
286
287         private void listMigrationWithStatus(CommandLineArgs cArgs,
288                         List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
289                         sortList(migratorClasses);
290                         engine.startTransaction();
291                         System.out.println("---------- List of all migrations ----------");
292                         migratorClasses.forEach(migratorClass -> {
293                                 boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class);
294                                 String enabled = enabledAnnotation ? "Enabled" : "Disabled";
295                                 StringBuilder sb = new StringBuilder();
296                                 sb.append(migratorClass.getSimpleName());
297                                 sb.append(" in package ");
298                                 sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1));
299                                 sb.append(" is ");
300                                 sb.append(enabled);
301                                 sb.append(" ");
302                                 sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]");
303                                 System.out.println(sb.toString());
304                         });
305                         engine.rollback();
306                         System.out.println("---------- Done ----------");
307                 }
308
309         private String getDbStatus(String name, TransactionalGraphEngine engine) {
310                 if (hasAlreadyRun(name, engine)) {
311                         return "Already executed in this env";
312                 }
313                 return "Will be run on next execution if Enabled";
314         }
315
316         private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) {
317                 return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
318         }
319         private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
320                 Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class).stream()
321                                 .filter(clazz -> clazz.isAnnotationPresent(MigrationPriority.class))
322                                 .collect(Collectors.toSet());
323                 /*
324                  * TODO- Change this to make sure only classes in the specific $release are added in the runList
325                  * Or add a annotation like exclude which folks again need to remember to add ??
326                  */
327
328                 migratorClasses.remove(PropertyMigrator.class);
329                 migratorClasses.remove(EdgeMigrator.class);
330                 return migratorClasses;
331         }
332
333
334         private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
335
336                 /*int sum = 0;
337                 for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
338                         if (migratorClass.isAnnotationPresent(Enabled.class)) {
339                                 sum += migratorClass.getAnnotation(MigrationPriority.class).value();
340                         }
341                 }
342
343                 if (sum >= DANGER_ZONE) {
344
345                         logAndPrint("Entered Danger Zone. Taking snapshot.");
346                 }*/
347
348                 //always take snapshot for now
349
350                 generateSnapshot(engine, "pre");
351
352         }
353
354
355     private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs,
356             List<Class<? extends Migrator>> migratorClasses) {
357         List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
358         if (cArgs.scripts.isEmpty()) {
359             return migratorClasses;
360         }
361         
362         for (Class<? extends Migrator> migratorClass : migratorClasses) {
363             if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) 
364                     || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) {
365                 migratorClassesToRun.add(migratorClass);
366             }
367         }
368         
369         return migratorClassesToRun;
370     }
371
372     private boolean migratorExplicitlySpecified(CommandLineArgs cArgs, String migratorName){
373         return !cArgs.scripts.isEmpty() && cArgs.scripts.contains(migratorName);
374     }
375     private boolean migratorToRunWhenDisabled(CommandLineArgs cArgs, String migratorName){
376         return !cArgs.runDisabled.isEmpty() && cArgs.runDisabled.contains(migratorName);
377     }
378
379         private void sortList(List<Class<? extends Migrator>> migratorClasses) {
380                 Collections.sort(migratorClasses, (m1, m2) -> {
381                         try {
382                                 if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
383                                         return 1;
384                                 } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) {
385                                         return -1;
386                                 } else {
387                                         return m1.getSimpleName().compareTo(m2.getSimpleName());
388                                 }
389                         } catch (Exception e) {
390                                 return 0;
391                         }
392                 });
393         }
394
395
396         private void generateSnapshot(TransactionalGraphEngine engine, String phase) {
397
398                 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
399                 String dateStr= fd.getDateTime();
400                 String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
401                 logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
402                 Graph transaction = null;
403                 try {
404
405                         Path pathToFile = Paths.get(fileName);
406                         if (!pathToFile.toFile().exists()) {
407                                 Files.createDirectories(pathToFile.getParent());
408                         }
409                         String [] dataSnapshotArgs = {"-c","THREADED_SNAPSHOT", "-fileName",fileName, "-caller","migration"};
410                         DataSnapshot dataSnapshot = new DataSnapshot();
411                         dataSnapshot.executeCommand(dataSnapshotArgs, true, false, null, "THREADED_SNAPSHOT", null);
412 //                      transaction = engine.startTransaction();
413 //                      transaction.io(IoCore.graphson()).writeGraph(fileName);
414 //                      engine.rollback();
415                 } catch (IOException e) {
416                         LoggingContext.statusCode(StatusCode.ERROR);
417                         LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
418                         logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
419                         LoggingContext.successStatusFields();
420                         engine.rollback();
421                 }
422
423                 logAndPrint( phase + " migration snapshot saved to " + fileName);
424         }
425         /**
426          * Log and print.
427          *
428          * @param msg
429          *            the msg
430          */
431         protected void logAndPrint(String msg) {
432                 System.out.println(msg);
433                 logger.info(msg);
434         }
435
436         /**
437          * Commit changes.
438          *
439          * @param engine
440          *            the graph transaction
441          * @param migrator
442          *            the migrator
443          * @param cArgs
444          */
445         protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) {
446
447                 String simpleName = migrator.getClass().getSimpleName();
448                 String message;
449                 if (migrator.getStatus().equals(Status.FAILURE)) {
450                         message = "Migration " + simpleName + " Failed. Rolling back.";
451                         LoggingContext.statusCode(StatusCode.ERROR);
452                         LoggingContext.responseCode(LoggingContext.DATA_ERROR);
453                         logAndPrint("\t" + message);
454                         LoggingContext.successStatusFields();
455                         migrator.rollback();
456                 } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
457                         message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
458                         LoggingContext.statusCode(StatusCode.ERROR);
459                         LoggingContext.responseCode(LoggingContext.DATA_ERROR);
460                         logAndPrint("\t" + message);
461                         LoggingContext.successStatusFields();
462                         migrator.rollback();
463                 } else {
464                         MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
465
466                         if (cArgs.commit) {
467                                 if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
468                                         engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
469                                 }
470                                 engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
471                                 .property(simpleName, true).iterate();
472                                 MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
473                                 notifications.add(migrator.getNotificationHelper());
474                                 migrator.commit();
475                                 message = "Migration " + simpleName + " Succeeded. Changes Committed.";
476                                 logAndPrint("\t"+ message +"\t");
477                         } else {
478                                 message = "--commit not specified. Not committing changes for " + simpleName + " to database.";
479                                 logAndPrint("\t" + message);
480                                 migrator.rollback();
481                         }
482
483                 }
484
485                 resultsSummary.add(message);
486
487         }
488
489         private void outputResultsSummary() {
490                 logAndPrint("---------------------------------");
491                 logAndPrint("-------------Summary-------------");
492                 for (String result : resultsSummary) {
493                         logAndPrint(result);
494                 }
495                 logAndPrint("---------------------------------");
496                 logAndPrint("---------------------------------");
497         }
498
499 }
500
501 class CommandLineArgs {
502
503     @Parameter(names = "--help", help = true)
504     public boolean help;
505
506     @Parameter(names = "-c", description = "location of configuration file")
507     public String config;
508
509     @Parameter(names = "-m", description = "names of migration scripts")
510     public List<String> scripts = new ArrayList<>();
511
512     @Parameter(names = "-l", description = "list the status of migrations")
513     public boolean list = false;
514
515     @Parameter(names = "-d", description = "location of data snapshot", hidden = true)
516     public String dataSnapshot;
517
518     @Parameter(names = "-f", description = "force migrations to be rerun")
519     public boolean forced = false;
520
521     @Parameter(names = "--commit", description = "commit changes to graph")
522     public boolean commit = false;
523
524     @Parameter(names = "-e", description = "exclude list of migrator classes")
525     public List<String> excludeClasses = new ArrayList<>();
526
527     @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot")
528     public boolean skipPreMigrationSnapShot = false;
529
530     @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot")
531     public boolean skipPostMigrationSnapShot = false;
532
533     @Parameter(names = "--runDisabled", description = "List of migrators which are to be run even when disabled")
534     public List<String> runDisabled = new ArrayList<>();
535
536 }