* limitations under the License.
* ============LICENSE_END=========================================================
*/
-package org.onap.aai.migration;
-import java.util.List;
+package org.onap.aai.migration;
import com.google.common.collect.Multimap;
-import org.javatuples.Pair;
-import org.apache.tinkerpop.gremlin.structure.Edge;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.javatuples.Pair;
import org.onap.aai.db.props.AAIProperties;
import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.EdgeRule;
import org.onap.aai.edges.EdgeRuleQuery;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.exceptions.AAIException;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.db.exceptions.EdgeMultiplicityException;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
-import org.onap.aai.edges.EdgeRule;
import org.onap.aai.setup.SchemaVersions;
+
/**
* A migration template for migrating all edge properties between "from" and "to" node from the DbedgeRules.json
*
@MigrationDangerRating(1)
public abstract class EdgeMigrator extends Migrator {
- private boolean success = true;
-
- public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
- super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
- }
-
- public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, List<Pair<String, String>> nodePairList) {
- super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
- }
-
-
- /**
- * Do not override this method as an inheritor of this class
- */
- @Override
- public void run() {
-
- executeModifyOperation();
-
- }
-
- /**
- * This is where inheritors should add their logic
- */
- protected void executeModifyOperation() {
-
- changeEdgeProperties();
-
- }
-
- protected void changeEdgeLabels() {
- //TODO: when json file has edge label as well as edge property changes
- }
-
-
-
- protected void changeEdgeProperties() {
- try {
- List<Pair<String, String>> nodePairList = this.getAffectedNodePairTypes();
- for (Pair<String, String> nodePair : nodePairList) {
-
- String NODE_A = nodePair.getValue0();
- String NODE_B = nodePair.getValue1();
- Multimap<String, EdgeRule> result = edgeIngestor.getRules(new EdgeRuleQuery.Builder(NODE_A, NODE_B).build());
-
- GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V();
- /*
- * Find Out-Edges from Node A to Node B and change them
- * Also Find Out-Edges from Node B to Node A and change them
- */
- g.union(__.has(AAIProperties.NODE_TYPE, NODE_A).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_B)),
- __.has(AAIProperties.NODE_TYPE, NODE_B).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_A)))
- .sideEffect(t -> {
- Edge e = t.get();
- try {
- Vertex out = e.outVertex();
- Vertex in = e.inVertex();
- if (out == null || in == null) {
- logger.error(
- e.id() + " invalid because one vertex was null: out=" + out + " in=" + in);
- } else {
- if (result.containsKey(e.label())) {
- EdgeRule rule = result.get(e.label()).iterator().next();
- e.properties().forEachRemaining(prop -> prop.remove());
- edgeSerializer.addProperties(e, rule);
- } else {
- logger.info("found vertices connected by unkwown label: out=" + out + " label="
- + e.label() + " in=" + in);
- }
- }
- } catch (Exception e1) {
- throw new RuntimeException(e1);
- }
- }).iterate();
- }
-
- } catch (Exception e) {
- logger.error("error encountered", e);
- success = false;
- }
- }
+ protected int processed = 0;
+ protected int skipped = 0;
+ protected Map<String, Integer> edgeMultiplicityExceptionCtr = new HashMap<>();
+ protected List<String> edgeMissingParentProperty = new ArrayList<>();
+
+ private boolean success = true;
+
+ public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor,
+ EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ /**
+ * Do not override this method as an inheritor of this class.
+ */
+ @Override
+ public void run() {
+
+ executeModifyOperation();
+
+ }
+
+ /**
+ * This is where inheritors should add their logic.
+ */
+ protected void executeModifyOperation() {
+
+ changeEdgeProperties();
+
+ }
+
+ protected void changeEdgeLabels() {
+ //TODO: when json file has edge label as well as edge property changes
+ }
+
+
+
+ protected void changeEdgeProperties() {
+ try {
+ List<Pair<String, String>> nodePairList = this.getAffectedNodePairTypes();
+ for (Pair<String, String> nodePair : nodePairList) {
+
+ String NODE_A = nodePair.getValue0();
+ String NODE_B = nodePair.getValue1();
+ Multimap<String, EdgeRule> result = edgeIngestor.getRules(new EdgeRuleQuery.Builder(NODE_A, NODE_B).build());
+
+ GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V();
+ /*
+ * Find Out-Edges from Node A to Node B and change them
+ * Also Find Out-Edges from Node B to Node A and change them
+ */
+ g.union(__.has(AAIProperties.NODE_TYPE, NODE_A).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_B)),
+ __.has(AAIProperties.NODE_TYPE, NODE_B).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_A)))
+ .sideEffect(t -> {
+ Edge e = t.get();
+ try {
+ Vertex out = e.outVertex();
+ Vertex in = e.inVertex();
+ if (out == null || in == null) {
+ logger.error(
+ e.id() + " invalid because one vertex was null: out=" + out + " in=" + in);
+ } else {
+ if (result.containsKey(e.label())) {
+ EdgeRule rule = result.get(e.label()).iterator().next();
+ e.properties().forEachRemaining(prop -> prop.remove());
+ edgeSerializer.addProperties(e, rule);
+ } else {
+ logger.info("found vertices connected by unkwown label: out=" + out + " label="
+ + e.label() + " in=" + in);
+ }
+ }
+ } catch (Exception e1) {
+ throw new RuntimeException(e1);
+ }
+ }).iterate();
+ }
+
+ } catch (Exception e) {
+ logger.error("error encountered", e);
+ success = false;
+ }
+ }
- @Override
- public Status getStatus() {
- if (success) {
- return Status.SUCCESS;
- } else {
- return Status.FAILURE;
- }
- }
-
- /**
- * List of node pairs("from" and "to"), you would like EdgeMigrator to migrate from json files
- * @return
- */
- public abstract List<Pair<String, String>> getAffectedNodePairTypes() ;
-
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ /**
+ * List of node pairs("from" and "to"), you would like EdgeMigrator to migrate from json files.
+ * @return
+ */
+ public abstract List<Pair<String, String>> getAffectedNodePairTypes() ;
+
+ /**
+ * Takes the set of edges, and rebuild them according to current edgeRules schema.
+ * @param edges takes set of edges as input.
+ */
+ protected void rebuildEdges(Set<Edge> edges) {
+ Vertex out = null;
+ Vertex in = null;
+ String oldEdgeString = null;
+ edgeMultiplicityExceptionCtr = new HashMap<>();
+ edgeMissingParentProperty = new ArrayList<>();
+ GraphTraversalSource graphTraversalSource = engine.asAdmin().getTraversalSource();
+ try {
+ for (Edge edge : edges) {
+ oldEdgeString = toStringForPrinting(edge, 1);
+ out = edge.outVertex();
+ in = edge.inVertex();
+ rebuildEdge(edge, graphTraversalSource);
+ }
+ } catch (Exception ex) {
+ logger.error(MIGRATION_ERROR + "exception occurred during migration, failing: out=" + out + " in=" + in
+ + "edge=" + oldEdgeString, ex);
+ success = false;
+ }
+ }
+
+ private void rebuildEdge(Edge edge, GraphTraversalSource graphTraversalSource)
+ throws AAIException {
+ boolean isCousin = false;
+ Vertex out = edge.outVertex();
+ Vertex in = edge.inVertex();
+ if (out == null || in == null) {
+ logger.error(edge.id() + " invalid because one vertex was null: out="
+ + edge.outVertex() + " in=" + edge.inVertex());
+ skipped++;
+ return;
+ }
+
+ if (edge.property("contains-other-v").isPresent()) {
+ isCousin = "NONE".equals(edge.property("contains-other-v").value());
+ } else if (edge.property("isParent").isPresent()) {
+ isCousin = !(Boolean) edge.property("isParent").value();
+ } else {
+ edgeMissingParentProperty.add(this.toStringForPrinting(edge, 1));
+ }
+
+ String inVertexNodeType = in.value(AAIProperties.NODE_TYPE);
+ String outVertexNodeType = out.value(AAIProperties.NODE_TYPE);
+ String label = null;
+
+ try {
+ Collection<EdgeRule> edgeRules = edgeIngestor.getRules(new EdgeRuleQuery.Builder(inVertexNodeType,
+ outVertexNodeType).build()).values();
+ Set<String> edgeLabels = edgeRules.stream().map(EdgeRule::getLabel).collect(Collectors.toSet());
+ if (edgeLabels.size() > 1) {
+ label = selectLabel(edge, edgeLabels);
+ if (label == null) {
+ logger.warn("For Multiple EdgeRules between " + "out=" + outVertexNodeType + " in="
+ + inVertexNodeType + ": did not find label for edge :" + edge.id());
+ }
+ }
+ } catch (Exception e) {
+ logger.error(edge.id() + " did not migrate as no edge rule found for: out=" + outVertexNodeType
+ + " in=" + inVertexNodeType);
+ skipped++;
+ return;
+ }
+
+ try {
+ edge.remove();
+ if (isCousin) {
+ edgeSerializer.addEdgeIfPossible(graphTraversalSource, in, out, label);
+ } else {
+ edgeSerializer.addTreeEdge(graphTraversalSource, out, in);
+ }
+ processed++;
+ } catch (EdgeMultiplicityException edgeMultiplicityException) {
+ logger.warn("Edge Multiplicity Exception: "
+ + "\nInV:\n" + this.toStringForPrinting(in, 1)
+ + "Edge:\n" + this.toStringForPrinting(edge, 1)
+ + "OutV:\n" + this.toStringForPrinting(out, 1)
+ );
+
+ final String mapKey = "OUT:" + outVertexNodeType + " "
+ + (isCousin ? EdgeType.COUSIN.toString() : EdgeType.TREE.toString()) + " "
+ + "IN:" + inVertexNodeType;
+ if (edgeMultiplicityExceptionCtr.containsKey(mapKey)) {
+ edgeMultiplicityExceptionCtr.put(mapKey, edgeMultiplicityExceptionCtr.get(mapKey) + 1);
+ } else {
+ edgeMultiplicityExceptionCtr.put(mapKey, 1);
+ }
+ }
+ }
+
+ /**
+ * For selecting label from multiple EdgeLabels
+ * (where labels got changed between edgeRules' versions),
+ * you should override this method in inheritor class.
+ * @param edge Edge
+ * @param edgeLabels set of edgeLabels
+ * @return
+ */
+ protected String selectLabel(Edge edge, Set<String> edgeLabels) {
+ return null;
+ }
}
* limitations under the License.
* ============LICENSE_END=========================================================
*/
-package org.onap.aai.migration;
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Collectors;
+package org.onap.aai.migration;
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.onap.aai.introspection.Loader;
import org.onap.aai.introspection.LoaderFactory;
import org.onap.aai.introspection.ModelType;
-import org.onap.aai.serialization.db.EdgeSerializer;
-import org.onap.aai.setup.SchemaVersions;
-import org.onap.aai.setup.SchemaVersion;
import org.onap.aai.logging.LoggingContext;
import org.onap.aai.logging.LoggingContext.StatusCode;
-import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.db.EdgeSerializer;
import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
import org.onap.aai.util.AAIConstants;
import org.onap.aai.util.FormatDate;
import org.reflections.Reflections;
import org.slf4j.MDC;
-import com.att.eelf.configuration.Configuration;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+
/**
* Runs a series of migrations from a defined directory based on the presence of
*/
public class MigrationControllerInternal {
- private EELFLogger logger;
- private final int DANGER_ZONE = 10;
- public static final String VERTEX_TYPE = "migration-list-1707";
- private final List<String> resultsSummary = new ArrayList<>();
- private final List<NotificationHelper> notifications = new ArrayList<>();
- private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots";
-
- private LoaderFactory loaderFactory;
- private EdgeIngestor edgeIngestor;
- private EdgeSerializer edgeSerializer;
- private final SchemaVersions schemaVersions;
-
- public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
- this.loaderFactory = loaderFactory;
- this.edgeIngestor = edgeIngestor;
- this.edgeSerializer = edgeSerializer;
- this.schemaVersions = schemaVersions;
- }
-
- /**
- * The main method.
- *
- * @param args
- * the arguments
- */
- public void run(String[] args) {
- // Set the logging file properties to be used by EELFManager
- System.setProperty("aai.service.name", MigrationController.class.getSimpleName());
- Properties props = System.getProperties();
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
- props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
-
- logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
- MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
-
- boolean loadSnapshot = false;
-
- CommandLineArgs cArgs = new CommandLineArgs();
-
- JCommander jCommander = new JCommander(cArgs, args);
- jCommander.setProgramName(MigrationController.class.getSimpleName());
-
- // Set flag to load from snapshot based on the presence of snapshot and
- // graph storage backend of inmemory
- if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) {
- try {
- PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
- if (config.getString("storage.backend").equals("inmemory")) {
- loadSnapshot = true;
- System.setProperty("load.snapshot.file", "true");
- System.setProperty("snapshot.location", cArgs.dataSnapshot);
- }
- } catch (ConfigurationException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
- return;
- }
- }
- System.setProperty("realtime.db.config", cArgs.config);
- logAndPrint("\n\n---------- Connecting to Graph ----------");
- AAIGraph.getInstance();
-
- logAndPrint("---------- Connection Established ----------");
- SchemaVersion version = schemaVersions.getDefaultVersion();
- QueryStyle queryStyle = QueryStyle.TRAVERSAL;
- ModelType introspectorFactoryType = ModelType.MOXY;
- Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
- TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
-
- if (cArgs.help) {
- jCommander.usage();
- engine.rollback();
- return;
- }
-
- Reflections reflections = new Reflections("org.onap.aai.migration");
- List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
- //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
- if (cArgs.list) {
- listMigrationWithStatus(cArgs, migratorClasses, engine);
- return;
- }
-
- logAndPrint("---------- Looking for migration scripts to be executed. ----------");
- //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
- if (!cArgs.excludeClasses.isEmpty()) {
- migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
- listMigrationWithStatus(cArgs, migratorClasses, engine);
- }
- List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
-
- sortList(migratorClassesToRun);
-
- if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
- logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
- logAndPrint("---------- Done ----------");
- LoggingContext.successStatusFields();
- }
-
- logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
- logAndPrint("---------- Executing Migration Scripts ----------");
-
-
- if (!cArgs.skipPreMigrationSnapShot) {
- takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
- }
-
- for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
- String name = migratorClass.getSimpleName();
- Migrator migrator;
- if (migratorClass.isAnnotationPresent(Enabled.class)) {
-
- try {
- engine.startTransaction();
- if (!cArgs.forced && hasAlreadyRun(name, engine)) {
- logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution");
- continue;
- }
- migrator = migratorClass
- .getConstructor(
- TransactionalGraphEngine.class,
- LoaderFactory.class,
- EdgeIngestor.class,
- EdgeSerializer.class,
- SchemaVersions.class
- ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
- } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
- LoggingContext.successStatusFields();
- engine.rollback();
- continue;
- }
- logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script.");
- logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs.");
- MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName());
-
- migrator.run();
-
- commitChanges(engine, migrator, cArgs);
- } else {
- logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled.");
- }
- }
- MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
- for (NotificationHelper notificationHelper : notifications) {
- try {
- notificationHelper.triggerEvents();
- } catch (AAIException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- logAndPrint("\tcould not event");
- logger.error("could not event", e);
- LoggingContext.successStatusFields();
- }
- }
- logAndPrint("---------- Done ----------");
-
- // Save post migration snapshot if snapshot was loaded
- if (!cArgs.skipPostMigrationSnapShot) {
- generateSnapshot(engine, "post");
- }
-
- outputResultsSummary();
- }
-
- /**
- * This method is used to remove excluded classes from migration from the
- * script command.
- *
- * @param excludeClasses
- * : Classes to be removed from Migration
- * @param migratorClasses
- * : Classes to execute migration.
- * @return
- */
- private List<Class<? extends Migrator>> filterMigrationClasses(
- List<String> excludeClasses,
- List<Class<? extends Migrator>> migratorClasses) {
-
- List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
- .stream()
- .filter(migratorClass -> !excludeClasses.contains(migratorClass
- .getSimpleName())).collect(Collectors.toList());
-
- return filteredMigratorClasses;
- }
-
- private void listMigrationWithStatus(CommandLineArgs cArgs,
- List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
- sortList(migratorClasses);
- engine.startTransaction();
- System.out.println("---------- List of all migrations ----------");
- migratorClasses.forEach(migratorClass -> {
- boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class);
- String enabled = enabledAnnotation ? "Enabled" : "Disabled";
- StringBuilder sb = new StringBuilder();
- sb.append(migratorClass.getSimpleName());
- sb.append(" in package ");
- sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1));
- sb.append(" is ");
- sb.append(enabled);
- sb.append(" ");
- sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]");
- System.out.println(sb.toString());
- });
- engine.rollback();
- System.out.println("---------- Done ----------");
- }
-
- private String getDbStatus(String name, TransactionalGraphEngine engine) {
- if (hasAlreadyRun(name, engine)) {
- return "Already executed in this env";
- }
- return "Will be run on next execution if Enabled";
- }
-
- private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) {
- return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
- }
- private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
- Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class);
- /*
- * TODO- Change this to make sure only classes in the specific $release are added in the runList
- * Or add a annotation like exclude which folks again need to remember to add ??
- */
-
- migratorClasses.remove(PropertyMigrator.class);
- migratorClasses.remove(EdgeMigrator.class);
- return migratorClasses;
- }
-
-
- private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
-
- /*int sum = 0;
- for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
- if (migratorClass.isAnnotationPresent(Enabled.class)) {
- sum += migratorClass.getAnnotation(MigrationPriority.class).value();
- }
- }
-
- if (sum >= DANGER_ZONE) {
-
- logAndPrint("Entered Danger Zone. Taking snapshot.");
- }*/
-
- //always take snapshot for now
-
- generateSnapshot(engine, "pre");
-
- }
-
-
- private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs,
- List<Class<? extends Migrator>> migratorClasses) {
- List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
-
- for (Class<? extends Migrator> migratorClass : migratorClasses) {
- if (!cArgs.scripts.isEmpty() && !cArgs.scripts.contains(migratorClass.getSimpleName())) {
- continue;
- } else {
- migratorClassesToRun.add(migratorClass);
- }
- }
- return migratorClassesToRun;
- }
-
-
- private void sortList(List<Class<? extends Migrator>> migratorClasses) {
- Collections.sort(migratorClasses, (m1, m2) -> {
- try {
- if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
- return 1;
- } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) {
- return -1;
- } else {
- return m1.getSimpleName().compareTo(m2.getSimpleName());
- }
- } catch (Exception e) {
- return 0;
- }
- });
- }
-
-
- private void generateSnapshot(TransactionalGraphEngine engine, String phase) {
-
- FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
- String dateStr= fd.getDateTime();
- String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
- logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
- Graph transaction = null;
- try {
-
- Path pathToFile = Paths.get(fileName);
- if (!pathToFile.toFile().exists()) {
- Files.createDirectories(pathToFile.getParent());
- }
- transaction = engine.startTransaction();
- transaction.io(IoCore.graphson()).writeGraph(fileName);
- engine.rollback();
- } catch (IOException e) {
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
- logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
- LoggingContext.successStatusFields();
- engine.rollback();
- }
-
- logAndPrint( phase + " migration snapshot saved to " + fileName);
- }
- /**
- * Log and print.
- *
- * @param msg
- * the msg
- */
- protected void logAndPrint(String msg) {
- System.out.println(msg);
- logger.info(msg);
- }
-
- /**
- * Commit changes.
- *
- * @param engine
- * the graph transaction
- * @param migrator
- * the migrator
- * @param cArgs
- */
- protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) {
-
- String simpleName = migrator.getClass().getSimpleName();
- String message;
- if (migrator.getStatus().equals(Status.FAILURE)) {
- message = "Migration " + simpleName + " Failed. Rolling back.";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- logAndPrint("\t" + message);
- LoggingContext.successStatusFields();
- migrator.rollback();
- } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
- message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
- LoggingContext.statusCode(StatusCode.ERROR);
- LoggingContext.responseCode(LoggingContext.DATA_ERROR);
- logAndPrint("\t" + message);
- LoggingContext.successStatusFields();
- migrator.rollback();
- } else {
- MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
-
- if (cArgs.commit) {
- if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
- engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
- }
- engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
- .property(simpleName, true).iterate();
- MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
- notifications.add(migrator.getNotificationHelper());
- migrator.commit();
- message = "Migration " + simpleName + " Succeeded. Changes Committed.";
- logAndPrint("\t"+ message +"\t");
- } else {
- message = "--commit not specified. Not committing changes for " + simpleName + " to database.";
- logAndPrint("\t" + message);
- migrator.rollback();
- }
-
- }
-
- resultsSummary.add(message);
-
- }
-
- private void outputResultsSummary() {
- logAndPrint("---------------------------------");
- logAndPrint("-------------Summary-------------");
- for (String result : resultsSummary) {
- logAndPrint(result);
- }
- logAndPrint("---------------------------------");
- logAndPrint("---------------------------------");
- }
+ private EELFLogger logger;
+ private final int DANGER_ZONE = 10;
+ public static final String VERTEX_TYPE = "migration-list-1707";
+ private final List<String> resultsSummary = new ArrayList<>();
+ private final List<NotificationHelper> notifications = new ArrayList<>();
+ private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots";
+
+ private LoaderFactory loaderFactory;
+ private EdgeIngestor edgeIngestor;
+ private EdgeSerializer edgeSerializer;
+ private final SchemaVersions schemaVersions;
+
+ public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.edgeIngestor = edgeIngestor;
+ this.edgeSerializer = edgeSerializer;
+ this.schemaVersions = schemaVersions;
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args
+ * the arguments
+ */
+ public void run(String[] args) {
+ // Set the logging file properties to be used by EELFManager
+ System.setProperty("aai.service.name", MigrationController.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
+
+ logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
+ MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
+
+ boolean loadSnapshot = false;
+
+ CommandLineArgs cArgs = new CommandLineArgs();
+
+ JCommander jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(MigrationController.class.getSimpleName());
+
+ // Set flag to load from snapshot based on the presence of snapshot and
+ // graph storage backend of inmemory
+ if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) {
+ try {
+ PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
+ if (config.getString("storage.backend").equals("inmemory")) {
+ loadSnapshot = true;
+ System.setProperty("load.snapshot.file", "true");
+ System.setProperty("snapshot.location", cArgs.dataSnapshot);
+ }
+ } catch (ConfigurationException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
+ return;
+ }
+ }
+ System.setProperty("realtime.db.config", cArgs.config);
+ logAndPrint("\n\n---------- Connecting to Graph ----------");
+ AAIGraph.getInstance();
+
+ logAndPrint("---------- Connection Established ----------");
+ SchemaVersion version = schemaVersions.getDefaultVersion();
+ QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ ModelType introspectorFactoryType = ModelType.MOXY;
+ Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+
+ if (cArgs.help) {
+ jCommander.usage();
+ engine.rollback();
+ return;
+ }
+
+ Reflections reflections = new Reflections("org.onap.aai.migration");
+ List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
+ //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
+ if (cArgs.list) {
+ listMigrationWithStatus(cArgs, migratorClasses, engine);
+ return;
+ }
+
+ logAndPrint("---------- Looking for migration scripts to be executed. ----------");
+ //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
+ if (!cArgs.excludeClasses.isEmpty()) {
+ migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
+ listMigrationWithStatus(cArgs, migratorClasses, engine);
+ }
+ List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
+
+ sortList(migratorClassesToRun);
+
+ if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
+ logAndPrint("---------- Done ----------");
+ LoggingContext.successStatusFields();
+ }
+
+ logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
+ logAndPrint("---------- Executing Migration Scripts ----------");
+
+
+ if (!cArgs.skipPreMigrationSnapShot) {
+ takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
+ }
+
+ for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
+ String name = migratorClass.getSimpleName();
+ Migrator migrator;
+ if (cArgs.runDisabled.contains(name) || migratorClass.isAnnotationPresent(Enabled.class)) {//Check either of enabled annotation or runDisabled flag
+
+ try {
+ engine.startTransaction();
+ if (!cArgs.forced && hasAlreadyRun(name, engine)) {
+ logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution");
+ continue;
+ }
+ migrator = migratorClass
+ .getConstructor(
+ TransactionalGraphEngine.class,
+ LoaderFactory.class,
+ EdgeIngestor.class,
+ EdgeSerializer.class,
+ SchemaVersions.class
+ ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
+ } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
+ LoggingContext.successStatusFields();
+ engine.rollback();
+ continue;
+ }
+ logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script.");
+ logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs.");
+ MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName());
+
+ migrator.run();
+
+ commitChanges(engine, migrator, cArgs);
+ } else {
+ logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled.");
+ }
+ }
+ MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
+ for (NotificationHelper notificationHelper : notifications) {
+ try {
+ notificationHelper.triggerEvents();
+ } catch (AAIException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint("\tcould not event");
+ logger.error("could not event", e);
+ LoggingContext.successStatusFields();
+ }
+ }
+ logAndPrint("---------- Done ----------");
+
+ // Save post migration snapshot if snapshot was loaded
+ if (!cArgs.skipPostMigrationSnapShot) {
+ generateSnapshot(engine, "post");
+ }
+
+ outputResultsSummary();
+ }
+
+ /**
+ * This method is used to remove excluded classes from migration from the
+ * script command.
+ *
+ * @param excludeClasses
+ * : Classes to be removed from Migration
+ * @param migratorClasses
+ * : Classes to execute migration.
+ * @return
+ */
+ private List<Class<? extends Migrator>> filterMigrationClasses(
+ List<String> excludeClasses,
+ List<Class<? extends Migrator>> migratorClasses) {
+
+ List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
+ .stream()
+ .filter(migratorClass -> !excludeClasses.contains(migratorClass
+ .getSimpleName())).collect(Collectors.toList());
+
+ return filteredMigratorClasses;
+ }
+
+ private void listMigrationWithStatus(CommandLineArgs cArgs,
+ List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
+ sortList(migratorClasses);
+ engine.startTransaction();
+ System.out.println("---------- List of all migrations ----------");
+ migratorClasses.forEach(migratorClass -> {
+ boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class);
+ String enabled = enabledAnnotation ? "Enabled" : "Disabled";
+ StringBuilder sb = new StringBuilder();
+ sb.append(migratorClass.getSimpleName());
+ sb.append(" in package ");
+ sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1));
+ sb.append(" is ");
+ sb.append(enabled);
+ sb.append(" ");
+ sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]");
+ System.out.println(sb.toString());
+ });
+ engine.rollback();
+ System.out.println("---------- Done ----------");
+ }
+
+ private String getDbStatus(String name, TransactionalGraphEngine engine) {
+ if (hasAlreadyRun(name, engine)) {
+ return "Already executed in this env";
+ }
+ return "Will be run on next execution if Enabled";
+ }
+
+ private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) {
+ return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
+ }
+ private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
+ Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class);
+ /*
+ * TODO- Change this to make sure only classes in the specific $release are added in the runList
+ * Or add a annotation like exclude which folks again need to remember to add ??
+ */
+
+ migratorClasses.remove(PropertyMigrator.class);
+ migratorClasses.remove(EdgeMigrator.class);
+ return migratorClasses;
+ }
+
+
+ private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
+
+ /*int sum = 0;
+ for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
+ if (migratorClass.isAnnotationPresent(Enabled.class)) {
+ sum += migratorClass.getAnnotation(MigrationPriority.class).value();
+ }
+ }
+
+ if (sum >= DANGER_ZONE) {
+
+ logAndPrint("Entered Danger Zone. Taking snapshot.");
+ }*/
+
+ //always take snapshot for now
+
+ generateSnapshot(engine, "pre");
+
+ }
+
+
+ private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs,
+ List<Class<? extends Migrator>> migratorClasses) {
+ List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
+ if (cArgs.scripts.isEmpty() && cArgs.runDisabled.isEmpty()) {
+ return migratorClasses;
+
+ }
+ for (Class<? extends Migrator> migratorClass : migratorClasses) {
+ if (migratorExplicitlySpecified(cArgs, migratorClass.getSimpleName()) || migratorToRunWhenDisabled(cArgs, migratorClass.getSimpleName())) {
+ migratorClassesToRun.add(migratorClass);
+ }
+ }
+ return migratorClassesToRun;
+ }
+ private boolean migratorExplicitlySpecified(CommandLineArgs cArgs, String migratorName){
+ return !cArgs.scripts.isEmpty() && cArgs.scripts.contains(migratorName);
+ }
+ private boolean migratorToRunWhenDisabled(CommandLineArgs cArgs, String migratorName){
+ return !cArgs.runDisabled.isEmpty() && cArgs.runDisabled.contains(migratorName);
+ }
+
+ private void sortList(List<Class<? extends Migrator>> migratorClasses) {
+ Collections.sort(migratorClasses, (m1, m2) -> {
+ try {
+ if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
+ return 1;
+ } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) {
+ return -1;
+ } else {
+ return m1.getSimpleName().compareTo(m2.getSimpleName());
+ }
+ } catch (Exception e) {
+ return 0;
+ }
+ });
+ }
+
+
+ private void generateSnapshot(TransactionalGraphEngine engine, String phase) {
+
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dateStr= fd.getDateTime();
+ String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
+ logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
+ Graph transaction = null;
+ try {
+
+ Path pathToFile = Paths.get(fileName);
+ if (!pathToFile.toFile().exists()) {
+ Files.createDirectories(pathToFile.getParent());
+ }
+ transaction = engine.startTransaction();
+ transaction.io(IoCore.graphson()).writeGraph(fileName);
+ engine.rollback();
+ } catch (IOException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
+ LoggingContext.successStatusFields();
+ engine.rollback();
+ }
+
+ logAndPrint( phase + " migration snapshot saved to " + fileName);
+ }
+ /**
+ * Log and print.
+ *
+ * @param msg
+ * the msg
+ */
+ protected void logAndPrint(String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+ /**
+ * Commit changes.
+ *
+ * @param engine
+ * the graph transaction
+ * @param migrator
+ * the migrator
+ * @param cArgs
+ */
+ protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) {
+
+ String simpleName = migrator.getClass().getSimpleName();
+ String message;
+ if (migrator.getStatus().equals(Status.FAILURE)) {
+ message = "Migration " + simpleName + " Failed. Rolling back.";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("\t" + message);
+ LoggingContext.successStatusFields();
+ migrator.rollback();
+ } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
+ message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("\t" + message);
+ LoggingContext.successStatusFields();
+ migrator.rollback();
+ } else {
+ MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
+
+ if (cArgs.commit) {
+ if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
+ engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
+ }
+ engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
+ .property(simpleName, true).iterate();
+ MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
+ notifications.add(migrator.getNotificationHelper());
+ migrator.commit();
+ message = "Migration " + simpleName + " Succeeded. Changes Committed.";
+ logAndPrint("\t"+ message +"\t");
+ } else {
+ message = "--commit not specified. Not committing changes for " + simpleName + " to database.";
+ logAndPrint("\t" + message);
+ migrator.rollback();
+ }
+
+ }
+
+ resultsSummary.add(message);
+
+ }
+
+ private void outputResultsSummary() {
+ logAndPrint("---------------------------------");
+ logAndPrint("-------------Summary-------------");
+ for (String result : resultsSummary) {
+ logAndPrint(result);
+ }
+ logAndPrint("---------------------------------");
+ logAndPrint("---------------------------------");
+ }
}
class CommandLineArgs {
- @Parameter(names = "--help", help = true)
- public boolean help;
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-c", description = "location of configuration file")
+ public String config;
+
+ @Parameter(names = "-m", description = "names of migration scripts")
+ public List<String> scripts = new ArrayList<>();
- @Parameter(names = "-c", description = "location of configuration file")
- public String config;
+ @Parameter(names = "-l", description = "list the status of migrations")
+ public boolean list = false;
- @Parameter(names = "-m", description = "names of migration scripts")
- public List<String> scripts = new ArrayList<>();
+ @Parameter(names = "-d", description = "location of data snapshot", hidden = true)
+ public String dataSnapshot;
- @Parameter(names = "-l", description = "list the status of migrations")
- public boolean list = false;
+ @Parameter(names = "-f", description = "force migrations to be rerun")
+ public boolean forced = false;
- @Parameter(names = "-d", description = "location of data snapshot", hidden = true)
- public String dataSnapshot;
+ @Parameter(names = "--commit", description = "commit changes to graph")
+ public boolean commit = false;
- @Parameter(names = "-f", description = "force migrations to be rerun")
- public boolean forced = false;
+ @Parameter(names = "-e", description = "exclude list of migrator classes")
+ public List<String> excludeClasses = new ArrayList<>();
- @Parameter(names = "--commit", description = "commit changes to graph")
- public boolean commit = false;
+ @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot")
+ public boolean skipPreMigrationSnapShot = false;
- @Parameter(names = "-e", description = "exclude list of migrator classes")
- public List<String> excludeClasses = new ArrayList<>();
+ @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot")
+ public boolean skipPostMigrationSnapShot = false;
- @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot")
- public boolean skipPreMigrationSnapShot = false;
+ @Parameter(names = "--runDisabled", description = "List of migrators which are to be run even when disabled")
+ public List<String> runDisabled = new ArrayList<>();
- @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot")
- public boolean skipPostMigrationSnapShot = false;
}
--- /dev/null
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.javatuples.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.onap.aai.AAISetup;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.edges.enums.EdgeProperty;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+public class EdgeMigratorTest extends AAISetup {
+
+ public static class EdgeMigratorImpl extends EdgeMigrator {
+
+ public EdgeMigratorImpl(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ private static final Map<String, String> LABEL_MAP = getLabelMap();
+
+ private static Map<String, String> getLabelMap() {
+ Map<String, String> labelMap = new HashMap<>();
+ labelMap.put("usesLogicalLink", "tosca.relationships.network.LinksTo");
+ labelMap.put("sourceLInterface", "org.onap.relationships.inventory.Source");
+ labelMap.put("targetLInterface", "org.onap.relationships.inventory.Destination");
+ labelMap.put("isMemberOf", "org.onap.relationships.inventory.MemberOf");
+ labelMap.put("isA", "org.onap.relationships.inventory.IsA");
+ labelMap.put("has", "org.onap.relationships.inventory.Uses");
+ return labelMap;
+ }
+
+ @Override
+ protected String selectLabel(Edge edge, Set<String> edgeLabels) {
+ return ( edgeLabels.contains(LABEL_MAP.get(edge.label())) ) ? LABEL_MAP.get(edge.label()) : null;
+ }
+
+ @Override
+ public List<Pair<String, String>> getAffectedNodePairTypes() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "RebuildAllEdges";
+ }
+ }
+
+ private final static ModelType introspectorFactoryType = ModelType.MOXY;
+ private final static QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ private final static DBConnectionType type = DBConnectionType.REALTIME;
+ private JanusGraph graph;
+ private GraphTraversalSource g;
+ private Graph tx;
+ private EdgeMigratorImpl edgeMigrator;
+
+ @Before
+ public void setUp() {
+ graph = JanusGraphFactory.build().set("storage.backend","inmemory").open();
+ JanusGraphManagement janusgraphManagement = graph.openManagement();
+ tx = graph.newTransaction();
+ g = tx.traversal();
+ Loader loader =
+ loaderFactory.createLoaderForVersion(introspectorFactoryType, schemaVersions.getDefaultVersion());
+ TransactionalGraphEngine dbEngine = new JanusGraphDBEngine(
+ queryStyle,
+ type,
+ loader);
+
+ Vertex gvnf = g.addV().property(AAIProperties.NODE_TYPE, "generic-vnf")
+ .property("vnf-id", "toscaMigration-test-vnf")
+ .next();
+
+ Vertex lInterface = g.addV().property(AAIProperties.NODE_TYPE, "l-interface")
+ .property("interface-name", "toscaMigration-test-lint")
+ .next();
+
+ Vertex logicalLink = g.addV().property(AAIProperties.NODE_TYPE, "logical-link")
+ .property("link-name", "toscaMigration-logical-link")
+ .next();
+
+ gvnf.addEdge("has", lInterface, EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString(),
+ EdgeProperty.DELETE_OTHER_V.toString(), AAIDirection.NONE.toString());
+
+ lInterface.addEdge("usesLogicalLink", logicalLink, EdgeProperty.CONTAINS.toString(),
+ AAIDirection.NONE.toString(),EdgeProperty.DELETE_OTHER_V.toString(), AAIDirection.NONE.toString());
+ lInterface.addEdge("sourceLInterface", logicalLink, EdgeProperty.CONTAINS.toString(),
+ AAIDirection.NONE.toString(),EdgeProperty.DELETE_OTHER_V.toString(), AAIDirection.NONE.toString());
+
+ Vertex pserver = g.addV().next();
+ pserver.property("aai-node-type","pserver","hostname","a-name");
+ Vertex vnfc = g.addV().next();
+ vnfc.property("aai-node-type","vnfc","vnfc-name","a-name");
+ pserver.addEdge("blah", vnfc, EdgeProperty.CONTAINS.toString(), AAIDirection.NONE.toString(),
+ EdgeProperty.DELETE_OTHER_V.toString(), AAIDirection.NONE.toString());
+
+ TransactionalGraphEngine spy = spy(dbEngine);
+ TransactionalGraphEngine.Admin adminSpy = spy(dbEngine.asAdmin());
+ GraphTraversalSource traversal = g;
+ when(spy.asAdmin()).thenReturn(adminSpy);
+ when(adminSpy.getTraversalSource()).thenReturn(traversal);
+ Mockito.doReturn(janusgraphManagement).when(adminSpy).getManagementSystem();
+ edgeMigrator = new EdgeMigratorImpl(spy, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ @After
+ public void cleanUp() {
+ tx.tx().rollback();
+ graph.close();
+ }
+
+ @Test
+ public void verifyVnfHasOnlyNewEdgeTest() {
+ edgeMigrator.rebuildEdges(g.E().toSet());
+ assertTrue("edge direction and label were migrated", g.V().has(AAIProperties.NODE_TYPE,
+ "generic-vnf").has("vnf-id", "toscaMigration-test-vnf").inE()
+ .hasLabel("org.onap.relationships.inventory.BelongsTo").hasNext());
+
+
+ assertFalse("if we look for old edge, it should be gone", g.V().has(AAIProperties.NODE_TYPE,
+ "generic-vnf").has("vnf-id", "toscaMigration-test-vnf").outE()
+ .hasLabel("hasLInterface").hasNext());
+ }
+
+ @Test
+ public void verifyGraphHasNoOldEdgeLabelsTest() {
+ edgeMigrator.rebuildEdges(g.E().toSet());
+ assertEquals("Graph should have none of the old edge label"
+ , Long.valueOf(0)
+ , g.E().hasLabel("hasLInterface","usesLogicalLink").count().next());
+ assertEquals("Graph should have none of the old edge label"
+ , Long.valueOf(3)
+ , g.E().hasLabel("org.onap.relationships.inventory.BelongsTo",
+ "tosca.relationships.network.LinksTo","org.onap.relationships.inventory.Source")
+ .count().next());
+ }
+
+ @Test
+ public void verifyGenericVnfHas1EdgeTest() {
+ edgeMigrator.rebuildEdges(g.E().toSet());
+ assertEquals("Generic vnf should have 1 edge"
+ , Long.valueOf(1)
+ , g.V().has(AAIProperties.NODE_TYPE, "generic-vnf")
+ .both()
+ .count().next());
+
+ }
+
+ @Test
+ public void verifyLogicalLinkHas2EdgesTest() {
+ edgeMigrator.rebuildEdges(g.E().toSet());
+ assertEquals("Logical Link should have 2 edges"
+ , Long.valueOf(2)
+ , g.V().has(AAIProperties.NODE_TYPE, "logical-link")
+ .both()
+ .count().next());
+
+ assertTrue("Logical Link has source edge"
+ , g.V().has(AAIProperties.NODE_TYPE, "logical-link")
+ .bothE("org.onap.relationships.inventory.Source").hasNext());
+
+ assertTrue("Logical Link has default edge"
+ , g.V().has(AAIProperties.NODE_TYPE, "logical-link")
+ .bothE("tosca.relationships.network.LinksTo").hasNext());
+
+ }
+
+ @Test
+ public void checkThatEdgeWithNoRulesDoesNotGetMigratedTest() {
+ edgeMigrator.rebuildEdges(g.E().toSet());
+ assertTrue("Edge with no rule did not get migrated ", g.E().hasLabel("blah").hasNext());
+ }
+
+ @Test
+ public void rebuildEdgesWithMulplicityExceptionTest() {
+ Vertex sriovvf = g.addV().property(AAIProperties.NODE_TYPE, "sriov-vf").next();
+ Vertex lInterface = g.addV().property(AAIProperties.NODE_TYPE, "l-interface")
+ .property("interface-name1", "toscaMigration-test-lint")
+ .next();
+
+ sriovvf.addEdge("test1",lInterface,EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString(),
+ EdgeProperty.DELETE_OTHER_V.toString(), AAIDirection.NONE.toString());
+ sriovvf.addEdge("test2",lInterface,EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString(),
+ EdgeProperty.DELETE_OTHER_V.toString(), AAIDirection.NONE.toString());
+ edgeMigrator.rebuildEdges(g.E().toSet());
+ assertEquals(1, edgeMigrator.edgeMultiplicityExceptionCtr.size());
+ assertEquals(0, edgeMigrator.edgeMissingParentProperty.size());
+ assertEquals(1, edgeMigrator.edgeMultiplicityExceptionCtr.values().stream().mapToInt(Number::intValue).sum());
+ }
+}
\ No newline at end of file
Code Review / aai / graphadmin.git / commitdiff