2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 package org.onap.aai.migration.v13;
22 import java.io.BufferedReader;
23 import java.io.FileNotFoundException;
24 import java.io.FileReader;
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.HashMap;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Optional;
31 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
32 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
33 import org.apache.tinkerpop.gremlin.structure.Direction;
34 import org.apache.tinkerpop.gremlin.structure.Edge;
35 import org.apache.tinkerpop.gremlin.structure.Vertex;
36 import org.javatuples.Pair;
37 import org.onap.aai.db.props.AAIProperties;
38 import org.onap.aai.edges.EdgeIngestor;
39 import org.onap.aai.introspection.LoaderFactory;
40 import org.onap.aai.migration.EdgeSwingMigrator;
41 import org.onap.aai.migration.MigrationDangerRating;
42 import org.onap.aai.migration.MigrationPriority;
43 import org.onap.aai.migration.Status;
44 import org.onap.aai.serialization.db.EdgeSerializer;
45 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
46 import org.onap.aai.setup.SchemaVersions;
50 @MigrationPriority(21)
51 @MigrationDangerRating(100)
52 public class MigrateBadWidgetModelsPartTwo extends EdgeSwingMigrator {
53 private boolean success = true;
54 private final GraphTraversalSource g;
56 // NOTE -- this migration is for "model-ver" nodes only. It needs to be run AFTER
57 // the MigrateWidgetModelsPartOne.
60 // migration restrictions that we will use for this migration
61 private final String NODE_TYPE_RESTRICTION = "model-element";
62 private final String EDGE_LABEL_RESTRICTION = "org.onap.relationships.inventory.IsA";
63 private final String EDGE_DIR_RESTRICTION = "IN";
65 GraphTraversal<Vertex, Vertex> widgetModelTraversal;
66 GraphTraversal<Vertex, Vertex> widgetModelVersionTraversal;
67 GraphTraversal<Vertex, Vertex> validModVerTraversal;
71 public MigrateBadWidgetModelsPartTwo(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
72 super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
73 this.g = this.engine.asAdmin().getTraversalSource();
78 public Status getStatus() {
80 return Status.SUCCESS;
82 return Status.FAILURE;
87 public Optional<String[]> getAffectedNodeTypes() {
88 return Optional.of(new String[]{"model", "model-element", "model-ver"});
92 public String getMigrationName() {
93 return "MigrateBadWidgetModelsPartTwo";
98 * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate.
102 public List<Pair<Vertex, Vertex>> getAffectedNodePairs() {
103 logAndPrintInfo("--------- GET AFFECTED NODE PAIRS -------------");
104 // Read the json file to populate the validWidgetModelVesionIdHash and also
105 // validWidgetModelInvIdHash which will be used to figure out which data is in the db with
107 ArrayList <String> fileLines = readInValidWidgetInfoFile();
109 // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType
110 // Note - we currently only have one valid version per model for widget models.
111 HashMap <String,String> validModelVersionIdHash = getModelVersionIdHash( fileLines );
113 // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType
114 // Note - we currently only have one valid version per model for widget models.
115 HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines );
117 // Now we will see what is actually in the DB
118 List<Pair<Vertex, Vertex>> fromToVertPairList = new ArrayList<Pair<Vertex, Vertex>>();
119 widgetModelTraversal = this.engine.asAdmin().getTraversalSource().V()
120 .has("aai-node-type", "model")
121 .has("model-type", "widget");
123 if(!(widgetModelTraversal.hasNext())){
124 logAndPrintInfo("unable to find widget models in database. ");
127 int validModelVerCount = 0;
128 while (widgetModelTraversal.hasNext()) {
129 Vertex widgetModVertex = widgetModelTraversal.next();
130 String invId = widgetModVertex.property("model-invariant-id").value().toString();
132 // Find the model-version nodes that belong to this model.
133 // We expect just one per model, but there could be more.
134 widgetModelVersionTraversal = this.engine.asAdmin().getTraversalSource()
136 .in("org.onap.relationships.inventory.BelongsTo")
137 .has("aai-node-type", "model-ver");
139 if(!(widgetModelVersionTraversal.hasNext())){
140 logAndPrintInfo("unable to find widget model version in database for model-invariant-id = [" + invId + "].");
143 while (widgetModelVersionTraversal.hasNext()) {
144 Vertex widgetModVersionVertex = widgetModelVersionTraversal.next();
145 String modVersionIdInDb = widgetModVersionVertex.property("model-version-id").value().toString();
146 String nodeType = widgetModVersionVertex.property("model-name").value().toString();
148 if( validModelVersionIdHash.containsKey(nodeType) ){
149 // We know what the model-version-id SHOULD be, so make sure we're using it.
150 String validModVerId = validModelVersionIdHash.get(nodeType);
151 if( !modVersionIdInDb.equals(validModVerId) ){
152 logAndPrintInfo(" Bad model-version-id found in DB for model-name = " + nodeType + ", verId = " + modVersionIdInDb );
153 validModVerTraversal = this.engine.asAdmin().getTraversalSource()
155 .has("model-version-id",validModVerId)
156 .has("aai-node-type","model-ver");
157 if(!(validModVerTraversal.hasNext())){
158 logAndPrintInfo("unable to find widget model version in database for valid model-version-id = [" + validModVerId + "].");
161 while (validModVerTraversal.hasNext()) {
164 logAndPrintInfo("ERROR - More than one model-ver found for model-version-id = [" + validModVerId + "].");
167 Vertex toVert = validModVerTraversal.next();
168 fromToVertPairList.add(new Pair<>(widgetModVersionVertex, toVert));
172 validModelVerCount++;
173 logAndPrintInfo("Valid model-version-id used in DB for model-name = [" + nodeType + "].");
177 logAndPrintInfo("unable to find a valid widget model-ver in database for model-name = [" + nodeType + "].");
182 return fromToVertPairList;
186 public String getNodeTypeRestriction(){
187 return NODE_TYPE_RESTRICTION;
190 public String getEdgeLabelRestriction(){
191 return EDGE_LABEL_RESTRICTION;
194 public String getEdgeDirRestriction(){
195 return EDGE_DIR_RESTRICTION;
199 * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate.
202 public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) {
204 // The first node in each pair is the model-ver that we were migrating edges AWAY FROM because
205 // it is an invalid model-ver node.
206 // Delete those as well as their parent model node (if the parent model node has no other users
207 // and is not on the validModelInvIdList).
209 int badModelVerCount = 0;
210 int modelVerDelCount = 0;
211 int modelDelCount = 0;
212 int parentPreventValidDelCount = 0;
214 HashMap <String,String> parentPreventInEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it.
215 HashMap <String,String> parentPreventOutEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it.
216 HashMap <String,String> parentPreventIsaEdgeDelHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it.
218 ArrayList <String> fileLines = readInValidWidgetInfoFile();
219 // validWidgetModelVersionIdHash: key = nodeType, value = validModelVersionId for that nodeType
220 // Note - we currently only have one valid version per model for widget models.
221 HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines );
224 for (Pair<Vertex, Vertex> nodePair : nodePairL) {
225 // The "fromNode" is the "bad/old" model-ver node that we moved off of
227 Vertex oldNode = nodePair.getValue0();
228 String oldModVerId = oldNode.property("model-version-id").value().toString();
229 Vertex parentModelNode = null;
231 //DOUBLE CHECK THAT THIS IS NOT a valid model-version-id
234 boolean okToDelete = true;
235 //---- delete the oldNode if the only edge it has is its "belongsTo/OUT" edge to its parent model.
236 // AND if its parent node does not have any named-query edges ("IsA" edges) pointing to it.
237 Iterator <Edge> edgeInIter = oldNode.edges(Direction.IN);
238 while( edgeInIter.hasNext() ){
239 Edge inE = edgeInIter.next();
240 Vertex otherSideNode4ThisEdge = inE.inVertex();
241 String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
242 // If there are any IN edges, we won't delete this thing.
244 logAndPrintInfo("We will not delete old model-ver node because it still has IN edges. This model-version-id = ["
245 + oldModVerId + "], has IN edge from a [" + otherSideNodeType + "] node. ");
248 // there were no OUT edges, make sure the only OUT edge is to it's parent
249 Iterator <Edge> edgeOutIter = oldNode.edges(Direction.OUT);
251 while( edgeOutIter.hasNext() ){
252 Edge badModVerE = edgeOutIter.next();
255 // If there are more than one OUT edges, we won't delete this thing.
257 parentModelNode = null;
258 logAndPrintInfo("We will not delete old model-ver node because it still has > 1 OUT-edges. model-version-id = [" + oldModVerId + "].");
261 String eLabel = badModVerE.label().toString();
262 Vertex otherSideNode4ThisEdge = badModVerE.inVertex();
263 String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
264 if( ! eLabel.equals("org.onap.relationships.inventory.BelongsTo") ){
265 logAndPrintInfo("We will not delete old model-ver node because it still has a non 'belongsTo' OUT-edge. model-version-id = ["
266 + oldModVerId + "], edgeLabel = [" + eLabel + "] edge goes to a [" + otherSideNodeType + "]. ");
270 if( ! otherSideNodeType.equals("model") ){
271 logAndPrintInfo("We will not delete old model-ver node (model-version-id = [" + oldModVerId + "]) "
272 + " because it still has an OUT edge to a [" + otherSideNodeType + "] node. ");
274 parentModelNode = null;
277 parentModelNode = otherSideNode4ThisEdge;
278 String parentInvId = parentModelNode.property("model-invariant-id").value().toString();
279 Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN);
280 while( pInIter.hasNext() ){
281 Edge inE = pInIter.next();
282 String inELabel = inE.label().toString();
283 if( ! inELabel.equals("org.onap.relationships.inventory.BelongsTo") ){
284 Vertex otherSideNode = inE.outVertex();
285 String otherSideNT = otherSideNode.value(AAIProperties.NODE_TYPE);
286 // If there are any IN edges still on the parent,
287 // we won't delete this model-ver since once the model-ver
288 // is gone, its hard to know what nodeType the model was
289 // for - so it would be hard to know what valid model-invariant-id
290 // to migrate its edges to.
292 parentPreventIsaEdgeDelHash.put(parentInvId,"");
293 logAndPrintInfo("We will not delete old model-ver node because its"
294 + " parent model still has IN edges. The model with model-invariant-id = ["
295 + parentInvId + "], has an non-belongsTo IN edge, label = ["
296 + inELabel + "] from a [" + otherSideNT + "] node. ");
306 logAndPrintInfo(" >>> DELETEING model-ver node with model-version-id = [" + oldModVerId + "]" );
311 if( parentModelNode != null && okToDelete ){
312 // Delete the corresponding parent model IF it now has no
313 // edges anymore (and is not in our known valid model list)
314 // and we were deleting the model-ver also.
315 boolean okToDelParent = true;
316 String parentModInvId = parentModelNode.property("model-invariant-id").value().toString();
318 if( validModelInvariantIdHash.containsValue(parentModInvId) ){
319 okToDelParent = false;
320 logAndPrintInfo("We will not delete old model node because it is on our valid widget list. "
321 + " model-invariant-id = [" + parentModInvId + "] ");
322 parentPreventValidDelCount++;
325 Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN);
326 while( pInIter.hasNext() ){
327 Edge inE = pInIter.next();
328 String inELabel = inE.label().toString();
329 Vertex otherSideNode4ThisEdge = inE.outVertex();
330 String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
331 // If there are any IN edges, we won't delete this thing.
332 okToDelParent = false;
333 parentPreventInEdgeIdHash.put(parentModInvId, "");
334 logAndPrintInfo("We will not delete old model node (yet) because it still has IN edges. This model-invariant-id = ["
335 + parentModInvId + "], has IN edge, label = ["
336 + inELabel + "] from a [" + otherSideNodeType + "] node. ");
338 Iterator <Edge> pOutIter = parentModelNode.edges(Direction.OUT);
339 while( pOutIter.hasNext() ){
340 Edge outE = pOutIter.next();
341 String outELabel = outE.label().toString();
342 Vertex otherSideNode4ThisEdge = outE.inVertex();
343 String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
344 // If there are any OUT edges, we won't delete this thing.
345 okToDelParent = false;
346 parentPreventOutEdgeIdHash.put(parentModInvId, "");
347 logAndPrintInfo("We will not delete old model node because it still has OUT edges. This model-invariant-id = ["
348 + parentModInvId + "], has OUT edge, label = ["
349 + outELabel + "] to a [" + otherSideNodeType + "] node. ");
354 if( parentPreventInEdgeIdHash.containsKey(parentModInvId) ){
355 // This parent had been prevented from being deleted until all its
356 // child model-ver's were deleted (it must have had more than one).
357 // So we can now remove it from the list of parent guys that
358 // could not be deleted.
359 parentPreventInEdgeIdHash.remove(parentModInvId);
361 logAndPrintInfo(" >>> DELETEING model node which was the parent of model-ver with model-version-id = ["
362 + oldModVerId + "]. This model-invariant-id = [" + parentModInvId + "]" );
364 parentModelNode.remove();
369 logAndPrintInfo(" >>> SUMMARY: total number of bad model-ver nodes found = " + badModelVerCount );
370 logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes deleted = " + modelVerDelCount );
371 logAndPrintInfo(" >>> SUMMARY: number of model nodes deleted = " + modelDelCount );
372 logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes not deleted because their PARENT still had IsA edges = "
373 + parentPreventIsaEdgeDelHash.size() );
374 logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they were valid = "
375 + parentPreventValidDelCount);
376 logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had IN edges = "
377 + parentPreventInEdgeIdHash.size() );
378 logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had OUT edges = "
379 + parentPreventOutEdgeIdHash.size() );
382 } catch (Exception e) {
383 logger.error("error encountered", e );
389 private ArrayList <String> readInValidWidgetInfoFile(){
391 ArrayList <String> fileLines = new ArrayList <String> ();
392 String homeDir = System.getProperty("AJSC_HOME");
393 String configDir = System.getProperty("BUNDLECONFIG_DIR");
394 if (homeDir == null) {
395 logAndPrintInfo("ERROR: Could not find sys prop AJSC_HOME");
399 if (configDir == null) {
400 logAndPrintInfo("ERROR: Could not find sys prop BUNDLECONFIG_DIR");
404 String fileName = homeDir + "/" + configDir + "/" + "migration-input-files/widget-model-migration-data/widget-model-migration-input.csv";
405 try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
406 String modelInfoLine;
407 while ((modelInfoLine = br.readLine()) != null) {
408 modelInfoLine = modelInfoLine.replace("\n", "").replace("\r", "");
409 if (!modelInfoLine.isEmpty()) {
410 fileLines.add(modelInfoLine);
414 catch (FileNotFoundException e) {
415 logger.error("ERROR: Could not find file " + fileName, e);
417 } catch (IOException e) {
418 logger.error("ERROR: Issue reading file " + fileName, e);
420 } catch (Exception e) {
421 logger.error("encountered exception", e);
428 HashMap <String,String> getModelVersionIdHash( ArrayList <String> fileLines ){
430 HashMap <String, String> versionIdHash = new HashMap <String,String> ();
432 if( fileLines == null ){
433 logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash");
435 return versionIdHash;
438 for(int i = 0; i < fileLines.size(); i++ ){
439 String mLine = fileLines.get(i);
440 String[] fields = mLine.split("\\,");
441 if (fields.length != 3) {
442 logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line.");
446 versionIdHash.put(fields[0],fields[1]);
450 // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is
451 // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but
452 // there are named-queries pointing at a bad widget-model for "vdc".
453 String virtDataCenterVerId = versionIdHash.get("virtual-data-center");
454 if( virtDataCenterVerId != null ){
455 versionIdHash.put("vdc",virtDataCenterVerId );
458 return versionIdHash;
462 HashMap <String,String> getModelInvariantIdHash( ArrayList <String> fileLines ){
463 HashMap <String, String> invIdHash = new HashMap <String,String> ();
465 if( fileLines == null ){
466 logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash");
471 for(int i = 0; i < fileLines.size(); i++ ){
472 String mLine = fileLines.get(i);
473 String[] fields = mLine.split("\\,");
474 if (fields.length != 3) {
475 logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line.");
479 invIdHash.put(fields[0],fields[2]);
483 // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is
484 // the correct model info for "virtual-data-center". Problem is that there is no vdc nodeType, but
485 // there are named-queries pointing at a bad widget-model for "vdc".
486 String virtDataCenterInvId = invIdHash.get("virtual-data-center");
487 if( invIdHash != null ){
488 invIdHash.put("vdc",virtDataCenterInvId );
499 protected void logAndPrintInfo(String msg) {
500 System.out.println(msg);