Upgrade aai-graphadmin to Janusgraph 0.5.3
[aai/graphadmin.git] / src / main / java / org / onap / aai / migration / v13 / MigrateBadWidgetModelsPartTwo.java
1 /**
2  * ============LICENSE_START=======================================================
3  * org.onap.aai
4  * ================================================================================
5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *    http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20 package org.onap.aai.migration.v13;
21
22 import java.io.BufferedReader;
23 import java.io.FileNotFoundException;
24 import java.io.FileReader;
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.HashMap;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Optional;
31 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
32 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
33 import org.apache.tinkerpop.gremlin.structure.Direction;
34 import org.apache.tinkerpop.gremlin.structure.Edge;
35 import org.apache.tinkerpop.gremlin.structure.Vertex;
36 import org.javatuples.Pair;
37 import org.onap.aai.db.props.AAIProperties;
38 import org.onap.aai.edges.EdgeIngestor;
39 import org.onap.aai.introspection.LoaderFactory;
40 import org.onap.aai.migration.EdgeSwingMigrator;
41 import org.onap.aai.migration.MigrationDangerRating;
42 import org.onap.aai.migration.MigrationPriority;
43 import org.onap.aai.migration.Status;
44 import org.onap.aai.serialization.db.EdgeSerializer;
45 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
46 import org.onap.aai.setup.SchemaVersions;
47
48
49 //@Enabled
50 @MigrationPriority(21)
51 @MigrationDangerRating(100)
52 public class MigrateBadWidgetModelsPartTwo extends EdgeSwingMigrator {
53         private boolean success = true;
54         private final GraphTraversalSource g;
55         
56         // NOTE -- this migration is for "model-ver" nodes only.  It needs to be run AFTER 
57         //   the MigrateWidgetModelsPartOne.
58         //  
59         
60         // migration restrictions that we will use for this migration
61         private final String NODE_TYPE_RESTRICTION = "model-element";
62         private final String EDGE_LABEL_RESTRICTION = "org.onap.relationships.inventory.IsA";
63         private final String EDGE_DIR_RESTRICTION = "IN";
64         
65         GraphTraversal<Vertex, Vertex> widgetModelTraversal;
66         GraphTraversal<Vertex, Vertex> widgetModelVersionTraversal;
67         GraphTraversal<Vertex, Vertex> validModVerTraversal;
68
69
70
71         public MigrateBadWidgetModelsPartTwo(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
72                 super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
73                 this.g = this.engine.asAdmin().getTraversalSource();
74         }
75
76
77         @Override
78         public Status getStatus() {
79                 if (success) {
80                         return Status.SUCCESS;
81                 } else {
82                         return Status.FAILURE;
83                 }
84         }
85
86         @Override
87         public Optional<String[]> getAffectedNodeTypes() {
88                 return Optional.of(new String[]{"model", "model-element", "model-ver"});
89         }
90
91         @Override
92         public String getMigrationName() {
93                 return "MigrateBadWidgetModelsPartTwo";
94         }
95         
96         
97         /**
98          * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate.
99          * @return
100          */
101         @Override
102         public List<Pair<Vertex, Vertex>> getAffectedNodePairs() {
103                 logAndPrintInfo("--------- GET AFFECTED NODE PAIRS -------------");
104                 // Read the json file to populate the validWidgetModelVesionIdHash and also
105                 // validWidgetModelInvIdHash which will be used to figure out which data is in the db with
106                 // an invalid id.
107                 ArrayList <String> fileLines = readInValidWidgetInfoFile();
108                 
109                 // validWidgetModelVersionIdHash:  key = nodeType, value = validModelVersionId for that nodeType
110                 //       Note - we currently only have one valid version per model for widget models.
111                 HashMap <String,String> validModelVersionIdHash = getModelVersionIdHash( fileLines ); 
112                 
113                 // validWidgetModelVersionIdHash:  key = nodeType, value = validModelVersionId for that nodeType
114                 //       Note - we currently only have one valid version per model for widget models.
115                 HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines );
116                 
117                 // Now we will see what is actually in the DB
118                 List<Pair<Vertex, Vertex>> fromToVertPairList = new ArrayList<Pair<Vertex, Vertex>>();
119                 widgetModelTraversal = this.engine.asAdmin().getTraversalSource().V()
120                         .has("aai-node-type", "model")
121                         .has("model-type", "widget");
122
123                 if(!(widgetModelTraversal.hasNext())){
124                         logAndPrintInfo("unable to find widget models in database. ");
125                 }
126                 
127                 int validModelVerCount = 0;
128                 while (widgetModelTraversal.hasNext()) {
129                         Vertex widgetModVertex = widgetModelTraversal.next();
130                         String invId = widgetModVertex.property("model-invariant-id").value().toString();
131                         
132                         // Find the model-version nodes that belong to this model.
133                         // We expect just one per model, but there could be more.
134                         widgetModelVersionTraversal = this.engine.asAdmin().getTraversalSource()
135                     .V(widgetModVertex)
136                     .in("org.onap.relationships.inventory.BelongsTo")
137                     .has("aai-node-type", "model-ver");
138                         
139                         if(!(widgetModelVersionTraversal.hasNext())){
140                                 logAndPrintInfo("unable to find widget model version in database for model-invariant-id = [" + invId + "].");
141                         }
142
143                         while (widgetModelVersionTraversal.hasNext()) {
144                                 Vertex widgetModVersionVertex = widgetModelVersionTraversal.next();
145                                 String modVersionIdInDb = widgetModVersionVertex.property("model-version-id").value().toString();
146                                 String nodeType = widgetModVersionVertex.property("model-name").value().toString();
147                                 
148                                 if( validModelVersionIdHash.containsKey(nodeType) ){
149                                         // We know what the model-version-id SHOULD be, so make sure we're using it.
150                                         String validModVerId = validModelVersionIdHash.get(nodeType);
151                                         if( !modVersionIdInDb.equals(validModVerId) ){
152                                                 logAndPrintInfo(" Bad model-version-id found in DB for model-name = " + nodeType + ", verId = " + modVersionIdInDb );
153                                                 validModVerTraversal = this.engine.asAdmin().getTraversalSource()
154                                                                 .V()
155                                                                 .has("model-version-id",validModVerId)
156                                                                 .has("aai-node-type","model-ver");
157                                     if(!(validModVerTraversal.hasNext())){
158                                                         logAndPrintInfo("unable to find widget model version in database for valid model-version-id = [" + validModVerId + "].");
159                                                 }
160                                     int ct = 0;
161                                     while (validModVerTraversal.hasNext()) {
162                                         ct++;
163                                                         if( ct > 1 ){
164                                                                 logAndPrintInfo("ERROR - More than one model-ver found for model-version-id = [" + validModVerId + "].");
165                                                                 break;
166                                                         }
167                                                         Vertex toVert = validModVerTraversal.next();
168                                                         fromToVertPairList.add(new Pair<>(widgetModVersionVertex, toVert));
169                                     }
170                                         }
171                                         else {
172                                                 validModelVerCount++;
173                                                 logAndPrintInfo("Valid model-version-id used in DB for model-name = [" + nodeType + "].");
174                                         }
175                                 }
176                                 else {
177                                         logAndPrintInfo("unable to find a valid widget model-ver in database for model-name = [" + nodeType + "].");
178                                 }
179                         }
180                 }
181                                 
182                 return fromToVertPairList;
183         }
184         
185         
186         public String getNodeTypeRestriction(){
187                 return NODE_TYPE_RESTRICTION;
188         }
189
190         public String getEdgeLabelRestriction(){
191                 return EDGE_LABEL_RESTRICTION;
192         }
193         
194         public String getEdgeDirRestriction(){
195                 return EDGE_DIR_RESTRICTION;
196         }
197         
198         /**
199          * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate.
200          * @return
201          */
202         public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) {
203                 
204                 // The first node in each pair is the model-ver that we were migrating edges AWAY FROM because
205                 //    it is an invalid model-ver node.
206                 // Delete those as well as their parent model node (if the parent model node has no other users
207                 //    and is not on the validModelInvIdList).
208                 
209                 int badModelVerCount = 0;
210                 int modelVerDelCount = 0;
211                 int modelDelCount = 0;
212                 int parentPreventValidDelCount = 0;
213                 
214                 HashMap <String,String> parentPreventInEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it.
215                 HashMap <String,String> parentPreventOutEdgeIdHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it.
216                 HashMap <String,String> parentPreventIsaEdgeDelHash = new HashMap <String,String> (); // using a hash so we can count the # of models, not edges to it.
217                 
218                 ArrayList <String> fileLines = readInValidWidgetInfoFile();
219                 // validWidgetModelVersionIdHash:  key = nodeType, value = validModelVersionId for that nodeType
220                 //       Note - we currently only have one valid version per model for widget models.
221                 HashMap <String,String> validModelInvariantIdHash = getModelInvariantIdHash( fileLines );
222                 
223                 try {
224                         for (Pair<Vertex, Vertex> nodePair : nodePairL) {
225                                 // The "fromNode" is the "bad/old" model-ver node that we moved off of
226                                 badModelVerCount++;
227                                 Vertex oldNode = nodePair.getValue0();  
228                                 String oldModVerId = oldNode.property("model-version-id").value().toString();
229                                 Vertex parentModelNode = null;
230                                 
231                                 //DOUBLE CHECK THAT THIS IS NOT a valid model-version-id
232                                 
233                                 
234                                 boolean okToDelete = true;
235                                 //---- delete the oldNode if the only edge it has is its "belongsTo/OUT" edge to its parent model.
236                                 //     AND if its parent node does not have any named-query edges ("IsA" edges) pointing to it.
237                                 Iterator <Edge> edgeInIter = oldNode.edges(Direction.IN);
238                                 while( edgeInIter.hasNext() ){
239                                         Edge inE = edgeInIter.next();
240                                         Vertex otherSideNode4ThisEdge = inE.inVertex();
241                                         String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
242                                         // If there are any IN edges, we won't delete this thing.
243                                         okToDelete = false;
244                                         logAndPrintInfo("We will not delete old model-ver node because it still has IN edges. This model-version-id = [" 
245                                                         + oldModVerId + "], has IN edge from a [" + otherSideNodeType + "] node. ");
246                                 }
247                                 if( okToDelete ){
248                                         // there were no OUT edges, make sure the only OUT edge is to it's parent
249                                         Iterator <Edge> edgeOutIter = oldNode.edges(Direction.OUT);
250                                         int edgeCount = 0;
251                                         while( edgeOutIter.hasNext() ){
252                                                 Edge badModVerE = edgeOutIter.next();
253                                                 edgeCount++;
254                                                 if( edgeCount > 1 ){
255                                                         // If there are more than one OUT edges, we won't delete this thing.
256                                                         okToDelete = false;
257                                                         parentModelNode = null;
258                                                         logAndPrintInfo("We will not delete old model-ver node because it still has > 1 OUT-edges.  model-version-id = [" + oldModVerId + "].");
259                                                 }
260                                                 else {
261                                                         String eLabel = badModVerE.label().toString();
262                                                         Vertex otherSideNode4ThisEdge = badModVerE.inVertex();
263                                                         String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
264                                                         if( ! eLabel.equals("org.onap.relationships.inventory.BelongsTo") ){
265                                                                 logAndPrintInfo("We will not delete old model-ver node because it still has a non 'belongsTo' OUT-edge.  model-version-id = [" 
266                                                                                 + oldModVerId + "], edgeLabel = [" + eLabel + "] edge goes to a [" + otherSideNodeType + "]. ");
267                                                                 okToDelete = false;
268                                                         }
269                                                         else {
270                                                                 if( ! otherSideNodeType.equals("model") ){
271                                                                         logAndPrintInfo("We will not delete old model-ver node (model-version-id = [" + oldModVerId + "]) "
272                                                                                 + " because it still has an OUT edge to a [" + otherSideNodeType + "] node. ");
273                                                                         okToDelete = false;
274                                                                         parentModelNode = null;
275                                                                 }
276                                                                 else {
277                                                                         parentModelNode = otherSideNode4ThisEdge;
278                                                                         String parentInvId = parentModelNode.property("model-invariant-id").value().toString();
279                                                                         Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN);
280                                                                         while( pInIter.hasNext() ){
281                                                                                 Edge inE = pInIter.next();
282                                                                                 String inELabel = inE.label().toString();
283                                                                                 if( ! inELabel.equals("org.onap.relationships.inventory.BelongsTo") ){
284                                                                                         Vertex otherSideNode = inE.outVertex();
285                                                                                         String otherSideNT = otherSideNode.value(AAIProperties.NODE_TYPE);
286                                                                                         // If there are any IN edges still on the parent,
287                                                                                         //   we won't delete this model-ver since once the model-ver
288                                                                                         //   is gone, its hard to know what nodeType the model was
289                                                                                         //   for - so it would be hard to know what valid model-invariant-id
290                                                                                         //   to migrate its edges to.
291                                                                                         okToDelete = false;
292                                                                                         parentPreventIsaEdgeDelHash.put(parentInvId,"");
293                                                                                         logAndPrintInfo("We will not delete old model-ver node because its"
294                                                                                                         + " parent model still has IN edges. The model with model-invariant-id = [" 
295                                                                                                         + parentInvId + "], has an non-belongsTo IN edge, label = [" 
296                                                                                                         + inELabel + "] from a [" + otherSideNT + "] node. ");
297                                                                                 }
298                                                                         }
299                                                                 }
300                                                         }
301                                                 }
302                                         }
303                                 }
304                                                 
305                                 if( okToDelete ){
306                                         logAndPrintInfo(" >>> DELETEING model-ver node with model-version-id = [" + oldModVerId + "]" );
307                                         modelVerDelCount++;
308                                         oldNode.remove();
309                                 }
310                                 
311                                 if( parentModelNode != null && okToDelete ){
312                                         // Delete the corresponding parent model IF it now has no 
313                                         //     edges anymore (and is not in our known valid model list)
314                                         //     and we were deleting the model-ver also.
315                                         boolean okToDelParent = true;
316                                         String parentModInvId = parentModelNode.property("model-invariant-id").value().toString();
317                                         
318                                         if( validModelInvariantIdHash.containsValue(parentModInvId) ){
319                                                 okToDelParent = false;
320                                                 logAndPrintInfo("We will not delete old model node because it is on our valid widget list. "
321                                                                 + " model-invariant-id = [" + parentModInvId + "] ");
322                                                 parentPreventValidDelCount++;
323                                         }       
324                                         else {
325                                                 Iterator <Edge> pInIter = parentModelNode.edges(Direction.IN);
326                                                 while( pInIter.hasNext() ){
327                                                         Edge inE = pInIter.next();
328                                                         String inELabel = inE.label().toString();
329                                                         Vertex otherSideNode4ThisEdge = inE.outVertex();
330                                                         String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
331                                                         // If there are any IN edges, we won't delete this thing.
332                                                         okToDelParent = false;
333                                                         parentPreventInEdgeIdHash.put(parentModInvId, "");
334                                                         logAndPrintInfo("We will not delete old model node (yet) because it still has IN edges. This model-invariant-id = [" 
335                                                                 + parentModInvId + "], has IN edge, label = [" 
336                                                                 + inELabel + "] from a [" + otherSideNodeType + "] node. ");
337                                                 }
338                                                 Iterator <Edge> pOutIter = parentModelNode.edges(Direction.OUT);
339                                                 while( pOutIter.hasNext() ){
340                                                         Edge outE = pOutIter.next();
341                                                         String outELabel = outE.label().toString();
342                                                         Vertex otherSideNode4ThisEdge = outE.inVertex();
343                                                         String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
344                                                         // If there are any OUT edges, we won't delete this thing.
345                                                         okToDelParent = false;
346                                                         parentPreventOutEdgeIdHash.put(parentModInvId, "");
347                                                         logAndPrintInfo("We will not delete old model node because it still has OUT edges. This model-invariant-id = [" 
348                                                                 + parentModInvId + "], has OUT edge, label = [" 
349                                                                 + outELabel + "]  to a [" + otherSideNodeType + "] node. ");
350                                                 }
351                                         }
352                                 
353                                         if( okToDelParent ){
354                                                 if( parentPreventInEdgeIdHash.containsKey(parentModInvId) ){
355                                                         // This parent had been prevented from being deleted until all its 
356                                                         // child model-ver's were deleted (it must have had more than one).
357                                                         // So we can now remove it from the list of parent guys that
358                                                         // could not be deleted.
359                                                         parentPreventInEdgeIdHash.remove(parentModInvId);
360                                                 }
361                                                 logAndPrintInfo(" >>> DELETEING model node which was the parent of model-ver with model-version-id = [" 
362                                                                 + oldModVerId + "]. This model-invariant-id = [" + parentModInvId + "]" );
363                                                 modelDelCount++;
364                                                 parentModelNode.remove();
365                                         }
366                                 }
367                         }
368                         
369                         logAndPrintInfo(" >>> SUMMARY: total number of bad model-ver nodes found = " + badModelVerCount );
370                         logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes deleted = " + modelVerDelCount );
371                         logAndPrintInfo(" >>> SUMMARY: number of model nodes deleted = " + modelDelCount );
372                         logAndPrintInfo(" >>> SUMMARY: number of model-ver nodes not deleted because their PARENT still had IsA edges = " 
373                                         + parentPreventIsaEdgeDelHash.size() );
374                         logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they were valid = " 
375                                         + parentPreventValidDelCount);
376                         logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had IN edges = " 
377                                         + parentPreventInEdgeIdHash.size() );
378                         logAndPrintInfo(" >>> SUMMARY: number of model nodes not deleted because they had OUT edges = " 
379                                         + parentPreventOutEdgeIdHash.size() );
380                         
381                         
382                 } catch (Exception e) {
383                         logger.error("error encountered", e );
384                         success = false;
385                 }       
386                 
387         }
388         
389         private ArrayList <String> readInValidWidgetInfoFile(){
390                 
391                 ArrayList <String> fileLines = new ArrayList <String> ();
392                 String homeDir = System.getProperty("AJSC_HOME");
393                 String configDir = System.getProperty("BUNDLECONFIG_DIR");
394                 if (homeDir == null) {
395                         logAndPrintInfo("ERROR: Could not find sys prop AJSC_HOME");
396                         success = false;
397                         return fileLines;
398                 }
399                 if (configDir == null) {
400                         logAndPrintInfo("ERROR: Could not find sys prop BUNDLECONFIG_DIR");
401                         success = false;
402                         return fileLines;
403                 }
404                 String fileName = homeDir + "/" + configDir + "/" + "migration-input-files/widget-model-migration-data/widget-model-migration-input.csv";
405                 try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
406                         String modelInfoLine;
407                         while ((modelInfoLine = br.readLine()) != null) {
408                                 modelInfoLine = modelInfoLine.replace("\n", "").replace("\r", "");
409                                 if (!modelInfoLine.isEmpty()) {
410                                         fileLines.add(modelInfoLine);
411                                 }
412                         }
413                 } 
414                 catch (FileNotFoundException e) {
415                         logger.error("ERROR: Could not find file " + fileName, e);
416                         success = false;
417                 } catch (IOException e) {
418                         logger.error("ERROR: Issue reading file " + fileName, e);
419                         success = false;
420                 } catch (Exception e) {
421                         logger.error("encountered exception", e);
422                         success = false;
423                 }
424                 return fileLines;
425         }
426         
427         
428         HashMap <String,String> getModelVersionIdHash( ArrayList <String> fileLines ){
429                 
430                 HashMap <String, String> versionIdHash = new HashMap <String,String> ();
431                 
432                 if( fileLines == null ){
433                         logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash");
434                         success = false;
435                         return versionIdHash;
436                 }
437                 
438                 for(int i = 0; i < fileLines.size(); i++ ){
439                         String mLine = fileLines.get(i);
440                         String[] fields = mLine.split("\\,");
441                         if (fields.length != 3) {
442                                 logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line.");
443                                 success = false;
444                         }
445                         else {
446                                 versionIdHash.put(fields[0],fields[1]);
447                         }
448                 }
449                 
450                 // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is 
451                 //   the correct model info for "virtual-data-center".  Problem is that there is no vdc nodeType, but
452                 //   there are named-queries pointing at a bad widget-model for "vdc".
453                 String virtDataCenterVerId = versionIdHash.get("virtual-data-center");
454                 if( virtDataCenterVerId != null ){
455                         versionIdHash.put("vdc",virtDataCenterVerId );
456                 }
457                 
458                 return versionIdHash;
459         }
460                         
461         
462         HashMap <String,String> getModelInvariantIdHash( ArrayList <String> fileLines ){
463                 HashMap <String, String> invIdHash = new HashMap <String,String> ();
464                 
465                 if( fileLines == null ){
466                         logAndPrintInfo("ERROR: null fileLines array passed to getModelVersionIdHash");
467                         success = false;
468                         return invIdHash;
469                 }
470                 
471                 for(int i = 0; i < fileLines.size(); i++ ){
472                         String mLine = fileLines.get(i);
473                         String[] fields = mLine.split("\\,");
474                         if (fields.length != 3) {
475                                 logAndPrintInfo("ERROR: row in data file did not contain 3 elements. should have: model-name,model-version-id,model-invariant-id on each line.");
476                                 success = false;
477                         }
478                         else {
479                                 invIdHash.put(fields[0],fields[2]);
480                         }
481                 }
482                 
483                 // Because of some bad data in the db, we will manually map the nodeType of "vdc" to what is 
484                 //   the correct model info for "virtual-data-center".  Problem is that there is no vdc nodeType, but
485                 //   there are named-queries pointing at a bad widget-model for "vdc".
486                 String virtDataCenterInvId = invIdHash.get("virtual-data-center");
487                 if( invIdHash != null ){
488                         invIdHash.put("vdc",virtDataCenterInvId );
489                 }
490                 return invIdHash;
491         }
492         
493         /**
494          * Log and print.
495          *
496          * @param msg
497          *            the msg
498          */
499         protected void logAndPrintInfo(String msg) {
500                 System.out.println(msg);
501                 logger.info(msg);
502         }
503
504                 
505                         
506 }