2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.aai.dbgen;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
26 import java.io.FileReader;
27 import java.io.FileWriter;
28 import java.io.IOException;
29 import java.util.ArrayList;
30 import java.util.Collection;
31 import java.util.HashMap;
32 import java.util.Iterator;
33 import java.util.LinkedHashSet;
34 import java.util.List;
36 import java.util.Map.Entry;
37 import java.util.Properties;
39 import java.util.UUID;
41 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
42 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
43 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
44 import org.apache.tinkerpop.gremlin.structure.Direction;
45 import org.apache.tinkerpop.gremlin.structure.Edge;
46 import org.apache.tinkerpop.gremlin.structure.Property;
47 import org.apache.tinkerpop.gremlin.structure.Vertex;
48 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
49 import org.openecomp.aai.db.props.AAIProperties;
50 import org.openecomp.aai.dbmap.AAIGraph;
51 import org.openecomp.aai.exceptions.AAIException;
52 import org.openecomp.aai.introspection.Introspector;
53 import org.openecomp.aai.introspection.Loader;
54 import org.openecomp.aai.introspection.LoaderFactory;
55 import org.openecomp.aai.introspection.ModelType;
56 import org.openecomp.aai.introspection.exceptions.AAIUnknownObjectException;
57 import org.openecomp.aai.logging.ErrorLogHelper;
58 import org.openecomp.aai.serialization.db.EdgeProperties;
59 import org.openecomp.aai.serialization.db.EdgeProperty;
60 import org.openecomp.aai.util.AAIConfig;
61 import org.openecomp.aai.util.AAIConstants;
62 import org.openecomp.aai.util.FormatDate;
64 import com.att.eelf.configuration.Configuration;
65 import com.att.eelf.configuration.EELFLogger;
66 import com.att.eelf.configuration.EELFManager;
67 import com.thinkaurelius.titan.core.TitanEdge;
68 import com.thinkaurelius.titan.core.TitanFactory;
69 import com.thinkaurelius.titan.core.TitanGraph;
70 import com.thinkaurelius.titan.core.TitanTransaction;
73 public class DataGrooming {
75 private static EELFLogger LOGGER;
76 private static final String FROMAPPID = "AAI-DB";
77 private static final String TRANSID = UUID.randomUUID().toString();
78 private static int dupeGrpsDeleted = 0;
83 * @param args the arguments
85 public static void main(String[] args) {
87 // Set the logging file properties to be used by EELFManager
88 Properties props = System.getProperties();
89 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS);
90 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
91 LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
92 String ver = "version"; // Placeholder
93 Boolean doAutoFix = false;
94 Boolean edgesOnlyFlag = false;
95 Boolean dontFixOrphansFlag = false;
96 Boolean skipHostCheck = false;
97 Boolean singleCommits = false;
98 Boolean dupeCheckOff = false;
99 Boolean dupeFixOn = false;
100 Boolean ghost2CheckOff = false;
101 Boolean ghost2FixOn = false;
102 Boolean neverUseCache = false;
104 int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
105 int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
107 String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
108 if( maxFixStr != null && !maxFixStr.equals("") ){
109 maxRecordsToFix = Integer.parseInt(maxFixStr);
111 String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
112 if( sleepStr != null && !sleepStr.equals("") ){
113 sleepMinutes = Integer.parseInt(sleepStr);
116 catch ( Exception e ){
117 // Don't worry, we'll just use the defaults that we got from AAIConstants
118 LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
121 String prevFileName = "";
123 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
124 String dteStr = fd.getDateTime();
125 String groomOutFileName = "dataGrooming." + dteStr + ".out";
127 if (args.length > 0) {
128 // They passed some arguments in that will affect processing
129 for (int i = 0; i < args.length; i++) {
130 String thisArg = args[i];
131 if (thisArg.equals("-edgesOnly")) {
132 edgesOnlyFlag = true;
133 } else if (thisArg.equals("-autoFix")) {
135 } else if (thisArg.equals("-skipHostCheck")) {
136 skipHostCheck = true;
137 } else if (thisArg.equals("-dontFixOrphans")) {
138 dontFixOrphansFlag = true;
139 } else if (thisArg.equals("-singleCommits")) {
140 singleCommits = true;
141 } else if (thisArg.equals("-dupeCheckOff")) {
143 } else if (thisArg.equals("-dupeFixOn")) {
145 } else if (thisArg.equals("-ghost2CheckOff")) {
146 ghost2CheckOff = true;
147 } else if (thisArg.equals("-neverUseCache")) {
148 neverUseCache = true;
149 } else if (thisArg.equals("-ghost2FixOn")) {
151 } else if (thisArg.equals("-maxFix")) {
153 if (i >= args.length) {
154 LOGGER.error(" No value passed with -maxFix option. ");
157 String nextArg = args[i];
159 maxRecordsToFix = Integer.parseInt(nextArg);
160 } catch (Exception e) {
161 LOGGER.error("Bad value passed with -maxFix option: ["
165 } else if (thisArg.equals("-sleepMinutes")) {
167 if (i >= args.length) {
168 LOGGER.error("No value passed with -sleepMinutes option.");
171 String nextArg = args[i];
173 sleepMinutes = Integer.parseInt(nextArg);
174 } catch (Exception e) {
175 LOGGER.error("Bad value passed with -sleepMinutes option: ["
179 } else if (thisArg.equals("-f")) {
181 if (i >= args.length) {
182 LOGGER.error(" No value passed with -f option. ");
185 prevFileName = args[i];
187 LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
189 LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache");
197 LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
200 catch (Exception ex){
201 LOGGER.error("ERROR - Could not create loader", ex);
207 if (!prevFileName.equals("")) {
208 // They are trying to fix some data based on a data in a
210 LOGGER.info(" Call doTheGrooming() with a previous fileName ["
211 + prevFileName + "] for cleanup. ");
212 Boolean finalShutdownFlag = true;
213 Boolean cacheDbOkFlag = false;
214 doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
215 maxRecordsToFix, groomOutFileName, ver, singleCommits,
216 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
217 finalShutdownFlag, cacheDbOkFlag);
218 } else if (doAutoFix) {
219 // They want us to run the processing twice -- first to look for
220 // delete candidates, then after
221 // napping for a while, run it again and delete any candidates
222 // that were found by the first run.
223 // Note: we will produce a separate output file for each of the
225 LOGGER.info(" Doing an auto-fix call to Grooming. ");
226 LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
227 Boolean finalShutdownFlag = false;
228 Boolean cacheDbOkFlag = true;
229 int fixCandCount = doTheGrooming("", edgesOnlyFlag,
230 dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
231 ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
232 finalShutdownFlag, cacheDbOkFlag);
233 if (fixCandCount == 0) {
234 LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
236 // We'll sleep a little and then run a fix-pass based on the
237 // first-run's output file.
239 LOGGER.info("About to sleep for " + sleepMinutes
241 int sleepMsec = sleepMinutes * 60 * 1000;
242 Thread.sleep(sleepMsec);
243 } catch (InterruptedException ie) {
244 LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
248 dteStr = fd.getDateTime();
249 String secondGroomOutFileName = "dataGrooming." + dteStr
251 LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
252 + "generated by the first pass for fixing: ["
253 + groomOutFileName + "]");
254 finalShutdownFlag = true;
255 cacheDbOkFlag = false;
256 doTheGrooming(groomOutFileName, edgesOnlyFlag,
257 dontFixOrphansFlag, maxRecordsToFix,
258 secondGroomOutFileName, ver, singleCommits,
259 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
260 finalShutdownFlag, cacheDbOkFlag);
263 // Do the grooming - plain vanilla (no fix-it-file, no
265 Boolean finalShutdownFlag = true;
266 LOGGER.info(" Call doTheGrooming() ");
267 Boolean cacheDbOkFlag = true;
269 // They have forbidden us from using a cached db connection.
270 cacheDbOkFlag = false;
272 doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
273 maxRecordsToFix, groomOutFileName, ver, singleCommits,
274 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
275 finalShutdownFlag, cacheDbOkFlag);
277 } catch (Exception ex) {
278 LOGGER.error("Exception while grooming data", ex);
281 LOGGER.info(" Done! ");
289 * @param fileNameForFixing the file name for fixing
290 * @param edgesOnlyFlag the edges only flag
291 * @param dontFixOrphansFlag the dont fix orphans flag
292 * @param maxRecordsToFix the max records to fix
293 * @param groomOutFileName the groom out file name
294 * @param version the version
295 * @param singleCommits the single commits
296 * @param dupeCheckOff the dupe check off
297 * @param dupeFixOn the dupe fix on
298 * @param ghost2CheckOff the ghost 2 check off
299 * @param ghost2FixOn the ghost 2 fix on
300 * @param finalShutdownFlag the final shutdown flag
301 * @param cacheDbOkFlag the cacheDbOk flag
304 private static int doTheGrooming(String fileNameForFixing,
305 Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
306 int maxRecordsToFix, String groomOutFileName, String version,
307 Boolean singleCommits,
308 Boolean dupeCheckOff, Boolean dupeFixOn,
309 Boolean ghost2CheckOff, Boolean ghost2FixOn,
310 Boolean finalShutdownFlag, Boolean cacheDbOkFlag) {
312 LOGGER.debug(" Entering doTheGrooming \n");
314 int cleanupCandidateCount = 0;
315 BufferedWriter bw = null;
316 TitanGraph graph = null;
317 TitanGraph graph2 = null;
319 boolean executeFinalCommit = false;
320 Set<String> deleteCandidateList = new LinkedHashSet<>();
321 Set<String> processedVertices = new LinkedHashSet<>();
322 TitanTransaction g = null;
323 TitanTransaction g2 = null;
326 String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
327 + "logs" + AAIConstants.AAI_FILESEP + "data"
328 + AAIConstants.AAI_FILESEP + "dataGrooming";
330 // Make sure the target directory exists
331 new File(targetDir).mkdirs();
333 if (!fileNameForFixing.equals("")) {
334 deleteCandidateList = getDeleteList(targetDir,
335 fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
339 if (deleteCandidateList.size() > maxRecordsToFix) {
340 LOGGER.warn(" >> WARNING >> Delete candidate list size ("
341 + deleteCandidateList.size()
342 + ") is too big. The maxFix we are using is: "
344 + ". No candidates will be deleted. ");
345 // Clear out the list so it won't be processed below.
346 deleteCandidateList = new LinkedHashSet<>();
349 String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
351 File groomOutFile = new File(fullOutputFileName);
353 groomOutFile.createNewFile();
354 } catch (IOException e) {
355 String emsg = " Problem creating output file ["
356 + fullOutputFileName + "], exception=" + e.getMessage();
357 throw new AAIException("AAI_6124", emsg);
360 LOGGER.info(" Will write to " + fullOutputFileName );
361 FileWriter fw = new FileWriter(groomOutFile.getAbsoluteFile());
362 bw = new BufferedWriter(fw);
363 ErrorLogHelper.loadProperties();
365 LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n");
368 // Since we're just reading (not deleting/fixing anything), we can use
369 // a cached connection to the DB
370 graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG);
373 graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
376 String emsg = "null graph object in DataGrooming\n";
377 throw new AAIException("AAI_6101", emsg);
380 LOGGER.debug(" Got the graph object. ");
382 g = graph.newTransaction();
384 String emsg = "null graphTransaction object in DataGrooming\n";
385 throw new AAIException("AAI_6101", emsg);
387 GraphTraversalSource source1 = g.traversal();
389 ArrayList<String> errArr = new ArrayList<>();
390 int totalNodeCount = 0;
391 HashMap<String, String> misMatchedHash = new HashMap<String, String>();
392 HashMap<String, Vertex> orphanNodeHash = new HashMap<String, Vertex>();
393 HashMap<String, Vertex> missingDepNodeHash = new HashMap<String, Vertex>();
394 HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
395 HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
396 HashMap<String, Vertex> ghostNodeHash = new HashMap<String, Vertex>();
397 ArrayList<String> dupeGroups = new ArrayList<>();
399 Loader loader = LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
401 Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
404 LOGGER.info(" Starting DataGrooming Processing ");
407 LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
410 for (Entry<String, Introspector> entry : entrySet) {
411 String nType = entry.getKey();
413 int thisNtDeleteCount = 0;
414 LOGGER.debug(" > Look at : [" + nType + "] ...");
415 ntList = ntList + "," + nType;
417 // Get a collection of the names of the key properties for this nodeType to use later
418 // Determine what the key fields are for this nodeType
419 Collection <String> keyProps = entry.getValue().getKeys();
421 // Get the types of nodes that this nodetype depends on for uniqueness (if any)
422 Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn();
424 // Loop through all the nodes of this Node type
425 int lastShownForNt = 0;
426 ArrayList <Vertex> tmpList = new ArrayList <> ();
427 Iterator <Vertex> iterv = source1.V().has("aai-node-type",nType);
428 while (iterv.hasNext()) {
429 // We put the nodes into an ArrayList because the graph.query iterator can time out
430 tmpList.add(iterv.next());
433 Iterator <Vertex> iter = tmpList.iterator();
434 while (iter.hasNext()) {
437 if( thisNtCount == lastShownForNt + 250 ){
438 lastShownForNt = thisNtCount;
439 LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
441 Vertex thisVtx = iter.next();
442 String thisVid = thisVtx.id().toString();
443 if (processedVertices.contains(thisVid)) {
444 LOGGER.debug("skipping already processed vertex: " + thisVid);
448 List <Vertex> secondGetList = new ArrayList <> ();
449 // -----------------------------------------------------------------------
450 // For each vertex of this nodeType, we want to:
451 // a) make sure that it can be retrieved using it's AAI defined key
452 // b) make sure that it is not a duplicate
453 // -----------------------------------------------------------------------
455 // For this instance of this nodeType, get the key properties
456 HashMap<String, Object> propHashWithKeys = new HashMap<>();
457 Iterator<String> keyPropI = keyProps.iterator();
458 while (keyPropI.hasNext()) {
459 String propName = keyPropI.next();
461 //delete an already deleted vertex
462 Object obj = thisVtx.<Object>property(propName).orElse(null);
464 propVal = obj.toString();
466 propHashWithKeys.put(propName, propVal);
469 // If this node is dependent on another for uniqueness, then do the query from that parent node
470 // Note - all of our nodes that are dependent on others for uniqueness are
471 // "children" of that node.
472 boolean depNodeOk = true;
473 if( depNodeTypes.isEmpty() ){
474 // This kind of node is not dependent on any other.
475 // Make sure we can get it back using it's key properties and that we only get one.
476 secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType,
477 propHashWithKeys, version );
480 // This kind of node is dependent on another for uniqueness.
481 // Start at it's parent (the dependent vertex) and make sure we can get it
482 // back using it's key properties and that we only get one.
483 Iterator <Vertex> vertI2 = source1.V(thisVtx).union(__.inE().has(EdgeProperties.out(EdgeProperty.IS_PARENT), true).outV(), __.outE().has(EdgeProperties.in(EdgeProperty.IS_PARENT)).inV());
484 Vertex parentVtx = null;
486 while( vertI2 != null && vertI2.hasNext() ){
487 parentVtx = vertI2.next();
493 //List<Vertex> vertI2 = g.traversal().V(thisVtx).union(__.outE().has("isParent-REV",true).outV(),__.inE().has("isParent",true).inV()).toList();
494 //if( vertI2.isEmpty()){
496 // It's Missing it's dependent/parent node
498 boolean zeroEdges = false;
500 Iterator<Edge> tmpEdgeIter = thisVtx.edges(Direction.BOTH);
502 while( tmpEdgeIter.hasNext() ){
506 if( edgeCount == 0 ){
509 } catch (Exception ex) {
510 LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex);
513 if (deleteCandidateList.contains(thisVid)) {
514 boolean okFlag = true;
516 processedVertices.add(thisVtx.id().toString());
520 } catch (Exception e) {
522 LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e);
525 LOGGER.info(" DELETED missing-dep-node VID = " + thisVid);
528 // We count nodes missing their depNodes two ways - the first if it has
529 // at least some edges, and the second if it has zero edges. Either
530 // way, they are effectively orphaned.
531 // NOTE - Only nodes that have dependent nodes are ever considered "orphaned".
533 missingDepNodeHash.put(thisVid, thisVtx);
536 orphanNodeHash.put(thisVid, thisVtx);
540 else if ( pCount > 1 ){
541 // Not sure how this could happen? Should we do something here?
545 // We found the parent - so use it to do the second-look.
546 // NOTE --- We're just going to do the same check from the other direction - because
547 // there could be duplicates or the pointer going the other way could be broken
548 ArrayList <Vertex> tmpListSec = new ArrayList <> ();
550 tmpListSec = getConnectedChildrenOfOneType( source1, parentVtx, nType ) ;
551 Iterator<Vertex> vIter = tmpListSec.iterator();
552 while (vIter.hasNext()) {
553 Vertex tmpV = vIter.next();
554 if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
555 secondGetList.add(tmpV);
561 if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
562 // We could not get the node back using it's own key info.
563 // So, it's a PHANTOM
564 if (deleteCandidateList.contains(thisVid)) {
565 boolean okFlag = true;
570 } catch (Exception e) {
572 LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e);
575 LOGGER.info(" DELETED VID = " + thisVid);
578 ghostNodeHash.put(thisVid, thisVtx);
581 else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
582 // Found some DUPLICATES - need to process them
583 LOGGER.info(" - now check Dupes for this guy - ");
584 List<String> tmpDupeGroups = checkAndProcessDupes(
585 TRANSID, FROMAPPID, g, source1, version,
586 nType, secondGetList, dupeFixOn,
587 deleteCandidateList, singleCommits, dupeGroups, loader);
588 Iterator<String> dIter = tmpDupeGroups.iterator();
589 while (dIter.hasNext()) {
590 // Add in any newly found dupes to our running list
591 String tmpGrp = dIter.next();
592 LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
593 dupeGroups.add(tmpGrp);
597 catch (AAIException e1) {
598 LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
599 errArr.add(e1.getErrorObject().toString());
601 catch (Exception e2) {
602 LOGGER.warn(" For nodeType = " + nType
603 + " Caught exception", e2);
604 errArr.add(e2.getMessage());
606 }// try block to enclose looping of a single vertex
607 catch (Exception exx) {
608 LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
611 } // while loop for each record of a nodeType
613 if ( (thisNtDeleteCount > 0) && singleCommits ) {
614 // NOTE - the singleCommits option is not used in normal processing
616 g = AAIGraph.getInstance().getGraph().newTransaction();
619 thisNtDeleteCount = 0;
620 LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " );
622 }// While-loop for each node type
623 }// end of check to make sure we weren't only supposed to do edges
626 // --------------------------------------------------------------------------------------
627 // Now, we're going to look for one-armed-edges. Ie. an edge that
629 // been deleted (because a vertex on one side was deleted) but
630 // somehow was not deleted.
631 // So the one end of it points to a vertexId -- but that vertex is
633 // --------------------------------------------------------------------------------------
635 // To do some strange checking - we need a second graph object
636 LOGGER.debug(" ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
637 // Note - graph2 just reads - but we want it to use a fresh connection to
638 // the database, so we are NOT using the CACHED DB CONFIG here.
639 graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
640 if (graph2 == null) {
641 String emsg = "null graph2 object in DataGrooming\n";
642 throw new AAIException("AAI_6101", emsg);
644 LOGGER.debug("Got the graph2 object... \n");
646 g2 = graph2.newTransaction();
648 String emsg = "null graphTransaction2 object in DataGrooming\n";
649 throw new AAIException("AAI_6101", emsg);
652 ArrayList<Vertex> vertList = new ArrayList<>();
653 Iterable<? extends Vertex> vIt3 = g.query().vertices();
654 Iterator<? extends Vertex> vItor3 = vIt3.iterator();
655 // Gotta hold these in a List - or else HBase times out as you cycle
657 while (vItor3.hasNext()) {
658 Vertex v = vItor3.next();
663 Iterator<Vertex> vItor2 = vertList.iterator();
664 LOGGER.info(" Checking for bad edges --- ");
666 while (vItor2.hasNext()) {
671 } catch (Exception vex) {
672 LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
677 String thisVertId = "";
679 thisVertId = v.id().toString();
680 } catch (Exception ev) {
681 LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. ");
684 if (ghostNodeHash.containsKey(thisVertId)) {
685 // This is a phantom node, so don't try to use it
686 LOGGER.info(" >> Skipping edge check for edges from vertexId = "
688 + ", since that guy is a Phantom Node");
691 if (counter == lastShown + 250) {
693 LOGGER.info("... Checking edges for vertex # "
696 Iterator<Edge> eItor = v.edges(Direction.BOTH);
697 while (eItor.hasNext()) {
703 } catch (Exception iex) {
704 LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
710 } catch (Exception err) {
711 LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
715 Vertex ghost2 = null;
717 Boolean keysMissing = true;
718 Boolean cantGetUsingVid = false;
721 Object ob = vIn.<Object>property("aai-node-type").orElse(null);
723 vNtI = ob.toString();
724 keysMissing = anyKeyFieldsMissing(vNtI, vIn, loader);
729 vIdI = ob.toString();
730 vIdLong = Long.parseLong(vIdI);
733 if( ! ghost2CheckOff ){
734 Vertex connectedVert = g2.getVertex(vIdLong);
735 if( connectedVert == null ) {
736 LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
737 cantGetUsingVid = true;
739 // If we can NOT get this ghost with the SECOND graph-object,
740 // it is still a ghost since even though we can get data about it using the FIRST graph
743 ghost2 = g.getVertex(vIdLong);
745 catch( Exception ex){
746 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
748 if( ghost2 != null ){
749 ghostNodeHash.put(vIdI, ghost2);
752 }// end of the ghost2 checking
754 catch (Exception err) {
755 LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
758 if (keysMissing || vIn == null || vNtI.equals("")
759 || cantGetUsingVid) {
760 // this is a bad edge because it points to a vertex
761 // that isn't there anymore or is corrupted
762 String thisEid = e.id().toString();
763 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
764 boolean okFlag = true;
765 if (!vIdI.equals("")) {
766 // try to get rid of the corrupted vertex
768 if( (ghost2 != null) && ghost2FixOn ){
775 // NOTE - the singleCommits option is not used in normal processing
777 g = AAIGraph.getInstance().getGraph().newTransaction();
780 } catch (Exception e1) {
782 LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
786 LOGGER.info(" DELETED vertex from bad edge = "
790 // remove the edge if we couldn't get the
795 // NOTE - the singleCommits option is not used in normal processing
797 g = AAIGraph.getInstance().getGraph().newTransaction();
800 } catch (Exception ex) {
801 // NOTE - often, the exception is just
802 // that this edge has already been
805 LOGGER.warn("WARNING when trying to delete edge = "
809 LOGGER.info(" DELETED edge = " + thisEid);
813 oneArmedEdgeHash.put(thisEid, e);
814 if ((vIn != null) && (vIn.id() != null)) {
815 emptyVertexHash.put(thisEid, vIn.id()
822 vOut = e.outVertex();
823 } catch (Exception err) {
824 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
830 cantGetUsingVid = false;
833 Object ob = vOut.<Object>property("aai-node-type").orElse(null);
835 vNtO = ob.toString();
836 keysMissing = anyKeyFieldsMissing(vNtO,
842 vIdO = ob.toString();
843 vIdLong = Long.parseLong(vIdO);
846 if( ! ghost2CheckOff ){
847 Vertex connectedVert = g2.getVertex(vIdLong);
848 if( connectedVert == null ) {
849 cantGetUsingVid = true;
850 LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
851 // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
853 ghost2 = g.getVertex(vIdLong);
855 catch( Exception ex){
856 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
858 if( ghost2 != null ){
859 ghostNodeHash.put(vIdO, ghost2);
863 } catch (Exception err) {
864 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
867 if (keysMissing || vOut == null || vNtO.equals("")
868 || cantGetUsingVid) {
869 // this is a bad edge because it points to a vertex
870 // that isn't there anymore
871 String thisEid = e.id().toString();
872 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
873 boolean okFlag = true;
874 if (!vIdO.equals("")) {
875 // try to get rid of the corrupted vertex
877 if( (ghost2 != null) && ghost2FixOn ){
884 // NOTE - the singleCommits option is not used in normal processing
886 g = AAIGraph.getInstance().getGraph().newTransaction();
889 } catch (Exception e1) {
891 LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
895 LOGGER.info(" DELETED vertex from bad edge = "
899 // remove the edge if we couldn't get the
904 // NOTE - the singleCommits option is not used in normal processing
906 g = AAIGraph.getInstance().getGraph().newTransaction();
909 } catch (Exception ex) {
910 // NOTE - often, the exception is just
911 // that this edge has already been
914 LOGGER.warn("WARNING when trying to delete edge = "
918 LOGGER.info(" DELETED edge = " + thisEid);
922 oneArmedEdgeHash.put(thisEid, e);
923 if ((vOut != null) && (vOut.id() != null)) {
924 emptyVertexHash.put(thisEid, vOut.id()
929 }// End of while-edges-loop
930 } catch (Exception exx) {
931 LOGGER.warn("WARNING from in the while-verts-loop ", exx);
933 }// End of while-vertices-loop
935 deleteCount = deleteCount + dupeGrpsDeleted;
936 if (!singleCommits && deleteCount > 0) {
938 LOGGER.info("About to do the commit for "
939 + deleteCount + " removes. ");
940 executeFinalCommit = true;
941 LOGGER.info("Commit was successful ");
942 } catch (Exception excom) {
943 LOGGER.error(" >>>> ERROR <<<< Could not commit changes. ", excom);
948 int ghostNodeCount = ghostNodeHash.size();
949 int orphanNodeCount = orphanNodeHash.size();
950 int missingDepNodeCount = missingDepNodeHash.size();
951 int oneArmedEdgeCount = oneArmedEdgeHash.size();
952 int dupeCount = dupeGroups.size();
954 deleteCount = deleteCount + dupeGrpsDeleted;
956 bw.write("\n\n ============ Summary ==============\n");
957 bw.write("Ran these nodeTypes: " + ntList + "\n\n");
958 bw.write("There were this many delete candidates from previous run = "
959 + deleteCandidateList.size() + "\n");
960 if (dontFixOrphansFlag) {
961 bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
963 bw.write("Deleted this many delete candidates = " + deleteCount
965 bw.write("Total number of nodes looked at = " + totalNodeCount
967 bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
968 bw.write("Orphan Nodes identified = " + orphanNodeCount + "\n");
969 bw.write("Bad Edges identified = " + oneArmedEdgeCount + "\n");
970 bw.write("Missing Dependent Edge (but not orphaned) node count = "
971 + missingDepNodeCount + "\n");
972 bw.write("Duplicate Groups count = " + dupeCount + "\n");
973 bw.write("MisMatching Label/aai-node-type count = "
974 + misMatchedHash.size() + "\n");
976 bw.write("\n ------------- Delete Candidates ---------\n");
977 for (Map.Entry<String, Vertex> entry : ghostNodeHash
979 String vid = entry.getKey();
980 bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
981 cleanupCandidateCount++;
983 for (Map.Entry<String, Vertex> entry : orphanNodeHash
985 String vid = entry.getKey();
986 bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
987 if (!dontFixOrphansFlag) {
988 cleanupCandidateCount++;
991 for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
992 String eid = entry.getKey();
993 bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
994 cleanupCandidateCount++;
996 for (Map.Entry<String, Vertex> entry : missingDepNodeHash
998 String vid = entry.getKey();
999 bw.write("DeleteCandidate: (maybe) missingDepNode Vid = ["
1001 cleanupCandidateCount++;
1003 bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
1005 bw.write("\n ------------- GHOST NODES - detail ");
1006 for (Map.Entry<String, Vertex> entry : ghostNodeHash
1009 String vid = entry.getKey();
1010 bw.write("\n ==> Phantom Vid = " + vid + "\n");
1011 ArrayList<String> retArr = showPropertiesForNode(
1012 TRANSID, FROMAPPID, entry.getValue());
1013 for (String info : retArr) {
1014 bw.write(info + "\n");
1017 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1019 for (String info : retArr) {
1020 bw.write(info + "\n");
1022 } catch (Exception dex) {
1023 LOGGER.error("error trying to print detail info for a ghost-node: ", dex);
1027 bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
1028 for (Map.Entry<String, Vertex> entry : orphanNodeHash
1031 String vid = entry.getKey();
1032 bw.write("\n> Orphan Node Vid = " + vid + "\n");
1033 ArrayList<String> retArr = showPropertiesForNode(
1034 TRANSID, FROMAPPID, entry.getValue());
1035 for (String info : retArr) {
1036 bw.write(info + "\n");
1039 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1041 for (String info : retArr) {
1042 bw.write(info + "\n");
1044 } catch (Exception dex) {
1045 LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex);
1049 bw.write("\n ------------- Missing Dependent Edge (but not orphan) NODES: ");
1050 for (Map.Entry<String, Vertex> entry : missingDepNodeHash
1053 String vid = entry.getKey();
1054 bw.write("\n> Missing edge to Dependent Node (but has edges) Vid = "
1056 ArrayList<String> retArr = showPropertiesForNode(
1057 TRANSID, FROMAPPID, entry.getValue());
1058 for (String info : retArr) {
1059 bw.write(info + "\n");
1062 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1064 for (String info : retArr) {
1065 bw.write(info + "\n");
1067 } catch (Exception dex) {
1068 LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex);
1072 bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
1073 for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1075 String eid = entry.getKey();
1076 Edge thisE = entry.getValue();
1077 String badVid = emptyVertexHash.get(eid);
1078 bw.write("\n> Edge pointing to bad vertex (Vid = "
1079 + badVid + ") EdgeId = " + eid + "\n");
1080 bw.write("Label: [" + thisE.label() + "]\n");
1081 Iterator<Property<Object>> pI = thisE.properties();
1082 while (pI.hasNext()) {
1083 Property<Object> propKey = pI.next();
1084 bw.write("Prop: [" + propKey + "], val = ["
1085 + propKey.value() + "]\n");
1087 } catch (Exception pex) {
1088 LOGGER.error("error trying to print empty/bad vertex data: ", pex);
1092 bw.write("\n ------------- Duplicates: ");
1093 Iterator<String> dupeIter = dupeGroups.iterator();
1094 int dupeSetCounter = 0;
1095 while (dupeIter.hasNext()) {
1097 String dset = (String) dupeIter.next();
1099 bw.write("\n --- Duplicate Group # " + dupeSetCounter
1100 + " Detail -----------\n");
1102 // We expect each line to have at least two vid's, followed
1103 // by the preferred one to KEEP
1104 String[] dupeArr = dset.split("\\|");
1105 ArrayList<String> idArr = new ArrayList<>();
1106 int lastIndex = dupeArr.length - 1;
1107 for (int i = 0; i <= lastIndex; i++) {
1108 if (i < lastIndex) {
1109 // This is not the last entry, it is one of the
1110 // dupes, so we want to show all its info
1111 bw.write(" >> Duplicate Group # "
1112 + dupeSetCounter + " Node # " + i
1114 String vidString = dupeArr[i];
1115 idArr.add(vidString);
1116 long longVertId = Long.parseLong(vidString);
1117 Iterator<Vertex> vtxIterator = g.vertices(longVertId);
1119 if (vtxIterator.hasNext()) {
1120 vtx = vtxIterator.next();
1122 ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
1123 for (String info : retArr) {
1124 bw.write(info + "\n");
1127 retArr = showAllEdgesForNode(TRANSID,
1129 for (String info : retArr) {
1130 bw.write(info + "\n");
1133 // This is the last entry which should tell us if we
1134 // have a preferred keeper
1135 String prefString = dupeArr[i];
1136 if (prefString.equals("KeepVid=UNDETERMINED")) {
1137 bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
1138 bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
1140 // If we know which to keep, then the prefString
1141 // should look like, "KeepVid=12345"
1142 String[] prefArr = prefString.split("=");
1143 if (prefArr.length != 2
1144 || (!prefArr[0].equals("KeepVid"))) {
1145 throw new Exception("Bad format. Expecting KeepVid=999999");
1147 String keepVidStr = prefArr[1];
1148 if (idArr.contains(keepVidStr)) {
1149 bw.write("\n The vertex we want to KEEP has vertexId = "
1151 bw.write("\n The others become delete candidates: \n");
1152 idArr.remove(keepVidStr);
1153 for (int x = 0; x < idArr.size(); x++) {
1154 cleanupCandidateCount++;
1155 bw.write("DeleteCandidate: Duplicate Vid = ["
1156 + idArr.get(x) + "]\n");
1159 throw new Exception("ERROR - Vertex Id to keep not found in list of dupes. dset = ["
1163 }// else we know which one to keep
1165 }// for each vertex in a group
1166 } catch (Exception dex) {
1167 LOGGER.error("error trying to print duplicate vertex data", dex);
1170 }// while - work on each group of dupes
1172 bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
1173 for (Map.Entry<String, String> entry : misMatchedHash.entrySet()) {
1174 String msg = entry.getValue();
1175 bw.write("MixedMsg = " + msg + "\n");
1178 bw.write("\n ------------- Got these errors while processing: \n");
1179 Iterator<String> errIter = errArr.iterator();
1180 while (errIter.hasNext()) {
1181 String line = (String) errIter.next();
1182 bw.write(line + "\n");
1187 LOGGER.info("\n ------------- Done doing all the checks ------------ ");
1188 LOGGER.info("Output will be written to " + fullOutputFileName);
1190 if (cleanupCandidateCount > 0) {
1191 // Technically, this is not an error -- but we're throwing this
1192 // error so that hopefully a
1193 // monitoring system will pick it up and do something with it.
1194 throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
1195 + "] and investigate delete candidates. ");
1197 } catch (AAIException e) {
1198 LOGGER.error("Caught AAIException while grooming data", e);
1199 ErrorLogHelper.logException(e);
1200 } catch (Exception ex) {
1201 LOGGER.error("Caught exception while grooming data", ex);
1202 ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
1208 } catch (IOException iox) {
1209 LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
1213 if (g != null && !g.isClosed()) {
1214 // Any changes that worked correctly should have already done
1217 if (executeFinalCommit) {
1221 } catch (Exception ex) {
1222 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1223 LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
1227 if (g2 != null && !g2.isClosed()) {
1228 // Any changes that worked correctly should have already done
1232 } catch (Exception ex) {
1233 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1234 LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
1238 if( finalShutdownFlag ){
1240 if( graph != null && graph.isOpen() ){
1244 } catch (Exception ex) {
1245 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1246 LOGGER.warn("WARNING from final graph.shutdown()", ex);
1250 if( graph2 != null && graph2.isOpen() ){
1251 graph2.tx().close();
1254 } catch (Exception ex) {
1255 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1256 LOGGER.warn("WARNING from final graph2.shutdown()", ex);
1262 return cleanupCandidateCount;
1264 }// end of doTheGrooming()
1268 * Vertex has these keys.
1270 * @param tmpV the tmp V
1271 * @param propHashWithKeys the prop hash with keys
1272 * @return the boolean
1274 private static Boolean vertexHasTheseKeys( Vertex tmpV, HashMap <String, Object> propHashWithKeys) {
1275 Iterator <?> it = propHashWithKeys.entrySet().iterator();
1276 while( it.hasNext() ){
1277 String propName = "";
1278 String propVal = "";
1279 Map.Entry <?,?>propEntry = (Map.Entry<?,?>)it.next();
1280 Object propNameObj = propEntry.getKey();
1281 if( propNameObj != null ){
1282 propName = propNameObj.toString();
1284 Object propValObj = propEntry.getValue();
1285 if( propValObj != null ){
1286 propVal = propValObj.toString();
1288 Object checkValObj = tmpV.<Object>property(propName).orElse(null);
1289 if( checkValObj == null ) {
1292 else if( !propVal.equals(checkValObj.toString()) ){
1301 * Any key fields missing.
1303 * @param nType the n type
1305 * @return the boolean
1307 private static Boolean anyKeyFieldsMissing(String nType, Vertex v, Loader loader) {
1310 Introspector obj = null;
1312 obj = loader.introspectorFromName(nType);
1313 } catch (AAIUnknownObjectException e) {
1314 // They gave us a non-empty nodeType but our NodeKeyProps does
1315 // not have data for it. Since we do not know what the
1316 // key params are for this type of node, we will just
1318 String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
1319 + "]. We cannot determine required keys for this nType. ";
1320 // NOTE - this will be caught below and a "false" returned
1321 throw new AAIException("AAI_6121", emsg);
1324 // Determine what the key fields are for this nodeType
1325 Collection <String> keyPropNamesColl = obj.getKeys();
1326 Iterator<String> keyPropI = keyPropNamesColl.iterator();
1327 while (keyPropI.hasNext()) {
1328 String propName = keyPropI.next();
1329 Object ob = v.<Object>property(propName).orElse(null);
1330 if (ob == null || ob.toString().equals("")) {
1331 // It is missing a key property
1335 } catch (AAIException e) {
1336 // Something was wrong -- but since we weren't able to check
1337 // the keys, we will not declare that it is missing keys.
1345 * Gets the delete list.
1347 * @param targetDir the target dir
1348 * @param fileName the file name
1349 * @param edgesOnlyFlag the edges only flag
1350 * @param dontFixOrphans the dont fix orphans
1351 * @param dupeFixOn the dupe fix on
1352 * @return the delete list
1353 * @throws AAIException the AAI exception
1355 private static Set<String> getDeleteList(String targetDir,
1356 String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
1357 Boolean dupeFixOn) throws AAIException {
1359 // Look in the file for lines formated like we expect - pull out any
1360 // Vertex Id's to delete on this run
1361 Set<String> delList = new LinkedHashSet<>();
1362 String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
1363 BufferedReader br = null;
1366 br = new BufferedReader(new FileReader(fullFileName));
1367 String line = br.readLine();
1368 while (line != null) {
1369 if (!line.equals("") && line.startsWith("DeleteCandidate")) {
1370 if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
1371 // We're not going to process edge guys
1372 } else if (dontFixOrphans && line.contains("Orphan")) {
1373 // We're not going to process orphans
1374 } else if (!dupeFixOn && line.contains("Duplicate")) {
1375 // We're not going to process Duplicates
1377 int begIndex = line.indexOf("id = ");
1378 int endIndex = line.indexOf("]");
1379 String vidVal = line.substring(begIndex + 6, endIndex);
1380 delList.add(vidVal);
1383 line = br.readLine();
1386 } catch (IOException e) {
1387 throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
1388 + "], exception= " + e.getMessage());
1393 }// end of getDeleteList
1396 * Gets the preferred dupe.
1398 * @param transId the trans id
1399 * @param fromAppId the from app id
1401 * @param dupeVertexList the dupe vertex list
1402 * @param ver the ver
1404 * @throws AAIException the AAI exception
1406 public static Vertex getPreferredDupe(String transId,
1407 String fromAppId, GraphTraversalSource g,
1408 ArrayList<Vertex> dupeVertexList, String ver, Loader loader)
1409 throws AAIException {
1411 // This method assumes that it is being passed a List of vertex objects
1413 // violate our uniqueness constraints.
1415 Vertex nullVtx = null;
1417 if (dupeVertexList == null) {
1420 int listSize = dupeVertexList.size();
1421 if (listSize == 0) {
1424 if (listSize == 1) {
1425 return (dupeVertexList.get(0));
1428 Vertex vtxPreferred = null;
1429 Vertex currentFaveVtx = dupeVertexList.get(0);
1430 for (int i = 1; i < listSize; i++) {
1431 Vertex vtxB = dupeVertexList.get(i);
1432 vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
1433 currentFaveVtx, vtxB, ver, loader);
1434 if (vtxPreferred == null) {
1435 // We couldn't choose one
1438 currentFaveVtx = vtxPreferred;
1442 return (currentFaveVtx);
1444 } // end of getPreferredDupe()
1447 * Pick one of two dupes.
1449 * @param transId the trans id
1450 * @param fromAppId the from app id
1452 * @param vtxA the vtx A
1453 * @param vtxB the vtx B
1454 * @param ver the ver
1456 * @throws AAIException the AAI exception
1458 public static Vertex pickOneOfTwoDupes(String transId,
1459 String fromAppId, GraphTraversalSource g, Vertex vtxA,
1460 Vertex vtxB, String ver, Loader loader) throws AAIException {
1462 Vertex nullVtx = null;
1463 Vertex preferredVtx = null;
1465 Long vidA = new Long(vtxA.id().toString());
1466 Long vidB = new Long(vtxB.id().toString());
1468 String vtxANodeType = "";
1469 String vtxBNodeType = "";
1470 Object objType = vtxA.<Object>property("aai-node-type").orElse(null);
1471 if (objType != null) {
1472 vtxANodeType = objType.toString();
1474 objType = vtxB.<Object>property("aai-node-type").orElse(null);
1475 if (objType != null) {
1476 vtxBNodeType = objType.toString();
1479 if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
1480 // Either they're not really dupes or there's some bad data - so
1485 // Check that node A and B both have the same key values (or else they
1487 // (We'll check dep-node later)
1488 // Determine what the key fields are for this nodeType
1489 Collection <String> keyProps = new ArrayList <>();
1491 keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
1492 } catch (AAIUnknownObjectException e) {
1493 throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
1496 Iterator<String> keyPropI = keyProps.iterator();
1497 while (keyPropI.hasNext()) {
1498 String propName = keyPropI.next();
1499 String vtxAKeyPropVal = "";
1500 objType = vtxA.<Object>property(propName).orElse(null);
1501 if (objType != null) {
1502 vtxAKeyPropVal = objType.toString();
1504 String vtxBKeyPropVal = "";
1505 objType = vtxB.<Object>property(propName).orElse(null);
1506 if (objType != null) {
1507 vtxBKeyPropVal = objType.toString();
1510 if (vtxAKeyPropVal.equals("")
1511 || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
1512 // Either they're not really dupes or they are missing some key
1513 // data - so don't pick one
1518 // Collect the vid's and aai-node-types of the vertices that each vertex
1519 // (A and B) is connected to.
1520 ArrayList<String> vtxIdsConn2A = new ArrayList<>();
1521 ArrayList<String> vtxIdsConn2B = new ArrayList<>();
1522 HashMap<String, String> nodeTypesConn2A = new HashMap<>();
1523 HashMap<String, String> nodeTypesConn2B = new HashMap<>();
1525 ArrayList<Vertex> vertListA = getConnectedNodes( g, vtxA );
1526 if (vertListA != null) {
1527 Iterator<Vertex> iter = vertListA.iterator();
1528 while (iter.hasNext()) {
1529 Vertex tvCon = iter.next();
1530 String conVid = tvCon.id().toString();
1532 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1533 if (objType != null) {
1534 nt = objType.toString();
1536 nodeTypesConn2A.put(nt, conVid);
1537 vtxIdsConn2A.add(conVid);
1541 ArrayList<Vertex> vertListB = getConnectedNodes( g, vtxB );
1542 if (vertListB != null) {
1543 Iterator<Vertex> iter = vertListB.iterator();
1544 while (iter.hasNext()) {
1545 Vertex tvCon = iter.next();
1546 String conVid = tvCon.id().toString();
1548 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1549 if (objType != null) {
1550 nt = objType.toString();
1552 nodeTypesConn2B.put(nt, conVid);
1553 vtxIdsConn2B.add(conVid);
1557 // 1 - If this kind of node needs a dependent node for uniqueness, then
1558 // verify that they both nodes
1559 // point to the same dependent node (otherwise they're not really
1561 // Note - there are sometimes more than one dependent node type since
1562 // one nodeType can be used in
1563 // different ways. But for a particular node, it will only have one
1564 // dependent node that it's
1566 Collection <String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
1568 if (depNodeTypes.isEmpty()) {
1569 // This kind of node is not dependent on any other. That is ok.
1571 String depNodeVtxId4A = "";
1572 String depNodeVtxId4B = "";
1573 Iterator<String> iter = depNodeTypes.iterator();
1574 while (iter.hasNext()) {
1575 String depNodeType = iter.next();
1576 if (nodeTypesConn2A.containsKey(depNodeType)) {
1577 // This is the dependent node type that vertex A is using
1578 depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
1580 if (nodeTypesConn2B.containsKey(depNodeType)) {
1581 // This is the dependent node type that vertex B is using
1582 depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
1585 if (depNodeVtxId4A.equals("")
1586 || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
1587 // Either they're not really dupes or there's some bad data - so
1588 // don't pick either one
1593 if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
1594 // 2 - If they both have edges to all the same vertices, then return
1595 // the one with the lower vertexId.
1596 boolean allTheSame = true;
1597 Iterator<String> iter = vtxIdsConn2A.iterator();
1598 while (iter.hasNext()) {
1599 String vtxIdConn2A = iter.next();
1600 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1608 preferredVtx = vtxA;
1610 preferredVtx = vtxB;
1613 } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
1614 // 3 - VertexA is connected to more things than vtxB.
1615 // We'll pick VtxA if its edges are a superset of vtxB's edges.
1616 boolean missingOne = false;
1617 Iterator<String> iter = vtxIdsConn2B.iterator();
1618 while (iter.hasNext()) {
1619 String vtxIdConn2B = iter.next();
1620 if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
1626 preferredVtx = vtxA;
1628 } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
1629 // 4 - VertexB is connected to more things than vtxA.
1630 // We'll pick VtxB if its edges are a superset of vtxA's edges.
1631 boolean missingOne = false;
1632 Iterator<String> iter = vtxIdsConn2A.iterator();
1633 while (iter.hasNext()) {
1634 String vtxIdConn2A = iter.next();
1635 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1641 preferredVtx = vtxB;
1644 preferredVtx = nullVtx;
1647 return (preferredVtx);
1649 } // end of pickOneOfTwoDupes()
1652 * Check and process dupes.
1654 * @param transId the trans id
1655 * @param fromAppId the from app id
1657 * @param version the version
1658 * @param nType the n type
1659 * @param passedVertList the passed vert list
1660 * @param dupeFixOn the dupe fix on
1661 * @param deleteCandidateList the delete candidate list
1662 * @param singleCommits the single commits
1663 * @param alreadyFoundDupeGroups the already found dupe groups
1664 * @param dbMaps the db maps
1665 * @return the array list
1667 private static List<String> checkAndProcessDupes(String transId,
1668 String fromAppId, TitanTransaction g, GraphTraversalSource source, String version, String nType,
1669 List<Vertex> passedVertList, Boolean dupeFixOn,
1670 Set<String> deleteCandidateList, Boolean singleCommits,
1671 ArrayList<String> alreadyFoundDupeGroups, Loader loader ) {
1673 ArrayList<String> returnList = new ArrayList<>();
1674 ArrayList<Vertex> checkVertList = new ArrayList<>();
1675 ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
1676 Boolean noFilterList = true;
1677 Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
1678 while (afItr.hasNext()) {
1679 String dupeGrpStr = afItr.next();
1680 String[] dupeArr = dupeGrpStr.split("\\|");
1681 int lastIndex = dupeArr.length - 1;
1682 for (int i = 0; i < lastIndex; i++) {
1683 // Note: we don't want the last one...
1684 String vidString = dupeArr[i];
1685 alreadyFoundDupeVidArr.add(vidString);
1686 noFilterList = false;
1690 // For a given set of Nodes that were found with a set of KEY
1691 // Parameters, (nodeType + key data) we will
1692 // see if we find any duplicate nodes that need to be cleaned up. Note -
1693 // it's legit to have more than one
1694 // node with the same key data if the nodes depend on a parent for
1695 // uniqueness -- as long as the two nodes
1696 // don't hang off the same Parent.
1697 // If we find duplicates, and we can figure out which of each set of
1698 // duplicates is the one that we
1699 // think should be preserved, we will record that. Whether we can tell
1700 // which one should be
1701 // preserved or not, we will return info about any sets of duplicates
1704 // Each element in the returned arrayList might look like this:
1705 // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
1706 // couldn't figure out which one to keep)
1707 // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
1708 // thought the third one was the one that should survive)
1710 // Because of the way the calling code loops over stuff, we can get the
1711 // same data multiple times - so we should
1712 // not process any vertices that we've already seen.
1715 Iterator<Vertex> pItr = passedVertList.iterator();
1716 while (pItr.hasNext()) {
1717 Vertex tvx = pItr.next();
1718 String passedId = tvx.id().toString();
1719 if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
1720 // We haven't seen this one before - so we should check it.
1721 checkVertList.add(tvx);
1725 if (checkVertList.size() < 2) {
1726 // Nothing new to check.
1730 if (loader.introspectorFromName(nType).isTopLevel()) {
1731 // If this was a node that does NOT depend on other nodes for
1732 // uniqueness, and we
1733 // found more than one node using its key -- record the found
1734 // vertices as duplicates.
1735 String dupesStr = "";
1736 for (int i = 0; i < checkVertList.size(); i++) {
1738 + ((checkVertList.get(i))).id()
1741 if (dupesStr != "") {
1742 Vertex prefV = getPreferredDupe(transId, fromAppId,
1743 source, checkVertList, version, loader);
1744 if (prefV == null) {
1745 // We could not determine which duplicate to keep
1746 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1747 returnList.add(dupesStr);
1749 dupesStr = dupesStr + "KeepVid=" + prefV.id();
1750 Boolean didRemove = false;
1752 didRemove = deleteNonKeepersIfAppropriate(g,
1753 dupesStr, prefV.id().toString(),
1754 deleteCandidateList, singleCommits);
1759 // keep them on our list
1760 returnList.add(dupesStr);
1765 // More than one node have the same key fields since they may
1766 // depend on a parent node for uniqueness. Since we're finding
1767 // more than one, we want to check to see if any of the
1768 // vertices that have this set of keys (and are the same nodeType)
1769 // are also pointing at the same 'parent' node.
1770 // Note: for a given set of key data, it is possible that there
1771 // could be more than one set of duplicates.
1772 HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
1773 transId, fromAppId, source, version, nType,
1774 checkVertList, loader);
1775 for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
1777 ArrayList<Vertex> thisParentsVertList = entry
1779 if (thisParentsVertList.size() > 1) {
1780 // More than one vertex found with the same key info
1781 // hanging off the same parent/dependent node
1782 String dupesStr = "";
1783 for (int i = 0; i < thisParentsVertList.size(); i++) {
1785 + ((thisParentsVertList
1786 .get(i))).id() + "|";
1788 if (dupesStr != "") {
1789 Vertex prefV = getPreferredDupe(transId,
1790 fromAppId, source, thisParentsVertList,
1793 if (prefV == null) {
1794 // We could not determine which duplicate to
1796 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1797 returnList.add(dupesStr);
1799 Boolean didRemove = false;
1800 dupesStr = dupesStr + "KeepVid="
1801 + prefV.id().toString();
1803 didRemove = deleteNonKeepersIfAppropriate(
1804 g, dupesStr, prefV.id()
1806 deleteCandidateList, singleCommits);
1811 // keep them on our list
1812 returnList.add(dupesStr);
1819 } catch (Exception e) {
1820 LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
1825 }// End of checkAndProcessDupes()
1828 * Group verts by dep nodes.
1830 * @param transId the trans id
1831 * @param fromAppId the from app id
1833 * @param version the version
1834 * @param nType the n type
1835 * @param passedVertList the passed vert list
1836 * @param dbMaps the db maps
1837 * @return the hash map
1838 * @throws AAIException the AAI exception
1840 private static HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
1841 String transId, String fromAppId, GraphTraversalSource g, String version,
1842 String nType, ArrayList<Vertex> passedVertList, Loader loader)
1843 throws AAIException {
1844 // Given a list of Titan Vertices of one nodeType (see AAI-8956), group
1845 // them together by the parent node they depend on.
1846 // Ie. if given a list of ip address nodes (assumed to all have the
1847 // same key info) they might sit under several different parent vertices.
1848 // Under Normal conditions, there would only be one per parent -- but
1849 // we're trying to find duplicates - so we
1850 // allow for the case where more than one is under the same parent node.
1852 HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
1853 if (loader.introspectorFromName(nType).isTopLevel()) {
1854 // This method really should not have been called if this is not the
1856 // that depends on a parent for uniqueness, so just return the empty
1861 // Find out what types of nodes the passed in nodes can depend on
1862 ArrayList<String> depNodeTypeL = new ArrayList<>();
1863 Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn();
1864 Iterator<String> ntItr = depNTColl.iterator();
1865 while (ntItr.hasNext()) {
1866 depNodeTypeL.add(ntItr.next());
1868 // For each vertex, we want find its depended-on/parent vertex so we
1869 // can track what other vertexes that are dependent on that same guy.
1870 if (passedVertList != null) {
1871 Iterator<Vertex> iter = passedVertList.iterator();
1872 while (iter.hasNext()) {
1873 Vertex thisVert = iter.next();
1874 Vertex tmpParentVtx = getConnectedParent( g, thisVert );
1875 if( tmpParentVtx != null ) {
1876 String parentNt = null;
1877 Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
1879 parentNt = obj.toString();
1881 if (depNTColl.contains(parentNt)) {
1882 // This must be the parent/dependent node
1883 String parentVid = tmpParentVtx.id().toString();
1884 if (retHash.containsKey(parentVid)) {
1885 // add this vert to the list for this parent key
1886 retHash.get(parentVid).add(thisVert);
1888 // This is the first one we found on this parent
1889 ArrayList<Vertex> vList = new ArrayList<>();
1890 vList.add(thisVert);
1891 retHash.put(parentVid, vList);
1900 }// end of groupVertsByDepNodes()
1903 * Delete non keepers if appropriate.
1906 * @param dupeInfoString the dupe info string
1907 * @param vidToKeep the vid to keep
1908 * @param deleteCandidateList the delete candidate list
1909 * @param singleCommits the single commits
1910 * @return the boolean
1912 private static Boolean deleteNonKeepersIfAppropriate(TitanTransaction g,
1913 String dupeInfoString, String vidToKeep,
1914 Set<String> deleteCandidateList, Boolean singleCommits) {
1916 Boolean deletedSomething = false;
1917 // This assumes that the dupeInfoString is in the format of
1918 // pipe-delimited vid's followed by
1919 // ie. "3456|9880|keepVid=3456"
1920 if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
1921 // No vid's on the candidate list -- so no deleting will happen on
1926 String[] dupeArr = dupeInfoString.split("\\|");
1927 ArrayList<String> idArr = new ArrayList<>();
1928 int lastIndex = dupeArr.length - 1;
1929 for (int i = 0; i <= lastIndex; i++) {
1930 if (i < lastIndex) {
1931 // This is not the last entry, it is one of the dupes,
1932 String vidString = dupeArr[i];
1933 idArr.add(vidString);
1935 // This is the last entry which should tell us if we have a
1937 String prefString = dupeArr[i];
1938 if (prefString.equals("KeepVid=UNDETERMINED")) {
1939 // They sent us a bad string -- nothing should be deleted if
1940 // no dupe could be tagged as preferred
1943 // If we know which to keep, then the prefString should look
1944 // like, "KeepVid=12345"
1945 String[] prefArr = prefString.split("=");
1946 if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
1947 LOGGER.error("Bad format. Expecting KeepVid=999999");
1950 String keepVidStr = prefArr[1];
1951 if (idArr.contains(keepVidStr)) {
1952 idArr.remove(keepVidStr);
1954 // So now, the idArr should just contain the vid's
1955 // that we want to remove.
1956 for (int x = 0; x < idArr.size(); x++) {
1957 boolean okFlag = true;
1958 String thisVid = idArr.get(x);
1959 if (deleteCandidateList.contains(thisVid)) {
1960 // This vid is a valid delete candidate from
1961 // a prev. run, so we can remove it.
1963 long longVertId = Long
1964 .parseLong(thisVid);
1966 .getVertex(longVertId);
1968 if (singleCommits) {
1969 // NOTE - the singleCommits option is not used in normal processing
1971 g = AAIGraph.getInstance().getGraph().newTransaction();
1973 } catch (Exception e) {
1975 LOGGER.error("ERROR trying to delete VID = " + thisVid, e);
1978 LOGGER.info(" DELETED VID = " + thisVid);
1979 deletedSomething = true;
1984 LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
1985 + dupeInfoString + "]");
1989 }// else we know which one to keep
1991 }// for each vertex in a group
1993 return deletedSomething;
1995 }// end of deleteNonKeepersIfAppropriate()
1999 * Gets the node just using key params.
2001 * @param transId the trans id
2002 * @param fromAppId the from app id
2003 * @param graph the graph
2004 * @param nodeType the node type
2005 * @param keyPropsHash the key props hash
2006 * @param apiVersion the api version
2007 * @return the node just using key params
2008 * @throws AAIException the AAI exception
2010 public static List <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, GraphTraversalSource graph, String nodeType,
2011 HashMap<String,Object> keyPropsHash, String apiVersion ) throws AAIException{
2013 List <Vertex> retVertList = new ArrayList <> ();
2015 // We assume that all NodeTypes have at least one key-property defined.
2016 // Note - instead of key-properties (the primary key properties), a user could pass
2017 // alternate-key values if they are defined for the nodeType.
2018 List<String> kName = new ArrayList<>();
2019 List<Object> kVal = new ArrayList<>();
2020 if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
2021 throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. ");
2025 for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){
2027 kName.add(i, entry.getKey());
2028 kVal.add(i, entry.getValue());
2030 int topPropIndex = i;
2032 String propsAndValuesForMsg = "";
2033 Iterator <Vertex> verts = null;
2036 if( topPropIndex == 0 ){
2037 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
2038 verts= graph.V().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType);
2040 else if( topPropIndex == 1 ){
2041 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2042 + kName.get(1) + " = " + kVal.get(1) + ") ";
2043 verts = graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType);
2045 else if( topPropIndex == 2 ){
2046 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2047 + kName.get(1) + " = " + kVal.get(1) + ", "
2048 + kName.get(2) + " = " + kVal.get(2) + ") ";
2049 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType);
2051 else if( topPropIndex == 3 ){
2052 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2053 + kName.get(1) + " = " + kVal.get(1) + ", "
2054 + kName.get(2) + " = " + kVal.get(2) + ", "
2055 + kName.get(3) + " = " + kVal.get(3) + ") ";
2056 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType);
2059 throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
2062 catch( Exception ex ){
2063 LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex);
2066 if( verts != null ){
2067 while( verts.hasNext() ){
2069 retVertList.add(tiV);
2073 if( retVertList.size() == 0 ){
2074 LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
2075 "], propsAndVal = " + propsAndValuesForMsg );
2080 }// End of getNodeJustUsingKeyParams()
2083 * Show all edges for node.
2085 * @param transId the trans id
2086 * @param fromAppId the from app id
2087 * @param tVert the t vert
2088 * @return the array list
2090 private static ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, Vertex tVert ){
2092 ArrayList <String> retArr = new ArrayList <> ();
2093 Iterator <Edge> eI = tVert.edges(Direction.IN);
2094 if( ! eI.hasNext() ){
2095 retArr.add("No IN edges were found for this vertex. ");
2097 while( eI.hasNext() ){
2098 TitanEdge ed = (TitanEdge) eI.next();
2099 String lab = ed.label();
2100 Vertex vtx = ed.otherVertex(tVert);
2102 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2105 String nType = vtx.<String>property("aai-node-type").orElse(null);
2106 String vid = vtx.id().toString();
2107 retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
2112 eI = tVert.edges(Direction.OUT);
2113 if( ! eI.hasNext() ){
2114 retArr.add("No OUT edges were found for this vertex. ");
2116 while( eI.hasNext() ){
2117 TitanEdge ed = (TitanEdge) eI.next();
2118 String lab = ed.label();
2119 Vertex vtx = ed.otherVertex(tVert);
2121 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2124 String nType = vtx.<String>property("aai-node-type").orElse(null);
2125 String vid = vtx.id().toString();
2126 retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
2134 * Show properties for node.
2136 * @param transId the trans id
2137 * @param fromAppId the from app id
2138 * @param tVert the t vert
2139 * @return the array list
2141 private static ArrayList <String> showPropertiesForNode( String transId, String fromAppId, Vertex tVert ){
2143 ArrayList <String> retArr = new ArrayList <> ();
2144 if( tVert == null ){
2145 retArr.add("null Node object passed to showPropertiesForNode()\n");
2148 String nodeType = "";
2149 Object ob = tVert.<Object>property("aai-node-type").orElse(null);
2154 nodeType = ob.toString();
2157 retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
2158 retArr.add(" Property Detail: ");
2159 Iterator<VertexProperty<Object>> pI = tVert.properties();
2160 while( pI.hasNext() ){
2161 VertexProperty<Object> tp = pI.next();
2162 Object val = tp.value();
2163 retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
2170 private static ArrayList <Vertex> getConnectedNodes(GraphTraversalSource g, Vertex startVtx )
2171 throws AAIException {
2173 ArrayList <Vertex> retArr = new ArrayList <> ();
2174 if( startVtx == null ){
2178 GraphTraversal<Vertex, Vertex> modPipe = null;
2179 modPipe = g.V(startVtx).both();
2180 if( modPipe != null && modPipe.hasNext() ){
2181 while( modPipe.hasNext() ){
2182 Vertex conVert = modPipe.next();
2183 retArr.add(conVert);
2189 }// End of getConnectedNodes()
2192 private static ArrayList <Vertex> getConnectedChildrenOfOneType( GraphTraversalSource g,
2193 Vertex startVtx, String childNType ) throws AAIException{
2195 ArrayList <Vertex> childList = new ArrayList <> ();
2196 Iterator <Vertex> vertI = g.V(startVtx).union(__.outE().has(EdgeProperties.out(EdgeProperty.IS_PARENT), true), __.inE().has(EdgeProperties.in(EdgeProperty.IS_PARENT), true)).bothV();
2197 Vertex tmpVtx = null;
2198 while( vertI != null && vertI.hasNext() ){
2199 tmpVtx = vertI.next();
2200 Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
2202 String tmpNt = ob.toString();
2203 if( tmpNt.equals(childNType)){
2204 childList.add(tmpVtx);
2211 }// End of getConnectedChildrenOfOneType()
2214 private static Vertex getConnectedParent( GraphTraversalSource g,
2215 Vertex startVtx ) throws AAIException{
2217 Vertex parentVtx = null;
2218 Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperties.out(EdgeProperty.IS_PARENT), true), __.outE().has(EdgeProperties.in(EdgeProperty.IS_PARENT), true)).bothV();
2219 while( vertI != null && vertI.hasNext() ){
2220 // Note - there better only be one!
2221 parentVtx = vertI.next();
2226 }// End of getConnectedParent()