2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.aai.dbgen;
23 import com.att.eelf.configuration.Configuration;
24 import com.att.eelf.configuration.EELFLogger;
25 import com.att.eelf.configuration.EELFManager;
26 import com.thinkaurelius.titan.core.TitanFactory;
27 import com.thinkaurelius.titan.core.TitanGraph;
28 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
29 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
30 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
31 import org.apache.tinkerpop.gremlin.structure.*;
32 import org.openecomp.aai.db.props.AAIProperties;
33 import org.openecomp.aai.dbmap.AAIGraph;
34 import org.openecomp.aai.exceptions.AAIException;
35 import org.openecomp.aai.introspection.Introspector;
36 import org.openecomp.aai.introspection.Loader;
37 import org.openecomp.aai.introspection.LoaderFactory;
38 import org.openecomp.aai.introspection.ModelType;
39 import org.openecomp.aai.introspection.exceptions.AAIUnknownObjectException;
40 import org.openecomp.aai.logging.ErrorLogHelper;
41 import org.openecomp.aai.serialization.db.AAIDirection;
42 import org.openecomp.aai.serialization.db.EdgeProperty;
43 import org.openecomp.aai.util.AAIConfig;
44 import org.openecomp.aai.util.AAIConstants;
45 import org.openecomp.aai.util.FormatDate;
49 import java.util.Map.Entry;
52 public class DataGrooming {
54 private static EELFLogger LOGGER;
55 private static final String FROMAPPID = "AAI-DB";
56 private static final String TRANSID = UUID.randomUUID().toString();
57 private static int dupeGrpsDeleted = 0;
62 * @param args the arguments
64 public static void main(String[] args) {
66 // Set the logging file properties to be used by EELFManager
67 Properties props = System.getProperties();
68 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS);
69 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
70 LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
71 String ver = "version"; // Placeholder
72 Boolean doAutoFix = false;
73 Boolean edgesOnlyFlag = false;
74 Boolean dontFixOrphansFlag = false;
75 Boolean skipHostCheck = false;
76 Boolean singleCommits = false;
77 Boolean dupeCheckOff = false;
78 Boolean dupeFixOn = false;
79 Boolean ghost2CheckOff = false;
80 Boolean ghost2FixOn = false;
81 Boolean neverUseCache = false;
82 Boolean skipEdgeCheckFlag = false;
84 int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look
85 // at all nodes of the passed-in nodeType.
86 long windowStartTime = 0; // Translation of the window into a starting timestamp
88 int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
89 int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
91 String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
92 if( maxFixStr != null && !maxFixStr.equals("") ){
93 maxRecordsToFix = Integer.parseInt(maxFixStr);
95 String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
96 if( sleepStr != null && !sleepStr.equals("") ){
97 sleepMinutes = Integer.parseInt(sleepStr);
100 catch ( Exception e ){
101 // Don't worry, we'll just use the defaults that we got from AAIConstants
102 LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
105 String prevFileName = "";
107 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
108 String dteStr = fd.getDateTime();
109 String groomOutFileName = "dataGrooming." + dteStr + ".out";
111 String argString = "";
112 for( int x = 0; x < args.length; x++ ) {
113 argString = argString + " " + args[x];
115 LOGGER.info(" DataGrooming called with these options: [" + argString + "]");
117 if (args.length > 0) {
118 // They passed some arguments in that will affect processing
120 for (int i = 0; i < args.length; i++) {
121 String thisArg = args[i];
122 if (thisArg.equals("-edgesOnly")) {
123 edgesOnlyFlag = true;
124 } else if (thisArg.equals("-autoFix")) {
126 } else if (thisArg.equals("-skipHostCheck")) {
127 skipHostCheck = true;
128 } else if (thisArg.equals("-dontFixOrphans")) {
129 dontFixOrphansFlag = true;
130 } else if (thisArg.equals("-singleCommits")) {
131 singleCommits = true;
132 } else if (thisArg.equals("-dupeCheckOff")) {
134 } else if (thisArg.equals("-dupeFixOn")) {
136 } else if (thisArg.equals("-ghost2CheckOff")) {
137 ghost2CheckOff = true;
138 } else if (thisArg.equals("-neverUseCache")) {
139 neverUseCache = true;
140 } else if (thisArg.equals("-ghost2FixOn")) {
142 } else if (thisArg.equals("-skipEdgeChecks")) {
143 skipEdgeCheckFlag = true;
144 } else if (thisArg.equals("-maxFix")) {
146 if (i >= args.length) {
147 LOGGER.error(" No value passed with -maxFix option. ");
150 String nextArg = args[i];
152 maxRecordsToFix = Integer.parseInt(nextArg);
153 } catch (Exception e) {
154 LOGGER.error("Bad value passed with -maxFix option: ["
158 } else if (thisArg.equals("-sleepMinutes")) {
160 if (i >= args.length) {
161 LOGGER.error("No value passed with -sleepMinutes option.");
164 String nextArg = args[i];
166 sleepMinutes = Integer.parseInt(nextArg);
167 } catch (Exception e) {
168 LOGGER.error("Bad value passed with -sleepMinutes option: ["
172 } else if (thisArg.equals("-timeWindowMinutes")) {
174 if (i >= args.length) {
175 LOGGER.error("No value passed with -timeWindowMinutes option.");
178 String nextArg = args[i];
180 timeWindowMinutes = Integer.parseInt(nextArg);
181 } catch (Exception e) {
182 LOGGER.error("Bad value passed with -timeWindowMinutes option: ["
186 if( timeWindowMinutes > 0 ){
187 // Translate the window value (ie. 30 minutes) into a unix timestamp like
188 // we use in the db - so we can select data created after that time.
189 windowStartTime = figureWindowStartTime( timeWindowMinutes );
191 } else if (thisArg.equals("-f")) {
193 if (i >= args.length) {
194 LOGGER.error(" No value passed with -f option. ");
197 prevFileName = args[i];
199 LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
201 LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache");
209 LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
212 catch (Exception ex){
213 LOGGER.error("ERROR - Could not create loader", ex);
219 if (!prevFileName.equals("")) {
220 // They are trying to fix some data based on a data in a
222 LOGGER.info(" Call doTheGrooming() with a previous fileName ["
223 + prevFileName + "] for cleanup. ");
224 Boolean finalShutdownFlag = true;
225 Boolean cacheDbOkFlag = false;
226 doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
227 maxRecordsToFix, groomOutFileName, ver, singleCommits,
228 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
229 finalShutdownFlag, cacheDbOkFlag,
230 skipEdgeCheckFlag, windowStartTime);
231 } else if (doAutoFix) {
232 // They want us to run the processing twice -- first to look for
233 // delete candidates, then after
234 // napping for a while, run it again and delete any candidates
235 // that were found by the first run.
236 // Note: we will produce a separate output file for each of the
238 LOGGER.info(" Doing an auto-fix call to Grooming. ");
239 LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
240 Boolean finalShutdownFlag = false;
241 Boolean cacheDbOkFlag = true;
242 int fixCandCount = doTheGrooming("", edgesOnlyFlag,
243 dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
244 ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
245 finalShutdownFlag, cacheDbOkFlag,
246 skipEdgeCheckFlag, windowStartTime);
247 if (fixCandCount == 0) {
248 LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
250 // We'll sleep a little and then run a fix-pass based on the
251 // first-run's output file.
253 LOGGER.info("About to sleep for " + sleepMinutes
255 int sleepMsec = sleepMinutes * 60 * 1000;
256 Thread.sleep(sleepMsec);
257 } catch (InterruptedException ie) {
258 LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
262 dteStr = fd.getDateTime();
263 String secondGroomOutFileName = "dataGrooming." + dteStr
265 LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
266 + "generated by the first pass for fixing: ["
267 + groomOutFileName + "]");
268 finalShutdownFlag = true;
269 cacheDbOkFlag = false;
270 doTheGrooming(groomOutFileName, edgesOnlyFlag,
271 dontFixOrphansFlag, maxRecordsToFix,
272 secondGroomOutFileName, ver, singleCommits,
273 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
274 finalShutdownFlag, cacheDbOkFlag,
275 skipEdgeCheckFlag, windowStartTime);
278 // Do the grooming - plain vanilla (no fix-it-file, no
280 Boolean finalShutdownFlag = true;
281 LOGGER.info(" Call doTheGrooming() ");
282 Boolean cacheDbOkFlag = true;
284 // They have forbidden us from using a cached db connection.
285 cacheDbOkFlag = false;
287 doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
288 maxRecordsToFix, groomOutFileName, ver, singleCommits,
289 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
290 finalShutdownFlag, cacheDbOkFlag,
291 skipEdgeCheckFlag, windowStartTime);
293 } catch (Exception ex) {
294 LOGGER.error("Exception while grooming data", ex);
297 LOGGER.info(" Done! ");
305 * @param fileNameForFixing the file name for fixing
306 * @param edgesOnlyFlag the edges only flag
307 * @param dontFixOrphansFlag the dont fix orphans flag
308 * @param maxRecordsToFix the max records to fix
309 * @param groomOutFileName the groom out file name
310 * @param version the version
311 * @param singleCommits the single commits
312 * @param dupeCheckOff the dupe check off
313 * @param dupeFixOn the dupe fix on
314 * @param ghost2CheckOff the ghost 2 check off
315 * @param ghost2FixOn the ghost 2 fix on
316 * @param finalShutdownFlag the final shutdown flag
317 * @param cacheDbOkFlag the cacheDbOk flag
320 private static int doTheGrooming(String fileNameForFixing,
321 Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
322 int maxRecordsToFix, String groomOutFileName, String version,
323 Boolean singleCommits,
324 Boolean dupeCheckOff, Boolean dupeFixOn,
325 Boolean ghost2CheckOff, Boolean ghost2FixOn,
326 Boolean finalShutdownFlag, Boolean cacheDbOkFlag,
327 Boolean skipEdgeCheckFlag, long windowStartTime) {
329 LOGGER.debug(" Entering doTheGrooming \n");
331 int cleanupCandidateCount = 0;
332 BufferedWriter bw = null;
333 TitanGraph graph = null;
334 TitanGraph graph2 = null;
336 boolean executeFinalCommit = false;
337 Set<String> deleteCandidateList = new LinkedHashSet<>();
338 Set<String> processedVertices = new LinkedHashSet<>();
343 String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
344 + "logs" + AAIConstants.AAI_FILESEP + "data"
345 + AAIConstants.AAI_FILESEP + "dataGrooming";
347 // Make sure the target directory exists
348 new File(targetDir).mkdirs();
350 if (!fileNameForFixing.equals("")) {
351 deleteCandidateList = getDeleteList(targetDir,
352 fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
356 if (deleteCandidateList.size() > maxRecordsToFix) {
357 LOGGER.warn(" >> WARNING >> Delete candidate list size ("
358 + deleteCandidateList.size()
359 + ") is too big. The maxFix we are using is: "
361 + ". No candidates will be deleted. ");
362 // Clear out the list so it won't be processed below.
363 deleteCandidateList = new LinkedHashSet<>();
366 String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
368 File groomOutFile = new File(fullOutputFileName);
370 groomOutFile.createNewFile();
371 } catch (IOException e) {
372 String emsg = " Problem creating output file ["
373 + fullOutputFileName + "], exception=" + e.getMessage();
374 throw new AAIException("AAI_6124", emsg);
377 LOGGER.info(" Will write to " + fullOutputFileName );
378 FileWriter fw = new FileWriter(groomOutFile.getAbsoluteFile());
379 bw = new BufferedWriter(fw);
380 ErrorLogHelper.loadProperties();
382 LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n");
385 // Since we're just reading (not deleting/fixing anything), we can use
386 // a cached connection to the DB
387 graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG);
390 graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
393 String emsg = "null graph object in DataGrooming\n";
394 throw new AAIException("AAI_6101", emsg);
397 LOGGER.debug(" Got the graph object. ");
399 g = graph.newTransaction();
401 String emsg = "null graphTransaction object in DataGrooming\n";
402 throw new AAIException("AAI_6101", emsg);
404 GraphTraversalSource source1 = g.traversal();
406 ArrayList<String> errArr = new ArrayList<>();
407 int totalNodeCount = 0;
408 HashMap<String, String> misMatchedHash = new HashMap<String, String>();
409 HashMap<String, Vertex> orphanNodeHash = new HashMap<String, Vertex>();
410 HashMap<String, Vertex> missingDepNodeHash = new HashMap<String, Vertex>();
411 HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
412 HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
413 HashMap<String, Vertex> ghostNodeHash = new HashMap<String, Vertex>();
414 ArrayList<String> dupeGroups = new ArrayList<>();
416 Loader loader = LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
418 Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
421 LOGGER.info(" Starting DataGrooming Processing ");
424 LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
427 for (Entry<String, Introspector> entry : entrySet) {
428 String nType = entry.getKey();
430 int thisNtDeleteCount = 0;
431 LOGGER.debug(" > Look at : [" + nType + "] ...");
432 ntList = ntList + "," + nType;
434 // Get a collection of the names of the key properties for this nodeType to use later
435 // Determine what the key fields are for this nodeType
436 Collection <String> keyProps = entry.getValue().getKeys();
438 // Get the types of nodes that this nodetype depends on for uniqueness (if any)
439 Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn();
441 // Loop through all the nodes of this Node type
442 int lastShownForNt = 0;
443 ArrayList <Vertex> tmpList = new ArrayList <> ();
444 Iterator <Vertex> iterv = source1.V().has("aai-node-type",nType);
445 while (iterv.hasNext()) {
446 // We put the nodes into an ArrayList because the graph.query iterator can time out
447 tmpList.add(iterv.next());
450 Iterator <Vertex> iter = tmpList.iterator();
451 while (iter.hasNext()) {
454 if( thisNtCount == lastShownForNt + 250 ){
455 lastShownForNt = thisNtCount;
456 LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
458 Vertex thisVtx = iter.next();
459 if( windowStartTime > 0 ){
460 // We only want nodes that are created after a passed-in timestamp
461 Object objTimeStamp = thisVtx.property("aai-created-ts").orElse(null);
462 if( objTimeStamp != null ){
463 long thisNodeCreateTime = (long)objTimeStamp;
464 if( thisNodeCreateTime < windowStartTime ){
465 // It is NOT in our window, so we can pass over it
471 String thisVid = thisVtx.id().toString();
472 if (processedVertices.contains(thisVid)) {
473 LOGGER.debug("skipping already processed vertex: " + thisVid);
477 List <Vertex> secondGetList = new ArrayList <> ();
478 // -----------------------------------------------------------------------
479 // For each vertex of this nodeType, we want to:
480 // a) make sure that it can be retrieved using it's AAI defined key
481 // b) make sure that it is not a duplicate
482 // -----------------------------------------------------------------------
484 // For this instance of this nodeType, get the key properties
485 HashMap<String, Object> propHashWithKeys = new HashMap<>();
486 Iterator<String> keyPropI = keyProps.iterator();
487 while (keyPropI.hasNext()) {
488 String propName = keyPropI.next();
490 //delete an already deleted vertex
491 Object obj = thisVtx.<Object>property(propName).orElse(null);
493 propVal = obj.toString();
495 propHashWithKeys.put(propName, propVal);
498 // If this node is dependent on another for uniqueness, then do the query from that parent node
499 // Note - all of our nodes that are dependent on others for uniqueness are
500 // "children" of that node.
501 boolean depNodeOk = true;
502 if( depNodeTypes.isEmpty() ){
503 // This kind of node is not dependent on any other.
504 // Make sure we can get it back using it's key properties and that we only get one.
505 secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType,
506 propHashWithKeys, version );
509 // This kind of node is dependent on another for uniqueness.
510 // Start at it's parent (the dependent vertex) and make sure we can get it
511 // back using it's key properties and that we only get one.
512 Iterator <Vertex> vertI2 = source1.V(thisVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
513 Vertex parentVtx = null;
515 while( vertI2 != null && vertI2.hasNext() ){
516 parentVtx = vertI2.next();
522 //List<Vertex> vertI2 = g.traversal().V(thisVtx).union(__.outE().has("isParent-REV",true).outV(),__.inE().has("isParent",true).inV()).toList();
523 //if( vertI2.isEmpty()){
525 // It's Missing it's dependent/parent node
527 boolean zeroEdges = false;
529 Iterator<Edge> tmpEdgeIter = thisVtx.edges(Direction.BOTH);
531 while( tmpEdgeIter.hasNext() ){
535 if( edgeCount == 0 ){
538 } catch (Exception ex) {
539 LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex);
542 if (deleteCandidateList.contains(thisVid)) {
543 boolean okFlag = true;
545 processedVertices.add(thisVtx.id().toString());
549 } catch (Exception e) {
551 LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e);
554 LOGGER.info(" DELETED missing-dep-node VID = " + thisVid);
557 // We count nodes missing their depNodes two ways - the first if it has
558 // at least some edges, and the second if it has zero edges. Either
559 // way, they are effectively orphaned.
560 // NOTE - Only nodes that have dependent nodes are ever considered "orphaned".
562 missingDepNodeHash.put(thisVid, thisVtx);
565 orphanNodeHash.put(thisVid, thisVtx);
569 else if ( pCount > 1 ){
570 // Not sure how this could happen? Should we do something here?
574 // We found the parent - so use it to do the second-look.
575 // NOTE --- We're just going to do the same check from the other direction - because
576 // there could be duplicates or the pointer going the other way could be broken
577 ArrayList <Vertex> tmpListSec = new ArrayList <> ();
579 tmpListSec = getConnectedChildrenOfOneType( source1, parentVtx, nType ) ;
580 Iterator<Vertex> vIter = tmpListSec.iterator();
581 while (vIter.hasNext()) {
582 Vertex tmpV = vIter.next();
583 if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
584 secondGetList.add(tmpV);
590 if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
591 // We could not get the node back using it's own key info.
592 // So, it's a PHANTOM
593 if (deleteCandidateList.contains(thisVid)) {
594 boolean okFlag = true;
599 } catch (Exception e) {
601 LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e);
604 LOGGER.info(" DELETED VID = " + thisVid);
607 ghostNodeHash.put(thisVid, thisVtx);
610 else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
611 // Found some DUPLICATES - need to process them
612 LOGGER.info(" - now check Dupes for this guy - ");
613 List<String> tmpDupeGroups = checkAndProcessDupes(
614 TRANSID, FROMAPPID, g, source1, version,
615 nType, secondGetList, dupeFixOn,
616 deleteCandidateList, singleCommits, dupeGroups, loader);
617 Iterator<String> dIter = tmpDupeGroups.iterator();
618 while (dIter.hasNext()) {
619 // Add in any newly found dupes to our running list
620 String tmpGrp = dIter.next();
621 LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
622 dupeGroups.add(tmpGrp);
626 catch (AAIException e1) {
627 LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
628 errArr.add(e1.getErrorObject().toString());
630 catch (Exception e2) {
631 LOGGER.warn(" For nodeType = " + nType
632 + " Caught exception", e2);
633 errArr.add(e2.getMessage());
635 }// try block to enclose looping of a single vertex
636 catch (Exception exx) {
637 LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
640 } // while loop for each record of a nodeType
642 if ( (thisNtDeleteCount > 0) && singleCommits ) {
643 // NOTE - the singleCommits option is not used in normal processing
645 g = AAIGraph.getInstance().getGraph().newTransaction();
648 thisNtDeleteCount = 0;
649 LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " );
651 }// While-loop for each node type
652 }// end of check to make sure we weren't only supposed to do edges
655 if( !skipEdgeCheckFlag ){
656 // --------------------------------------------------------------------------------------
657 // Now, we're going to look for one-armed-edges. Ie. an edge that
659 // been deleted (because a vertex on one side was deleted) but
660 // somehow was not deleted.
661 // So the one end of it points to a vertexId -- but that vertex is
663 // --------------------------------------------------------------------------------------
665 // To do some strange checking - we need a second graph object
666 LOGGER.debug(" ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
667 // Note - graph2 just reads - but we want it to use a fresh connection to
668 // the database, so we are NOT using the CACHED DB CONFIG here.
669 graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
670 if (graph2 == null) {
671 String emsg = "null graph2 object in DataGrooming\n";
672 throw new AAIException("AAI_6101", emsg);
674 LOGGER.debug("Got the graph2 object... \n");
676 g2 = graph2.newTransaction();
678 String emsg = "null graphTransaction2 object in DataGrooming\n";
679 throw new AAIException("AAI_6101", emsg);
682 ArrayList<Vertex> vertList = new ArrayList<>();
683 Iterator<Vertex> vItor3 = g.traversal().V();
684 // Gotta hold these in a List - or else HBase times out as you cycle
686 while (vItor3.hasNext()) {
687 Vertex v = vItor3.next();
692 Iterator<Vertex> vItor2 = vertList.iterator();
693 LOGGER.info(" Checking for bad edges --- ");
695 while (vItor2.hasNext()) {
700 } catch (Exception vex) {
701 LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
706 String thisVertId = "";
708 thisVertId = v.id().toString();
709 } catch (Exception ev) {
710 LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. ");
713 if (ghostNodeHash.containsKey(thisVertId)) {
714 // This is a phantom node, so don't try to use it
715 LOGGER.info(" >> Skipping edge check for edges from vertexId = "
717 + ", since that guy is a Phantom Node");
721 if( windowStartTime > 0 ){
722 // We only want to look at nodes that are created after a passed-in timestamp
723 Object objTimeStamp = v.property("aai-created-ts").orElse(null);
724 if( objTimeStamp != null ){
725 long thisNodeCreateTime = (long)objTimeStamp;
726 if( thisNodeCreateTime < windowStartTime ){
727 // It is NOT in our window, so we can pass over it
733 if (counter == lastShown + 250) {
735 LOGGER.info("... Checking edges for vertex # "
738 Iterator<Edge> eItor = v.edges(Direction.BOTH);
739 while (eItor.hasNext()) {
745 } catch (Exception iex) {
746 LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
752 } catch (Exception err) {
753 LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
757 Vertex ghost2 = null;
759 Boolean keysMissing = true;
760 Boolean cantGetUsingVid = false;
763 Object ob = vIn.<Object>property("aai-node-type").orElse(null);
765 vNtI = ob.toString();
766 keysMissing = anyKeyFieldsMissing(vNtI, vIn, loader);
771 vIdI = ob.toString();
772 vIdLong = Long.parseLong(vIdI);
775 if( ! ghost2CheckOff ){
776 Vertex connectedVert = g2.traversal().V(vIdLong).next();
777 if( connectedVert == null ) {
778 LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
779 cantGetUsingVid = true;
781 // If we can NOT get this ghost with the SECOND graph-object,
782 // it is still a ghost since even though we can get data about it using the FIRST graph
785 ghost2 = g.traversal().V(vIdLong).next();
787 catch( Exception ex){
788 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
790 if( ghost2 != null ){
791 ghostNodeHash.put(vIdI, ghost2);
794 }// end of the ghost2 checking
796 catch (Exception err) {
797 LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
800 if (keysMissing || vIn == null || vNtI.equals("")
801 || cantGetUsingVid) {
802 // this is a bad edge because it points to a vertex
803 // that isn't there anymore or is corrupted
804 String thisEid = e.id().toString();
805 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
806 boolean okFlag = true;
807 if (!vIdI.equals("")) {
808 // try to get rid of the corrupted vertex
810 if( (ghost2 != null) && ghost2FixOn ){
817 // NOTE - the singleCommits option is not used in normal processing
819 g = AAIGraph.getInstance().getGraph().newTransaction();
822 } catch (Exception e1) {
824 LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
828 LOGGER.info(" DELETED vertex from bad edge = "
832 // remove the edge if we couldn't get the
837 // NOTE - the singleCommits option is not used in normal processing
839 g = AAIGraph.getInstance().getGraph().newTransaction();
842 } catch (Exception ex) {
843 // NOTE - often, the exception is just
844 // that this edge has already been
847 LOGGER.warn("WARNING when trying to delete edge = "
851 LOGGER.info(" DELETED edge = " + thisEid);
855 oneArmedEdgeHash.put(thisEid, e);
856 if ((vIn != null) && (vIn.id() != null)) {
857 emptyVertexHash.put(thisEid, vIn.id()
864 vOut = e.outVertex();
865 } catch (Exception err) {
866 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
872 cantGetUsingVid = false;
875 Object ob = vOut.<Object>property("aai-node-type").orElse(null);
877 vNtO = ob.toString();
878 keysMissing = anyKeyFieldsMissing(vNtO,
884 vIdO = ob.toString();
885 vIdLong = Long.parseLong(vIdO);
888 if( ! ghost2CheckOff ){
889 Vertex connectedVert = g2.traversal().V(vIdLong).next();
890 if( connectedVert == null ) {
891 cantGetUsingVid = true;
892 LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
893 // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
895 ghost2 = g.traversal().V(vIdLong).next();
897 catch( Exception ex){
898 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
900 if( ghost2 != null ){
901 ghostNodeHash.put(vIdO, ghost2);
905 } catch (Exception err) {
906 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
909 if (keysMissing || vOut == null || vNtO.equals("")
910 || cantGetUsingVid) {
911 // this is a bad edge because it points to a vertex
912 // that isn't there anymore
913 String thisEid = e.id().toString();
914 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
915 boolean okFlag = true;
916 if (!vIdO.equals("")) {
917 // try to get rid of the corrupted vertex
919 if( (ghost2 != null) && ghost2FixOn ){
926 // NOTE - the singleCommits option is not used in normal processing
928 g = AAIGraph.getInstance().getGraph().newTransaction();
931 } catch (Exception e1) {
933 LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
937 LOGGER.info(" DELETED vertex from bad edge = "
941 // remove the edge if we couldn't get the
946 // NOTE - the singleCommits option is not used in normal processing
948 g = AAIGraph.getInstance().getGraph().newTransaction();
951 } catch (Exception ex) {
952 // NOTE - often, the exception is just
953 // that this edge has already been
956 LOGGER.warn("WARNING when trying to delete edge = "
960 LOGGER.info(" DELETED edge = " + thisEid);
964 oneArmedEdgeHash.put(thisEid, e);
965 if ((vOut != null) && (vOut.id() != null)) {
966 emptyVertexHash.put(thisEid, vOut.id()
971 }// End of while-edges-loop
972 } catch (Exception exx) {
973 LOGGER.warn("WARNING from in the while-verts-loop ", exx);
975 }// End of while-vertices-loop (the edge-checking)
976 } // end of -- if we're not skipping the edge-checking
979 deleteCount = deleteCount + dupeGrpsDeleted;
980 if (!singleCommits && deleteCount > 0) {
982 LOGGER.info("About to do the commit for "
983 + deleteCount + " removes. ");
984 executeFinalCommit = true;
985 LOGGER.info("Commit was successful ");
986 } catch (Exception excom) {
987 LOGGER.error(" >>>> ERROR <<<< Could not commit changes. ", excom);
992 int ghostNodeCount = ghostNodeHash.size();
993 int orphanNodeCount = orphanNodeHash.size();
994 int missingDepNodeCount = missingDepNodeHash.size();
995 int oneArmedEdgeCount = oneArmedEdgeHash.size();
996 int dupeCount = dupeGroups.size();
998 deleteCount = deleteCount + dupeGrpsDeleted;
1000 bw.write("\n\n ============ Summary ==============\n");
1001 bw.write("Ran these nodeTypes: " + ntList + "\n\n");
1002 bw.write("There were this many delete candidates from previous run = "
1003 + deleteCandidateList.size() + "\n");
1004 if (dontFixOrphansFlag) {
1005 bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
1007 bw.write("Deleted this many delete candidates = " + deleteCount
1009 bw.write("Total number of nodes looked at = " + totalNodeCount
1011 bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
1012 bw.write("Orphan Nodes identified = " + orphanNodeCount + "\n");
1013 bw.write("Bad Edges identified = " + oneArmedEdgeCount + "\n");
1014 bw.write("Missing Dependent Edge (but not orphaned) node count = "
1015 + missingDepNodeCount + "\n");
1016 bw.write("Duplicate Groups count = " + dupeCount + "\n");
1017 bw.write("MisMatching Label/aai-node-type count = "
1018 + misMatchedHash.size() + "\n");
1020 bw.write("\n ------------- Delete Candidates ---------\n");
1021 for (Entry<String, Vertex> entry : ghostNodeHash
1023 String vid = entry.getKey();
1024 bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
1025 cleanupCandidateCount++;
1027 for (Entry<String, Vertex> entry : orphanNodeHash
1029 String vid = entry.getKey();
1030 bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
1031 if (!dontFixOrphansFlag) {
1032 cleanupCandidateCount++;
1035 for (Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1036 String eid = entry.getKey();
1037 bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
1038 cleanupCandidateCount++;
1040 for (Entry<String, Vertex> entry : missingDepNodeHash
1042 String vid = entry.getKey();
1043 bw.write("DeleteCandidate: (maybe) missingDepNode Vid = ["
1045 cleanupCandidateCount++;
1047 bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
1049 bw.write("\n ------------- GHOST NODES - detail ");
1050 for (Entry<String, Vertex> entry : ghostNodeHash
1053 String vid = entry.getKey();
1054 bw.write("\n ==> Phantom Vid = " + vid + "\n");
1055 ArrayList<String> retArr = showPropertiesForNode(
1056 TRANSID, FROMAPPID, entry.getValue());
1057 for (String info : retArr) {
1058 bw.write(info + "\n");
1061 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1063 for (String info : retArr) {
1064 bw.write(info + "\n");
1066 } catch (Exception dex) {
1067 LOGGER.error("error trying to print detail info for a ghost-node: ", dex);
1071 bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
1072 for (Entry<String, Vertex> entry : orphanNodeHash
1075 String vid = entry.getKey();
1076 bw.write("\n> Orphan Node Vid = " + vid + "\n");
1077 ArrayList<String> retArr = showPropertiesForNode(
1078 TRANSID, FROMAPPID, entry.getValue());
1079 for (String info : retArr) {
1080 bw.write(info + "\n");
1083 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1085 for (String info : retArr) {
1086 bw.write(info + "\n");
1088 } catch (Exception dex) {
1089 LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex);
1093 bw.write("\n ------------- Missing Dependent Edge (but not orphan) NODES: ");
1094 for (Entry<String, Vertex> entry : missingDepNodeHash
1097 String vid = entry.getKey();
1098 bw.write("\n> Missing edge to Dependent Node (but has edges) Vid = "
1100 ArrayList<String> retArr = showPropertiesForNode(
1101 TRANSID, FROMAPPID, entry.getValue());
1102 for (String info : retArr) {
1103 bw.write(info + "\n");
1106 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1108 for (String info : retArr) {
1109 bw.write(info + "\n");
1111 } catch (Exception dex) {
1112 LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex);
1116 bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
1117 for (Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1119 String eid = entry.getKey();
1120 Edge thisE = entry.getValue();
1121 String badVid = emptyVertexHash.get(eid);
1122 bw.write("\n> Edge pointing to bad vertex (Vid = "
1123 + badVid + ") EdgeId = " + eid + "\n");
1124 bw.write("Label: [" + thisE.label() + "]\n");
1125 Iterator<Property<Object>> pI = thisE.properties();
1126 while (pI.hasNext()) {
1127 Property<Object> propKey = pI.next();
1128 bw.write("Prop: [" + propKey + "], val = ["
1129 + propKey.value() + "]\n");
1131 } catch (Exception pex) {
1132 LOGGER.error("error trying to print empty/bad vertex data: ", pex);
1136 bw.write("\n ------------- Duplicates: ");
1137 Iterator<String> dupeIter = dupeGroups.iterator();
1138 int dupeSetCounter = 0;
1139 while (dupeIter.hasNext()) {
1141 String dset = (String) dupeIter.next();
1143 bw.write("\n --- Duplicate Group # " + dupeSetCounter
1144 + " Detail -----------\n");
1146 // We expect each line to have at least two vid's, followed
1147 // by the preferred one to KEEP
1148 String[] dupeArr = dset.split("\\|");
1149 ArrayList<String> idArr = new ArrayList<>();
1150 int lastIndex = dupeArr.length - 1;
1151 for (int i = 0; i <= lastIndex; i++) {
1152 if (i < lastIndex) {
1153 // This is not the last entry, it is one of the
1154 // dupes, so we want to show all its info
1155 bw.write(" >> Duplicate Group # "
1156 + dupeSetCounter + " Node # " + i
1158 String vidString = dupeArr[i];
1159 idArr.add(vidString);
1160 long longVertId = Long.parseLong(vidString);
1161 Iterator<Vertex> vtxIterator = g.vertices(longVertId);
1163 if (vtxIterator.hasNext()) {
1164 vtx = vtxIterator.next();
1166 ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
1167 for (String info : retArr) {
1168 bw.write(info + "\n");
1171 retArr = showAllEdgesForNode(TRANSID,
1173 for (String info : retArr) {
1174 bw.write(info + "\n");
1177 // This is the last entry which should tell us if we
1178 // have a preferred keeper
1179 String prefString = dupeArr[i];
1180 if (prefString.equals("KeepVid=UNDETERMINED")) {
1181 bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
1182 bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
1184 // If we know which to keep, then the prefString
1185 // should look like, "KeepVid=12345"
1186 String[] prefArr = prefString.split("=");
1187 if (prefArr.length != 2
1188 || (!prefArr[0].equals("KeepVid"))) {
1189 throw new Exception("Bad format. Expecting KeepVid=999999");
1191 String keepVidStr = prefArr[1];
1192 if (idArr.contains(keepVidStr)) {
1193 bw.write("\n The vertex we want to KEEP has vertexId = "
1195 bw.write("\n The others become delete candidates: \n");
1196 idArr.remove(keepVidStr);
1197 for (int x = 0; x < idArr.size(); x++) {
1198 cleanupCandidateCount++;
1199 bw.write("DeleteCandidate: Duplicate Vid = ["
1200 + idArr.get(x) + "]\n");
1203 throw new Exception("ERROR - Vertex Id to keep not found in list of dupes. dset = ["
1207 }// else we know which one to keep
1209 }// for each vertex in a group
1210 } catch (Exception dex) {
1211 LOGGER.error("error trying to print duplicate vertex data", dex);
1214 }// while - work on each group of dupes
1216 bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
1217 for (Entry<String, String> entry : misMatchedHash.entrySet()) {
1218 String msg = entry.getValue();
1219 bw.write("MixedMsg = " + msg + "\n");
1222 bw.write("\n ------------- Got these errors while processing: \n");
1223 Iterator<String> errIter = errArr.iterator();
1224 while (errIter.hasNext()) {
1225 String line = (String) errIter.next();
1226 bw.write(line + "\n");
1231 LOGGER.info("\n ------------- Done doing all the checks ------------ ");
1232 LOGGER.info("Output will be written to " + fullOutputFileName);
1234 if (cleanupCandidateCount > 0) {
1235 // Technically, this is not an error -- but we're throwing this
1236 // error so that hopefully a
1237 // monitoring system will pick it up and do something with it.
1238 throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
1239 + "] and investigate delete candidates. ");
1241 } catch (AAIException e) {
1242 LOGGER.error("Caught AAIException while grooming data", e);
1243 ErrorLogHelper.logException(e);
1244 } catch (Exception ex) {
1245 LOGGER.error("Caught exception while grooming data", ex);
1246 ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
1252 } catch (IOException iox) {
1253 LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
1257 if (g != null && g.tx().isOpen()) {
1258 // Any changes that worked correctly should have already done
1261 if (executeFinalCommit) {
1265 } catch (Exception ex) {
1266 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1267 LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
1271 if (g2 != null && g2.tx().isOpen()) {
1272 // Any changes that worked correctly should have already done
1276 } catch (Exception ex) {
1277 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1278 LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
1282 if( finalShutdownFlag ){
1284 if( graph != null && graph.isOpen() ){
1288 } catch (Exception ex) {
1289 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1290 LOGGER.warn("WARNING from final graph.shutdown()", ex);
1294 if( graph2 != null && graph2.isOpen() ){
1295 graph2.tx().close();
1298 } catch (Exception ex) {
1299 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1300 LOGGER.warn("WARNING from final graph2.shutdown()", ex);
1306 return cleanupCandidateCount;
1308 }// end of doTheGrooming()
1312 * Vertex has these keys.
1314 * @param tmpV the tmp V
1315 * @param propHashWithKeys the prop hash with keys
1316 * @return the boolean
1318 private static Boolean vertexHasTheseKeys( Vertex tmpV, HashMap <String, Object> propHashWithKeys) {
1319 Iterator <?> it = propHashWithKeys.entrySet().iterator();
1320 while( it.hasNext() ){
1321 String propName = "";
1322 String propVal = "";
1323 Entry <?,?>propEntry = (Entry<?,?>)it.next();
1324 Object propNameObj = propEntry.getKey();
1325 if( propNameObj != null ){
1326 propName = propNameObj.toString();
1328 Object propValObj = propEntry.getValue();
1329 if( propValObj != null ){
1330 propVal = propValObj.toString();
1332 Object checkValObj = tmpV.<Object>property(propName).orElse(null);
1333 if( checkValObj == null ) {
1336 else if( !propVal.equals(checkValObj.toString()) ){
1345 * Any key fields missing.
1347 * @param nType the n type
1349 * @return the boolean
1351 private static Boolean anyKeyFieldsMissing(String nType, Vertex v, Loader loader) {
1354 Introspector obj = null;
1356 obj = loader.introspectorFromName(nType);
1357 } catch (AAIUnknownObjectException e) {
1358 // They gave us a non-empty nodeType but our NodeKeyProps does
1359 // not have data for it. Since we do not know what the
1360 // key params are for this type of node, we will just
1362 String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
1363 + "]. We cannot determine required keys for this nType. ";
1364 // NOTE - this will be caught below and a "false" returned
1365 throw new AAIException("AAI_6121", emsg);
1368 // Determine what the key fields are for this nodeType
1369 Collection <String> keyPropNamesColl = obj.getKeys();
1370 Iterator<String> keyPropI = keyPropNamesColl.iterator();
1371 while (keyPropI.hasNext()) {
1372 String propName = keyPropI.next();
1373 Object ob = v.<Object>property(propName).orElse(null);
1374 if (ob == null || ob.toString().equals("")) {
1375 // It is missing a key property
1379 } catch (AAIException e) {
1380 // Something was wrong -- but since we weren't able to check
1381 // the keys, we will not declare that it is missing keys.
1389 * Gets the delete list.
1391 * @param targetDir the target dir
1392 * @param fileName the file name
1393 * @param edgesOnlyFlag the edges only flag
1394 * @param dontFixOrphans the dont fix orphans
1395 * @param dupeFixOn the dupe fix on
1396 * @return the delete list
1397 * @throws AAIException the AAI exception
1399 private static Set<String> getDeleteList(String targetDir,
1400 String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
1401 Boolean dupeFixOn) throws AAIException {
1403 // Look in the file for lines formated like we expect - pull out any
1404 // Vertex Id's to delete on this run
1405 Set<String> delList = new LinkedHashSet<>();
1406 String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
1407 BufferedReader br = null;
1410 br = new BufferedReader(new FileReader(fullFileName));
1411 String line = br.readLine();
1412 while (line != null) {
1413 if (!line.equals("") && line.startsWith("DeleteCandidate")) {
1414 if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
1415 // We're not going to process edge guys
1416 } else if (dontFixOrphans && line.contains("Orphan")) {
1417 // We're not going to process orphans
1418 } else if (!dupeFixOn && line.contains("Duplicate")) {
1419 // We're not going to process Duplicates
1421 int begIndex = line.indexOf("id = ");
1422 int endIndex = line.indexOf("]");
1423 String vidVal = line.substring(begIndex + 6, endIndex);
1424 delList.add(vidVal);
1427 line = br.readLine();
1430 } catch (IOException e) {
1431 throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
1432 + "], exception= " + e.getMessage());
1437 }// end of getDeleteList
1440 * Gets the preferred dupe.
1442 * @param transId the trans id
1443 * @param fromAppId the from app id
1445 * @param dupeVertexList the dupe vertex list
1446 * @param ver the ver
1448 * @throws AAIException the AAI exception
1450 public static Vertex getPreferredDupe(String transId,
1451 String fromAppId, GraphTraversalSource g,
1452 ArrayList<Vertex> dupeVertexList, String ver, Loader loader)
1453 throws AAIException {
1455 // This method assumes that it is being passed a List of vertex objects
1457 // violate our uniqueness constraints.
1459 Vertex nullVtx = null;
1461 if (dupeVertexList == null) {
1464 int listSize = dupeVertexList.size();
1465 if (listSize == 0) {
1468 if (listSize == 1) {
1469 return (dupeVertexList.get(0));
1472 Vertex vtxPreferred = null;
1473 Vertex currentFaveVtx = dupeVertexList.get(0);
1474 for (int i = 1; i < listSize; i++) {
1475 Vertex vtxB = dupeVertexList.get(i);
1476 vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
1477 currentFaveVtx, vtxB, ver, loader);
1478 if (vtxPreferred == null) {
1479 // We couldn't choose one
1482 currentFaveVtx = vtxPreferred;
1486 return (currentFaveVtx);
1488 } // end of getPreferredDupe()
1491 * Pick one of two dupes.
1493 * @param transId the trans id
1494 * @param fromAppId the from app id
1496 * @param vtxA the vtx A
1497 * @param vtxB the vtx B
1498 * @param ver the ver
1500 * @throws AAIException the AAI exception
1502 public static Vertex pickOneOfTwoDupes(String transId,
1503 String fromAppId, GraphTraversalSource g, Vertex vtxA,
1504 Vertex vtxB, String ver, Loader loader) throws AAIException {
1506 Vertex nullVtx = null;
1507 Vertex preferredVtx = null;
1509 Long vidA = new Long(vtxA.id().toString());
1510 Long vidB = new Long(vtxB.id().toString());
1512 String vtxANodeType = "";
1513 String vtxBNodeType = "";
1514 Object objType = vtxA.<Object>property("aai-node-type").orElse(null);
1515 if (objType != null) {
1516 vtxANodeType = objType.toString();
1518 objType = vtxB.<Object>property("aai-node-type").orElse(null);
1519 if (objType != null) {
1520 vtxBNodeType = objType.toString();
1523 if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
1524 // Either they're not really dupes or there's some bad data - so
1529 // Check that node A and B both have the same key values (or else they
1531 // (We'll check dep-node later)
1532 // Determine what the key fields are for this nodeType
1533 Collection <String> keyProps = new ArrayList <>();
1535 keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
1536 } catch (AAIUnknownObjectException e) {
1537 throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
1540 Iterator<String> keyPropI = keyProps.iterator();
1541 while (keyPropI.hasNext()) {
1542 String propName = keyPropI.next();
1543 String vtxAKeyPropVal = "";
1544 objType = vtxA.<Object>property(propName).orElse(null);
1545 if (objType != null) {
1546 vtxAKeyPropVal = objType.toString();
1548 String vtxBKeyPropVal = "";
1549 objType = vtxB.<Object>property(propName).orElse(null);
1550 if (objType != null) {
1551 vtxBKeyPropVal = objType.toString();
1554 if (vtxAKeyPropVal.equals("")
1555 || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
1556 // Either they're not really dupes or they are missing some key
1557 // data - so don't pick one
1562 // Collect the vid's and aai-node-types of the vertices that each vertex
1563 // (A and B) is connected to.
1564 ArrayList<String> vtxIdsConn2A = new ArrayList<>();
1565 ArrayList<String> vtxIdsConn2B = new ArrayList<>();
1566 HashMap<String, String> nodeTypesConn2A = new HashMap<>();
1567 HashMap<String, String> nodeTypesConn2B = new HashMap<>();
1569 ArrayList<Vertex> vertListA = getConnectedNodes( g, vtxA );
1570 if (vertListA != null) {
1571 Iterator<Vertex> iter = vertListA.iterator();
1572 while (iter.hasNext()) {
1573 Vertex tvCon = iter.next();
1574 String conVid = tvCon.id().toString();
1576 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1577 if (objType != null) {
1578 nt = objType.toString();
1580 nodeTypesConn2A.put(nt, conVid);
1581 vtxIdsConn2A.add(conVid);
1585 ArrayList<Vertex> vertListB = getConnectedNodes( g, vtxB );
1586 if (vertListB != null) {
1587 Iterator<Vertex> iter = vertListB.iterator();
1588 while (iter.hasNext()) {
1589 Vertex tvCon = iter.next();
1590 String conVid = tvCon.id().toString();
1592 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1593 if (objType != null) {
1594 nt = objType.toString();
1596 nodeTypesConn2B.put(nt, conVid);
1597 vtxIdsConn2B.add(conVid);
1601 // 1 - If this kind of node needs a dependent node for uniqueness, then
1602 // verify that they both nodes
1603 // point to the same dependent node (otherwise they're not really
1605 // Note - there are sometimes more than one dependent node type since
1606 // one nodeType can be used in
1607 // different ways. But for a particular node, it will only have one
1608 // dependent node that it's
1610 Collection <String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
1612 if (depNodeTypes.isEmpty()) {
1613 // This kind of node is not dependent on any other. That is ok.
1615 String depNodeVtxId4A = "";
1616 String depNodeVtxId4B = "";
1617 Iterator<String> iter = depNodeTypes.iterator();
1618 while (iter.hasNext()) {
1619 String depNodeType = iter.next();
1620 if (nodeTypesConn2A.containsKey(depNodeType)) {
1621 // This is the dependent node type that vertex A is using
1622 depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
1624 if (nodeTypesConn2B.containsKey(depNodeType)) {
1625 // This is the dependent node type that vertex B is using
1626 depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
1629 if (depNodeVtxId4A.equals("")
1630 || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
1631 // Either they're not really dupes or there's some bad data - so
1632 // don't pick either one
1637 if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
1638 // 2 - If they both have edges to all the same vertices, then return
1639 // the one with the lower vertexId.
1640 boolean allTheSame = true;
1641 Iterator<String> iter = vtxIdsConn2A.iterator();
1642 while (iter.hasNext()) {
1643 String vtxIdConn2A = iter.next();
1644 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1652 preferredVtx = vtxA;
1654 preferredVtx = vtxB;
1657 } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
1658 // 3 - VertexA is connected to more things than vtxB.
1659 // We'll pick VtxA if its edges are a superset of vtxB's edges.
1660 boolean missingOne = false;
1661 Iterator<String> iter = vtxIdsConn2B.iterator();
1662 while (iter.hasNext()) {
1663 String vtxIdConn2B = iter.next();
1664 if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
1670 preferredVtx = vtxA;
1672 } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
1673 // 4 - VertexB is connected to more things than vtxA.
1674 // We'll pick VtxB if its edges are a superset of vtxA's edges.
1675 boolean missingOne = false;
1676 Iterator<String> iter = vtxIdsConn2A.iterator();
1677 while (iter.hasNext()) {
1678 String vtxIdConn2A = iter.next();
1679 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1685 preferredVtx = vtxB;
1688 preferredVtx = nullVtx;
1691 return (preferredVtx);
1693 } // end of pickOneOfTwoDupes()
1696 * Check and process dupes.
1698 * @param transId the trans id
1699 * @param fromAppId the from app id
1701 * @param version the version
1702 * @param nType the n type
1703 * @param passedVertList the passed vert list
1704 * @param dupeFixOn the dupe fix on
1705 * @param deleteCandidateList the delete candidate list
1706 * @param singleCommits the single commits
1707 * @param alreadyFoundDupeGroups the already found dupe groups
1708 * @param dbMaps the db maps
1709 * @return the array list
1711 private static List<String> checkAndProcessDupes(String transId,
1712 String fromAppId, Graph g, GraphTraversalSource source, String version, String nType,
1713 List<Vertex> passedVertList, Boolean dupeFixOn,
1714 Set<String> deleteCandidateList, Boolean singleCommits,
1715 ArrayList<String> alreadyFoundDupeGroups, Loader loader ) {
1717 ArrayList<String> returnList = new ArrayList<>();
1718 ArrayList<Vertex> checkVertList = new ArrayList<>();
1719 ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
1720 Boolean noFilterList = true;
1721 Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
1722 while (afItr.hasNext()) {
1723 String dupeGrpStr = afItr.next();
1724 String[] dupeArr = dupeGrpStr.split("\\|");
1725 int lastIndex = dupeArr.length - 1;
1726 for (int i = 0; i < lastIndex; i++) {
1727 // Note: we don't want the last one...
1728 String vidString = dupeArr[i];
1729 alreadyFoundDupeVidArr.add(vidString);
1730 noFilterList = false;
1734 // For a given set of Nodes that were found with a set of KEY
1735 // Parameters, (nodeType + key data) we will
1736 // see if we find any duplicate nodes that need to be cleaned up. Note -
1737 // it's legit to have more than one
1738 // node with the same key data if the nodes depend on a parent for
1739 // uniqueness -- as long as the two nodes
1740 // don't hang off the same Parent.
1741 // If we find duplicates, and we can figure out which of each set of
1742 // duplicates is the one that we
1743 // think should be preserved, we will record that. Whether we can tell
1744 // which one should be
1745 // preserved or not, we will return info about any sets of duplicates
1748 // Each element in the returned arrayList might look like this:
1749 // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
1750 // couldn't figure out which one to keep)
1751 // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
1752 // thought the third one was the one that should survive)
1754 // Because of the way the calling code loops over stuff, we can get the
1755 // same data multiple times - so we should
1756 // not process any vertices that we've already seen.
1759 Iterator<Vertex> pItr = passedVertList.iterator();
1760 while (pItr.hasNext()) {
1761 Vertex tvx = pItr.next();
1762 String passedId = tvx.id().toString();
1763 if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
1764 // We haven't seen this one before - so we should check it.
1765 checkVertList.add(tvx);
1769 if (checkVertList.size() < 2) {
1770 // Nothing new to check.
1774 if (loader.introspectorFromName(nType).isTopLevel()) {
1775 // If this was a node that does NOT depend on other nodes for
1776 // uniqueness, and we
1777 // found more than one node using its key -- record the found
1778 // vertices as duplicates.
1779 String dupesStr = "";
1780 for (int i = 0; i < checkVertList.size(); i++) {
1782 + ((checkVertList.get(i))).id()
1785 if (dupesStr != "") {
1786 Vertex prefV = getPreferredDupe(transId, fromAppId,
1787 source, checkVertList, version, loader);
1788 if (prefV == null) {
1789 // We could not determine which duplicate to keep
1790 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1791 returnList.add(dupesStr);
1793 dupesStr = dupesStr + "KeepVid=" + prefV.id();
1794 Boolean didRemove = false;
1796 didRemove = deleteNonKeepersIfAppropriate(g,
1797 dupesStr, prefV.id().toString(),
1798 deleteCandidateList, singleCommits);
1803 // keep them on our list
1804 returnList.add(dupesStr);
1809 // More than one node have the same key fields since they may
1810 // depend on a parent node for uniqueness. Since we're finding
1811 // more than one, we want to check to see if any of the
1812 // vertices that have this set of keys (and are the same nodeType)
1813 // are also pointing at the same 'parent' node.
1814 // Note: for a given set of key data, it is possible that there
1815 // could be more than one set of duplicates.
1816 HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
1817 transId, fromAppId, source, version, nType,
1818 checkVertList, loader);
1819 for (Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
1821 ArrayList<Vertex> thisParentsVertList = entry
1823 if (thisParentsVertList.size() > 1) {
1824 // More than one vertex found with the same key info
1825 // hanging off the same parent/dependent node
1826 String dupesStr = "";
1827 for (int i = 0; i < thisParentsVertList.size(); i++) {
1829 + ((thisParentsVertList
1830 .get(i))).id() + "|";
1832 if (dupesStr != "") {
1833 Vertex prefV = getPreferredDupe(transId,
1834 fromAppId, source, thisParentsVertList,
1837 if (prefV == null) {
1838 // We could not determine which duplicate to
1840 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1841 returnList.add(dupesStr);
1843 Boolean didRemove = false;
1844 dupesStr = dupesStr + "KeepVid="
1845 + prefV.id().toString();
1847 didRemove = deleteNonKeepersIfAppropriate(
1848 g, dupesStr, prefV.id()
1850 deleteCandidateList, singleCommits);
1855 // keep them on our list
1856 returnList.add(dupesStr);
1863 } catch (Exception e) {
1864 LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
1869 }// End of checkAndProcessDupes()
1872 * Group verts by dep nodes.
1874 * @param transId the trans id
1875 * @param fromAppId the from app id
1877 * @param version the version
1878 * @param nType the n type
1879 * @param passedVertList the passed vert list
1880 * @param dbMaps the db maps
1881 * @return the hash map
1882 * @throws AAIException the AAI exception
1884 private static HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
1885 String transId, String fromAppId, GraphTraversalSource g, String version,
1886 String nType, ArrayList<Vertex> passedVertList, Loader loader)
1887 throws AAIException {
1888 // Given a list of Titan Vertices of one nodeType (see AAI-8956), group
1889 // them together by the parent node they depend on.
1890 // Ie. if given a list of ip address nodes (assumed to all have the
1891 // same key info) they might sit under several different parent vertices.
1892 // Under Normal conditions, there would only be one per parent -- but
1893 // we're trying to find duplicates - so we
1894 // allow for the case where more than one is under the same parent node.
1896 HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
1897 if (loader.introspectorFromName(nType).isTopLevel()) {
1898 // This method really should not have been called if this is not the
1900 // that depends on a parent for uniqueness, so just return the empty
1905 // Find out what types of nodes the passed in nodes can depend on
1906 ArrayList<String> depNodeTypeL = new ArrayList<>();
1907 Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn();
1908 Iterator<String> ntItr = depNTColl.iterator();
1909 while (ntItr.hasNext()) {
1910 depNodeTypeL.add(ntItr.next());
1912 // For each vertex, we want find its depended-on/parent vertex so we
1913 // can track what other vertexes that are dependent on that same guy.
1914 if (passedVertList != null) {
1915 Iterator<Vertex> iter = passedVertList.iterator();
1916 while (iter.hasNext()) {
1917 Vertex thisVert = iter.next();
1918 Vertex tmpParentVtx = getConnectedParent( g, thisVert );
1919 if( tmpParentVtx != null ) {
1920 String parentNt = null;
1921 Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
1923 parentNt = obj.toString();
1925 if (depNTColl.contains(parentNt)) {
1926 // This must be the parent/dependent node
1927 String parentVid = tmpParentVtx.id().toString();
1928 if (retHash.containsKey(parentVid)) {
1929 // add this vert to the list for this parent key
1930 retHash.get(parentVid).add(thisVert);
1932 // This is the first one we found on this parent
1933 ArrayList<Vertex> vList = new ArrayList<>();
1934 vList.add(thisVert);
1935 retHash.put(parentVid, vList);
1944 }// end of groupVertsByDepNodes()
1947 * Delete non keepers if appropriate.
1950 * @param dupeInfoString the dupe info string
1951 * @param vidToKeep the vid to keep
1952 * @param deleteCandidateList the delete candidate list
1953 * @param singleCommits the single commits
1954 * @return the boolean
1956 private static Boolean deleteNonKeepersIfAppropriate(Graph g,
1957 String dupeInfoString, String vidToKeep,
1958 Set<String> deleteCandidateList, Boolean singleCommits) {
1960 Boolean deletedSomething = false;
1961 // This assumes that the dupeInfoString is in the format of
1962 // pipe-delimited vid's followed by
1963 // ie. "3456|9880|keepVid=3456"
1964 if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
1965 // No vid's on the candidate list -- so no deleting will happen on
1970 String[] dupeArr = dupeInfoString.split("\\|");
1971 ArrayList<String> idArr = new ArrayList<>();
1972 int lastIndex = dupeArr.length - 1;
1973 for (int i = 0; i <= lastIndex; i++) {
1974 if (i < lastIndex) {
1975 // This is not the last entry, it is one of the dupes,
1976 String vidString = dupeArr[i];
1977 idArr.add(vidString);
1979 // This is the last entry which should tell us if we have a
1981 String prefString = dupeArr[i];
1982 if (prefString.equals("KeepVid=UNDETERMINED")) {
1983 // They sent us a bad string -- nothing should be deleted if
1984 // no dupe could be tagged as preferred
1987 // If we know which to keep, then the prefString should look
1988 // like, "KeepVid=12345"
1989 String[] prefArr = prefString.split("=");
1990 if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
1991 LOGGER.error("Bad format. Expecting KeepVid=999999");
1994 String keepVidStr = prefArr[1];
1995 if (idArr.contains(keepVidStr)) {
1996 idArr.remove(keepVidStr);
1998 // So now, the idArr should just contain the vid's
1999 // that we want to remove.
2000 for (int x = 0; x < idArr.size(); x++) {
2001 boolean okFlag = true;
2002 String thisVid = idArr.get(x);
2003 if (deleteCandidateList.contains(thisVid)) {
2004 // This vid is a valid delete candidate from
2005 // a prev. run, so we can remove it.
2007 long longVertId = Long
2008 .parseLong(thisVid);
2010 .traversal().V(longVertId).next();
2012 if (singleCommits) {
2013 // NOTE - the singleCommits option is not used in normal processing
2015 g = AAIGraph.getInstance().getGraph().newTransaction();
2017 } catch (Exception e) {
2019 LOGGER.error("ERROR trying to delete VID = " + thisVid, e);
2022 LOGGER.info(" DELETED VID = " + thisVid);
2023 deletedSomething = true;
2028 LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
2029 + dupeInfoString + "]");
2033 }// else we know which one to keep
2035 }// for each vertex in a group
2037 return deletedSomething;
2039 }// end of deleteNonKeepersIfAppropriate()
2043 * Gets the node just using key params.
2045 * @param transId the trans id
2046 * @param fromAppId the from app id
2047 * @param graph the graph
2048 * @param nodeType the node type
2049 * @param keyPropsHash the key props hash
2050 * @param apiVersion the api version
2051 * @return the node just using key params
2052 * @throws AAIException the AAI exception
2054 public static List <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, GraphTraversalSource graph, String nodeType,
2055 HashMap<String,Object> keyPropsHash, String apiVersion ) throws AAIException{
2057 List <Vertex> retVertList = new ArrayList <> ();
2059 // We assume that all NodeTypes have at least one key-property defined.
2060 // Note - instead of key-properties (the primary key properties), a user could pass
2061 // alternate-key values if they are defined for the nodeType.
2062 List<String> kName = new ArrayList<>();
2063 List<Object> kVal = new ArrayList<>();
2064 if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
2065 throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. ");
2069 for( Entry<String, Object> entry : keyPropsHash.entrySet() ){
2071 kName.add(i, entry.getKey());
2072 kVal.add(i, entry.getValue());
2074 int topPropIndex = i;
2076 String propsAndValuesForMsg = "";
2077 Iterator <Vertex> verts = null;
2080 if( topPropIndex == 0 ){
2081 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
2082 verts= graph.V().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType);
2084 else if( topPropIndex == 1 ){
2085 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2086 + kName.get(1) + " = " + kVal.get(1) + ") ";
2087 verts = graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType);
2089 else if( topPropIndex == 2 ){
2090 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2091 + kName.get(1) + " = " + kVal.get(1) + ", "
2092 + kName.get(2) + " = " + kVal.get(2) + ") ";
2093 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType);
2095 else if( topPropIndex == 3 ){
2096 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2097 + kName.get(1) + " = " + kVal.get(1) + ", "
2098 + kName.get(2) + " = " + kVal.get(2) + ", "
2099 + kName.get(3) + " = " + kVal.get(3) + ") ";
2100 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType);
2103 throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
2106 catch( Exception ex ){
2107 LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex);
2110 if( verts != null ){
2111 while( verts.hasNext() ){
2113 retVertList.add(tiV);
2117 if( retVertList.size() == 0 ){
2118 LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
2119 "], propsAndVal = " + propsAndValuesForMsg );
2124 }// End of getNodeJustUsingKeyParams()
2127 * Show all edges for node.
2129 * @param transId the trans id
2130 * @param fromAppId the from app id
2131 * @param tVert the t vert
2132 * @return the array list
2134 private static ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, Vertex tVert ){
2136 ArrayList <String> retArr = new ArrayList <> ();
2137 Iterator <Edge> eI = tVert.edges(Direction.IN);
2138 if( ! eI.hasNext() ){
2139 retArr.add("No IN edges were found for this vertex. ");
2141 while( eI.hasNext() ){
2142 Edge ed = eI.next();
2143 String lab = ed.label();
2145 if (tVert.equals(ed.inVertex())) {
2146 vtx = ed.outVertex();
2148 vtx = ed.inVertex();
2151 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2154 String nType = vtx.<String>property("aai-node-type").orElse(null);
2155 String vid = vtx.id().toString();
2156 retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
2161 eI = tVert.edges(Direction.OUT);
2162 if( ! eI.hasNext() ){
2163 retArr.add("No OUT edges were found for this vertex. ");
2165 while( eI.hasNext() ){
2166 Edge ed = eI.next();
2167 String lab = ed.label();
2169 if (tVert.equals(ed.inVertex())) {
2170 vtx = ed.outVertex();
2172 vtx = ed.inVertex();
2175 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2178 String nType = vtx.<String>property("aai-node-type").orElse(null);
2179 String vid = vtx.id().toString();
2180 retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
2188 * Show properties for node.
2190 * @param transId the trans id
2191 * @param fromAppId the from app id
2192 * @param tVert the t vert
2193 * @return the array list
2195 private static ArrayList <String> showPropertiesForNode( String transId, String fromAppId, Vertex tVert ){
2197 ArrayList <String> retArr = new ArrayList <> ();
2198 if( tVert == null ){
2199 retArr.add("null Node object passed to showPropertiesForNode()\n");
2202 String nodeType = "";
2203 Object ob = tVert.<Object>property("aai-node-type").orElse(null);
2208 nodeType = ob.toString();
2211 retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
2212 retArr.add(" Property Detail: ");
2213 Iterator<VertexProperty<Object>> pI = tVert.properties();
2214 while( pI.hasNext() ){
2215 VertexProperty<Object> tp = pI.next();
2216 Object val = tp.value();
2217 retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
2224 private static ArrayList <Vertex> getConnectedNodes(GraphTraversalSource g, Vertex startVtx )
2225 throws AAIException {
2227 ArrayList <Vertex> retArr = new ArrayList <> ();
2228 if( startVtx == null ){
2232 GraphTraversal<Vertex, Vertex> modPipe = null;
2233 modPipe = g.V(startVtx).both();
2234 if( modPipe != null && modPipe.hasNext() ){
2235 while( modPipe.hasNext() ){
2236 Vertex conVert = modPipe.next();
2237 retArr.add(conVert);
2243 }// End of getConnectedNodes()
2246 private static ArrayList <Vertex> getConnectedChildrenOfOneType( GraphTraversalSource g,
2247 Vertex startVtx, String childNType ) throws AAIException{
2249 ArrayList <Vertex> childList = new ArrayList <> ();
2250 Iterator <Vertex> vertI = g.V(startVtx).union(__.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).inV(), __.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).outV());
2252 Vertex tmpVtx = null;
2253 while( vertI != null && vertI.hasNext() ){
2254 tmpVtx = vertI.next();
2255 Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
2257 String tmpNt = ob.toString();
2258 if( tmpNt.equals(childNType)){
2259 childList.add(tmpVtx);
2266 }// End of getConnectedChildrenOfOneType()
2269 private static Vertex getConnectedParent( GraphTraversalSource g,
2270 Vertex startVtx ) throws AAIException{
2272 Vertex parentVtx = null;
2273 Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
2275 while( vertI != null && vertI.hasNext() ){
2276 // Note - there better only be one!
2277 parentVtx = vertI.next();
2282 }// End of getConnectedParent()
2285 private static long figureWindowStartTime( int timeWindowMinutes ){
2286 // Given a window size, calculate what the start-timestamp would be.
2288 if( timeWindowMinutes <= 0 ){
2289 // This just means that there is no window...
2292 long unixTimeNow = System.currentTimeMillis();
2293 long windowInMillis = timeWindowMinutes * 60 * 1000;
2295 long startTimeStamp = unixTimeNow - windowInMillis;
2297 return startTimeStamp;
2298 } // End of figureWindowStartTime()