2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.aai.dbgen;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
26 import java.io.FileReader;
27 import java.io.FileWriter;
28 import java.io.IOException;
29 import java.util.ArrayList;
30 import java.util.Collection;
31 import java.util.HashMap;
32 import java.util.Iterator;
33 import java.util.LinkedHashSet;
34 import java.util.List;
36 import java.util.Map.Entry;
37 import java.util.Properties;
39 import java.util.UUID;
41 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
42 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
43 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
44 import org.apache.tinkerpop.gremlin.structure.Direction;
45 import org.apache.tinkerpop.gremlin.structure.Edge;
46 import org.apache.tinkerpop.gremlin.structure.Graph;
47 import org.apache.tinkerpop.gremlin.structure.Property;
48 import org.apache.tinkerpop.gremlin.structure.Vertex;
49 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
50 import org.openecomp.aai.db.props.AAIProperties;
51 import org.openecomp.aai.dbmap.AAIGraph;
52 import org.openecomp.aai.exceptions.AAIException;
53 import org.openecomp.aai.introspection.Introspector;
54 import org.openecomp.aai.introspection.Loader;
55 import org.openecomp.aai.introspection.LoaderFactory;
56 import org.openecomp.aai.introspection.ModelType;
57 import org.openecomp.aai.introspection.exceptions.AAIUnknownObjectException;
58 import org.openecomp.aai.logging.ErrorLogHelper;
59 import org.openecomp.aai.serialization.db.AAIDirection;
60 import org.openecomp.aai.serialization.db.EdgeProperty;
61 import org.openecomp.aai.util.AAIConfig;
62 import org.openecomp.aai.util.AAIConstants;
63 import org.openecomp.aai.util.FormatDate;
65 import com.att.eelf.configuration.Configuration;
66 import com.att.eelf.configuration.EELFLogger;
67 import com.att.eelf.configuration.EELFManager;
68 import com.thinkaurelius.titan.core.TitanFactory;
69 import com.thinkaurelius.titan.core.TitanGraph;
72 public class DataGrooming {
74 private static EELFLogger LOGGER;
75 private static final String FROMAPPID = "AAI-DB";
76 private static final String TRANSID = UUID.randomUUID().toString();
77 private static int dupeGrpsDeleted = 0;
82 * @param args the arguments
84 public static void main(String[] args) {
86 // Set the logging file properties to be used by EELFManager
87 Properties props = System.getProperties();
88 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS);
89 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
90 LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
91 String ver = "version"; // Placeholder
92 Boolean doAutoFix = false;
93 Boolean edgesOnlyFlag = false;
94 Boolean dontFixOrphansFlag = false;
95 Boolean skipHostCheck = false;
96 Boolean singleCommits = false;
97 Boolean dupeCheckOff = false;
98 Boolean dupeFixOn = false;
99 Boolean ghost2CheckOff = false;
100 Boolean ghost2FixOn = false;
101 Boolean neverUseCache = false;
102 Boolean skipEdgeCheckFlag = false;
104 int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look
105 // at all nodes of the passed-in nodeType.
106 long windowStartTime = 0; // Translation of the window into a starting timestamp
108 int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
109 int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
111 String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
112 if( maxFixStr != null && !maxFixStr.equals("") ){
113 maxRecordsToFix = Integer.parseInt(maxFixStr);
115 String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
116 if( sleepStr != null && !sleepStr.equals("") ){
117 sleepMinutes = Integer.parseInt(sleepStr);
120 catch ( Exception e ){
121 // Don't worry, we'll just use the defaults that we got from AAIConstants
122 LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
125 String prevFileName = "";
127 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
128 String dteStr = fd.getDateTime();
129 String groomOutFileName = "dataGrooming." + dteStr + ".out";
131 String argString = "";
132 for( int x = 0; x < args.length; x++ ) {
133 argString = argString + " " + args[x];
135 LOGGER.info(" DataGrooming called with these options: [" + argString + "]");
137 if (args.length > 0) {
138 // They passed some arguments in that will affect processing
140 for (int i = 0; i < args.length; i++) {
141 String thisArg = args[i];
142 if (thisArg.equals("-edgesOnly")) {
143 edgesOnlyFlag = true;
144 } else if (thisArg.equals("-autoFix")) {
146 } else if (thisArg.equals("-skipHostCheck")) {
147 skipHostCheck = true;
148 } else if (thisArg.equals("-dontFixOrphans")) {
149 dontFixOrphansFlag = true;
150 } else if (thisArg.equals("-singleCommits")) {
151 singleCommits = true;
152 } else if (thisArg.equals("-dupeCheckOff")) {
154 } else if (thisArg.equals("-dupeFixOn")) {
156 } else if (thisArg.equals("-ghost2CheckOff")) {
157 ghost2CheckOff = true;
158 } else if (thisArg.equals("-neverUseCache")) {
159 neverUseCache = true;
160 } else if (thisArg.equals("-ghost2FixOn")) {
162 } else if (thisArg.equals("-skipEdgeChecks")) {
163 skipEdgeCheckFlag = true;
164 } else if (thisArg.equals("-maxFix")) {
166 if (i >= args.length) {
167 LOGGER.error(" No value passed with -maxFix option. ");
170 String nextArg = args[i];
172 maxRecordsToFix = Integer.parseInt(nextArg);
173 } catch (Exception e) {
174 LOGGER.error("Bad value passed with -maxFix option: ["
178 } else if (thisArg.equals("-sleepMinutes")) {
180 if (i >= args.length) {
181 LOGGER.error("No value passed with -sleepMinutes option.");
184 String nextArg = args[i];
186 sleepMinutes = Integer.parseInt(nextArg);
187 } catch (Exception e) {
188 LOGGER.error("Bad value passed with -sleepMinutes option: ["
192 } else if (thisArg.equals("-timeWindowMinutes")) {
194 if (i >= args.length) {
195 LOGGER.error("No value passed with -timeWindowMinutes option.");
198 String nextArg = args[i];
200 timeWindowMinutes = Integer.parseInt(nextArg);
201 } catch (Exception e) {
202 LOGGER.error("Bad value passed with -timeWindowMinutes option: ["
206 if( timeWindowMinutes > 0 ){
207 // Translate the window value (ie. 30 minutes) into a unix timestamp like
208 // we use in the db - so we can select data created after that time.
209 windowStartTime = figureWindowStartTime( timeWindowMinutes );
211 } else if (thisArg.equals("-f")) {
213 if (i >= args.length) {
214 LOGGER.error(" No value passed with -f option. ");
217 prevFileName = args[i];
219 LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
221 LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache");
229 LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
232 catch (Exception ex){
233 LOGGER.error("ERROR - Could not create loader", ex);
239 if (!prevFileName.equals("")) {
240 // They are trying to fix some data based on a data in a
242 LOGGER.info(" Call doTheGrooming() with a previous fileName ["
243 + prevFileName + "] for cleanup. ");
244 Boolean finalShutdownFlag = true;
245 Boolean cacheDbOkFlag = false;
246 doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
247 maxRecordsToFix, groomOutFileName, ver, singleCommits,
248 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
249 finalShutdownFlag, cacheDbOkFlag,
250 skipEdgeCheckFlag, windowStartTime);
251 } else if (doAutoFix) {
252 // They want us to run the processing twice -- first to look for
253 // delete candidates, then after
254 // napping for a while, run it again and delete any candidates
255 // that were found by the first run.
256 // Note: we will produce a separate output file for each of the
258 LOGGER.info(" Doing an auto-fix call to Grooming. ");
259 LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
260 Boolean finalShutdownFlag = false;
261 Boolean cacheDbOkFlag = true;
262 int fixCandCount = doTheGrooming("", edgesOnlyFlag,
263 dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
264 ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
265 finalShutdownFlag, cacheDbOkFlag,
266 skipEdgeCheckFlag, windowStartTime);
267 if (fixCandCount == 0) {
268 LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
270 // We'll sleep a little and then run a fix-pass based on the
271 // first-run's output file.
273 LOGGER.info("About to sleep for " + sleepMinutes
275 int sleepMsec = sleepMinutes * 60 * 1000;
276 Thread.sleep(sleepMsec);
277 } catch (InterruptedException ie) {
278 LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
282 dteStr = fd.getDateTime();
283 String secondGroomOutFileName = "dataGrooming." + dteStr
285 LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
286 + "generated by the first pass for fixing: ["
287 + groomOutFileName + "]");
288 finalShutdownFlag = true;
289 cacheDbOkFlag = false;
290 doTheGrooming(groomOutFileName, edgesOnlyFlag,
291 dontFixOrphansFlag, maxRecordsToFix,
292 secondGroomOutFileName, ver, singleCommits,
293 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
294 finalShutdownFlag, cacheDbOkFlag,
295 skipEdgeCheckFlag, windowStartTime);
298 // Do the grooming - plain vanilla (no fix-it-file, no
300 Boolean finalShutdownFlag = true;
301 LOGGER.info(" Call doTheGrooming() ");
302 Boolean cacheDbOkFlag = true;
304 // They have forbidden us from using a cached db connection.
305 cacheDbOkFlag = false;
307 doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
308 maxRecordsToFix, groomOutFileName, ver, singleCommits,
309 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
310 finalShutdownFlag, cacheDbOkFlag,
311 skipEdgeCheckFlag, windowStartTime);
313 } catch (Exception ex) {
314 LOGGER.error("Exception while grooming data", ex);
317 LOGGER.info(" Done! ");
325 * @param fileNameForFixing the file name for fixing
326 * @param edgesOnlyFlag the edges only flag
327 * @param dontFixOrphansFlag the dont fix orphans flag
328 * @param maxRecordsToFix the max records to fix
329 * @param groomOutFileName the groom out file name
330 * @param version the version
331 * @param singleCommits the single commits
332 * @param dupeCheckOff the dupe check off
333 * @param dupeFixOn the dupe fix on
334 * @param ghost2CheckOff the ghost 2 check off
335 * @param ghost2FixOn the ghost 2 fix on
336 * @param finalShutdownFlag the final shutdown flag
337 * @param cacheDbOkFlag the cacheDbOk flag
340 private static int doTheGrooming(String fileNameForFixing,
341 Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
342 int maxRecordsToFix, String groomOutFileName, String version,
343 Boolean singleCommits,
344 Boolean dupeCheckOff, Boolean dupeFixOn,
345 Boolean ghost2CheckOff, Boolean ghost2FixOn,
346 Boolean finalShutdownFlag, Boolean cacheDbOkFlag,
347 Boolean skipEdgeCheckFlag, long windowStartTime) {
349 LOGGER.debug(" Entering doTheGrooming \n");
351 int cleanupCandidateCount = 0;
352 BufferedWriter bw = null;
353 TitanGraph graph = null;
354 TitanGraph graph2 = null;
356 boolean executeFinalCommit = false;
357 Set<String> deleteCandidateList = new LinkedHashSet<>();
358 Set<String> processedVertices = new LinkedHashSet<>();
363 String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
364 + "logs" + AAIConstants.AAI_FILESEP + "data"
365 + AAIConstants.AAI_FILESEP + "dataGrooming";
367 // Make sure the target directory exists
368 new File(targetDir).mkdirs();
370 if (!fileNameForFixing.equals("")) {
371 deleteCandidateList = getDeleteList(targetDir,
372 fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
376 if (deleteCandidateList.size() > maxRecordsToFix) {
377 LOGGER.warn(" >> WARNING >> Delete candidate list size ("
378 + deleteCandidateList.size()
379 + ") is too big. The maxFix we are using is: "
381 + ". No candidates will be deleted. ");
382 // Clear out the list so it won't be processed below.
383 deleteCandidateList = new LinkedHashSet<>();
386 String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
388 File groomOutFile = new File(fullOutputFileName);
390 groomOutFile.createNewFile();
391 } catch (IOException e) {
392 String emsg = " Problem creating output file ["
393 + fullOutputFileName + "], exception=" + e.getMessage();
394 throw new AAIException("AAI_6124", emsg);
397 LOGGER.info(" Will write to " + fullOutputFileName );
398 bw = new BufferedWriter(new FileWriter(groomOutFile.getAbsoluteFile()));
399 ErrorLogHelper.loadProperties();
401 LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n");
404 // Since we're just reading (not deleting/fixing anything), we can use
405 // a cached connection to the DB
406 graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG);
409 graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
412 String emsg = "null graph object in DataGrooming\n";
413 throw new AAIException("AAI_6101", emsg);
416 LOGGER.debug(" Got the graph object. ");
418 g = graph.newTransaction();
420 String emsg = "null graphTransaction object in DataGrooming\n";
421 throw new AAIException("AAI_6101", emsg);
423 GraphTraversalSource source1 = g.traversal();
425 ArrayList<String> errArr = new ArrayList<>();
426 int totalNodeCount = 0;
427 HashMap<String, String> misMatchedHash = new HashMap<String, String>();
428 HashMap<String, Vertex> orphanNodeHash = new HashMap<String, Vertex>();
429 HashMap<String, Vertex> missingDepNodeHash = new HashMap<String, Vertex>();
430 HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
431 HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
432 HashMap<String, Vertex> ghostNodeHash = new HashMap<String, Vertex>();
433 ArrayList<String> dupeGroups = new ArrayList<>();
435 Loader loader = LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
437 Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
440 LOGGER.info(" Starting DataGrooming Processing ");
443 LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
446 for (Entry<String, Introspector> entry : entrySet) {
447 String nType = entry.getKey();
449 int thisNtDeleteCount = 0;
450 LOGGER.debug(" > Look at : [" + nType + "] ...");
451 ntList = ntList + "," + nType;
453 // Get a collection of the names of the key properties for this nodeType to use later
454 // Determine what the key fields are for this nodeType
455 Collection <String> keyProps = entry.getValue().getKeys();
457 // Get the types of nodes that this nodetype depends on for uniqueness (if any)
458 Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn();
460 // Loop through all the nodes of this Node type
461 int lastShownForNt = 0;
462 ArrayList <Vertex> tmpList = new ArrayList <> ();
463 Iterator <Vertex> iterv = source1.V().has("aai-node-type",nType);
464 while (iterv.hasNext()) {
465 // We put the nodes into an ArrayList because the graph.query iterator can time out
466 tmpList.add(iterv.next());
469 Iterator <Vertex> iter = tmpList.iterator();
470 while (iter.hasNext()) {
473 if( thisNtCount == lastShownForNt + 250 ){
474 lastShownForNt = thisNtCount;
475 LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
477 Vertex thisVtx = iter.next();
478 if( windowStartTime > 0 ){
479 // We only want nodes that are created after a passed-in timestamp
480 Object objTimeStamp = thisVtx.property("aai-created-ts").orElse(null);
481 if( objTimeStamp != null ){
482 long thisNodeCreateTime = (long)objTimeStamp;
483 if( thisNodeCreateTime < windowStartTime ){
484 // It is NOT in our window, so we can pass over it
490 String thisVid = thisVtx.id().toString();
491 if (processedVertices.contains(thisVid)) {
492 LOGGER.debug("skipping already processed vertex: " + thisVid);
496 List <Vertex> secondGetList = new ArrayList <> ();
497 // -----------------------------------------------------------------------
498 // For each vertex of this nodeType, we want to:
499 // a) make sure that it can be retrieved using it's AAI defined key
500 // b) make sure that it is not a duplicate
501 // -----------------------------------------------------------------------
503 // For this instance of this nodeType, get the key properties
504 HashMap<String, Object> propHashWithKeys = new HashMap<>();
505 Iterator<String> keyPropI = keyProps.iterator();
506 while (keyPropI.hasNext()) {
507 String propName = keyPropI.next();
509 //delete an already deleted vertex
510 Object obj = thisVtx.<Object>property(propName).orElse(null);
512 propVal = obj.toString();
514 propHashWithKeys.put(propName, propVal);
517 // If this node is dependent on another for uniqueness, then do the query from that parent node
518 // Note - all of our nodes that are dependent on others for uniqueness are
519 // "children" of that node.
520 boolean depNodeOk = true;
521 if( depNodeTypes.isEmpty() ){
522 // This kind of node is not dependent on any other.
523 // Make sure we can get it back using it's key properties and that we only get one.
524 secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType,
525 propHashWithKeys, version );
528 // This kind of node is dependent on another for uniqueness.
529 // Start at it's parent (the dependent vertex) and make sure we can get it
530 // back using it's key properties and that we only get one.
531 Iterator <Vertex> vertI2 = source1.V(thisVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
532 Vertex parentVtx = null;
534 while( vertI2 != null && vertI2.hasNext() ){
535 parentVtx = vertI2.next();
541 //List<Vertex> vertI2 = g.traversal().V(thisVtx).union(__.outE().has("isParent-REV",true).outV(),__.inE().has("isParent",true).inV()).toList();
542 //if( vertI2.isEmpty()){
544 // It's Missing it's dependent/parent node
546 boolean zeroEdges = false;
548 Iterator<Edge> tmpEdgeIter = thisVtx.edges(Direction.BOTH);
550 while( tmpEdgeIter.hasNext() ){
554 if( edgeCount == 0 ){
557 } catch (Exception ex) {
558 LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex);
561 if (deleteCandidateList.contains(thisVid)) {
562 boolean okFlag = true;
564 processedVertices.add(thisVtx.id().toString());
568 } catch (Exception e) {
570 LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e);
573 LOGGER.info(" DELETED missing-dep-node VID = " + thisVid);
576 // We count nodes missing their depNodes two ways - the first if it has
577 // at least some edges, and the second if it has zero edges. Either
578 // way, they are effectively orphaned.
579 // NOTE - Only nodes that have dependent nodes are ever considered "orphaned".
581 missingDepNodeHash.put(thisVid, thisVtx);
584 orphanNodeHash.put(thisVid, thisVtx);
588 else if ( pCount > 1 ){
589 // Not sure how this could happen? Should we do something here?
593 // We found the parent - so use it to do the second-look.
594 // NOTE --- We're just going to do the same check from the other direction - because
595 // there could be duplicates or the pointer going the other way could be broken
596 ArrayList <Vertex> tmpListSec = new ArrayList <> ();
598 tmpListSec = getConnectedChildrenOfOneType( source1, parentVtx, nType ) ;
599 Iterator<Vertex> vIter = tmpListSec.iterator();
600 while (vIter.hasNext()) {
601 Vertex tmpV = vIter.next();
602 if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
603 secondGetList.add(tmpV);
609 if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
610 // We could not get the node back using it's own key info.
611 // So, it's a PHANTOM
612 if (deleteCandidateList.contains(thisVid)) {
613 boolean okFlag = true;
618 } catch (Exception e) {
620 LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e);
623 LOGGER.info(" DELETED VID = " + thisVid);
626 ghostNodeHash.put(thisVid, thisVtx);
629 else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
630 // Found some DUPLICATES - need to process them
631 LOGGER.info(" - now check Dupes for this guy - ");
632 List<String> tmpDupeGroups = checkAndProcessDupes(
633 TRANSID, FROMAPPID, g, source1, version,
634 nType, secondGetList, dupeFixOn,
635 deleteCandidateList, singleCommits, dupeGroups, loader);
636 Iterator<String> dIter = tmpDupeGroups.iterator();
637 while (dIter.hasNext()) {
638 // Add in any newly found dupes to our running list
639 String tmpGrp = dIter.next();
640 LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
641 dupeGroups.add(tmpGrp);
645 catch (AAIException e1) {
646 LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
647 errArr.add(e1.getErrorObject().toString());
649 catch (Exception e2) {
650 LOGGER.warn(" For nodeType = " + nType
651 + " Caught exception", e2);
652 errArr.add(e2.getMessage());
654 }// try block to enclose looping of a single vertex
655 catch (Exception exx) {
656 LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
659 } // while loop for each record of a nodeType
661 if ( (thisNtDeleteCount > 0) && singleCommits ) {
662 // NOTE - the singleCommits option is not used in normal processing
664 g = AAIGraph.getInstance().getGraph().newTransaction();
667 thisNtDeleteCount = 0;
668 LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " );
670 }// While-loop for each node type
671 }// end of check to make sure we weren't only supposed to do edges
674 if( !skipEdgeCheckFlag ){
675 // --------------------------------------------------------------------------------------
676 // Now, we're going to look for one-armed-edges. Ie. an edge that
678 // been deleted (because a vertex on one side was deleted) but
679 // somehow was not deleted.
680 // So the one end of it points to a vertexId -- but that vertex is
682 // --------------------------------------------------------------------------------------
684 // To do some strange checking - we need a second graph object
685 LOGGER.debug(" ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
686 // Note - graph2 just reads - but we want it to use a fresh connection to
687 // the database, so we are NOT using the CACHED DB CONFIG here.
688 graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
689 if (graph2 == null) {
690 String emsg = "null graph2 object in DataGrooming\n";
691 throw new AAIException("AAI_6101", emsg);
693 LOGGER.debug("Got the graph2 object... \n");
695 g2 = graph2.newTransaction();
697 String emsg = "null graphTransaction2 object in DataGrooming\n";
698 throw new AAIException("AAI_6101", emsg);
701 ArrayList<Vertex> vertList = new ArrayList<>();
702 Iterator<Vertex> vItor3 = g.traversal().V();
703 // Gotta hold these in a List - or else HBase times out as you cycle
705 while (vItor3.hasNext()) {
706 Vertex v = vItor3.next();
711 Iterator<Vertex> vItor2 = vertList.iterator();
712 LOGGER.info(" Checking for bad edges --- ");
714 while (vItor2.hasNext()) {
719 } catch (Exception vex) {
720 LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
725 String thisVertId = "";
727 thisVertId = v.id().toString();
728 } catch (Exception ev) {
729 LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. ");
732 if (ghostNodeHash.containsKey(thisVertId)) {
733 // This is a phantom node, so don't try to use it
734 LOGGER.info(" >> Skipping edge check for edges from vertexId = "
736 + ", since that guy is a Phantom Node");
740 if( windowStartTime > 0 ){
741 // We only want to look at nodes that are created after a passed-in timestamp
742 Object objTimeStamp = v.property("aai-created-ts").orElse(null);
743 if( objTimeStamp != null ){
744 long thisNodeCreateTime = (long)objTimeStamp;
745 if( thisNodeCreateTime < windowStartTime ){
746 // It is NOT in our window, so we can pass over it
752 if (counter == lastShown + 250) {
754 LOGGER.info("... Checking edges for vertex # "
757 Iterator<Edge> eItor = v.edges(Direction.BOTH);
758 while (eItor.hasNext()) {
764 } catch (Exception iex) {
765 LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
771 } catch (Exception err) {
772 LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
776 Vertex ghost2 = null;
778 Boolean keysMissing = true;
779 Boolean cantGetUsingVid = false;
782 Object ob = vIn.<Object>property("aai-node-type").orElse(null);
784 vNtI = ob.toString();
785 keysMissing = anyKeyFieldsMissing(vNtI, vIn, loader);
790 vIdI = ob.toString();
791 vIdLong = Long.parseLong(vIdI);
794 if( ! ghost2CheckOff ){
795 Vertex connectedVert = g2.traversal().V(vIdLong).next();
796 if( connectedVert == null ) {
797 LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
798 cantGetUsingVid = true;
800 // If we can NOT get this ghost with the SECOND graph-object,
801 // it is still a ghost since even though we can get data about it using the FIRST graph
804 ghost2 = g.traversal().V(vIdLong).next();
806 catch( Exception ex){
807 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
809 if( ghost2 != null ){
810 ghostNodeHash.put(vIdI, ghost2);
813 }// end of the ghost2 checking
815 catch (Exception err) {
816 LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
819 if (keysMissing || vIn == null || vNtI.equals("")
820 || cantGetUsingVid) {
821 // this is a bad edge because it points to a vertex
822 // that isn't there anymore or is corrupted
823 String thisEid = e.id().toString();
824 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
825 boolean okFlag = true;
826 if (!vIdI.equals("")) {
827 // try to get rid of the corrupted vertex
829 if( (ghost2 != null) && ghost2FixOn ){
836 // NOTE - the singleCommits option is not used in normal processing
838 g = AAIGraph.getInstance().getGraph().newTransaction();
841 } catch (Exception e1) {
843 LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
847 LOGGER.info(" DELETED vertex from bad edge = "
851 // remove the edge if we couldn't get the
856 // NOTE - the singleCommits option is not used in normal processing
858 g = AAIGraph.getInstance().getGraph().newTransaction();
861 } catch (Exception ex) {
862 // NOTE - often, the exception is just
863 // that this edge has already been
866 LOGGER.warn("WARNING when trying to delete edge = "
870 LOGGER.info(" DELETED edge = " + thisEid);
874 oneArmedEdgeHash.put(thisEid, e);
875 if ((vIn != null) && (vIn.id() != null)) {
876 emptyVertexHash.put(thisEid, vIn.id()
883 vOut = e.outVertex();
884 } catch (Exception err) {
885 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
891 cantGetUsingVid = false;
894 Object ob = vOut.<Object>property("aai-node-type").orElse(null);
896 vNtO = ob.toString();
897 keysMissing = anyKeyFieldsMissing(vNtO,
903 vIdO = ob.toString();
904 vIdLong = Long.parseLong(vIdO);
907 if( ! ghost2CheckOff ){
908 Vertex connectedVert = g2.traversal().V(vIdLong).next();
909 if( connectedVert == null ) {
910 cantGetUsingVid = true;
911 LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
912 // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
914 ghost2 = g.traversal().V(vIdLong).next();
916 catch( Exception ex){
917 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
919 if( ghost2 != null ){
920 ghostNodeHash.put(vIdO, ghost2);
924 } catch (Exception err) {
925 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
928 if (keysMissing || vOut == null || vNtO.equals("")
929 || cantGetUsingVid) {
930 // this is a bad edge because it points to a vertex
931 // that isn't there anymore
932 String thisEid = e.id().toString();
933 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
934 boolean okFlag = true;
935 if (!vIdO.equals("")) {
936 // try to get rid of the corrupted vertex
938 if( (ghost2 != null) && ghost2FixOn ){
941 else if (vOut != null) {
945 // NOTE - the singleCommits option is not used in normal processing
947 g = AAIGraph.getInstance().getGraph().newTransaction();
950 } catch (Exception e1) {
952 LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
956 LOGGER.info(" DELETED vertex from bad edge = "
960 // remove the edge if we couldn't get the
965 // NOTE - the singleCommits option is not used in normal processing
967 g = AAIGraph.getInstance().getGraph().newTransaction();
970 } catch (Exception ex) {
971 // NOTE - often, the exception is just
972 // that this edge has already been
975 LOGGER.warn("WARNING when trying to delete edge = "
979 LOGGER.info(" DELETED edge = " + thisEid);
983 oneArmedEdgeHash.put(thisEid, e);
984 if ((vOut != null) && (vOut.id() != null)) {
985 emptyVertexHash.put(thisEid, vOut.id()
990 }// End of while-edges-loop
991 } catch (Exception exx) {
992 LOGGER.warn("WARNING from in the while-verts-loop ", exx);
994 }// End of while-vertices-loop (the edge-checking)
995 } // end of -- if we're not skipping the edge-checking
998 deleteCount = deleteCount + dupeGrpsDeleted;
999 if (!singleCommits && deleteCount > 0) {
1001 LOGGER.info("About to do the commit for "
1002 + deleteCount + " removes. ");
1003 executeFinalCommit = true;
1004 LOGGER.info("Commit was successful ");
1005 } catch (Exception excom) {
1006 LOGGER.error(" >>>> ERROR <<<< Could not commit changes. ", excom);
1011 int ghostNodeCount = ghostNodeHash.size();
1012 int orphanNodeCount = orphanNodeHash.size();
1013 int missingDepNodeCount = missingDepNodeHash.size();
1014 int oneArmedEdgeCount = oneArmedEdgeHash.size();
1015 int dupeCount = dupeGroups.size();
1017 deleteCount = deleteCount + dupeGrpsDeleted;
1019 bw.write("\n\n ============ Summary ==============\n");
1020 bw.write("Ran these nodeTypes: " + ntList + "\n\n");
1021 bw.write("There were this many delete candidates from previous run = "
1022 + deleteCandidateList.size() + "\n");
1023 if (dontFixOrphansFlag) {
1024 bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
1026 bw.write("Deleted this many delete candidates = " + deleteCount
1028 bw.write("Total number of nodes looked at = " + totalNodeCount
1030 bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
1031 bw.write("Orphan Nodes identified = " + orphanNodeCount + "\n");
1032 bw.write("Bad Edges identified = " + oneArmedEdgeCount + "\n");
1033 bw.write("Missing Dependent Edge (but not orphaned) node count = "
1034 + missingDepNodeCount + "\n");
1035 bw.write("Duplicate Groups count = " + dupeCount + "\n");
1036 bw.write("MisMatching Label/aai-node-type count = "
1037 + misMatchedHash.size() + "\n");
1039 bw.write("\n ------------- Delete Candidates ---------\n");
1040 for (Map.Entry<String, Vertex> entry : ghostNodeHash
1042 String vid = entry.getKey();
1043 bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
1044 cleanupCandidateCount++;
1046 for (Map.Entry<String, Vertex> entry : orphanNodeHash
1048 String vid = entry.getKey();
1049 bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
1050 if (!dontFixOrphansFlag) {
1051 cleanupCandidateCount++;
1054 for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1055 String eid = entry.getKey();
1056 bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
1057 cleanupCandidateCount++;
1059 for (Map.Entry<String, Vertex> entry : missingDepNodeHash
1061 String vid = entry.getKey();
1062 bw.write("DeleteCandidate: (maybe) missingDepNode Vid = ["
1064 cleanupCandidateCount++;
1066 bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
1068 bw.write("\n ------------- GHOST NODES - detail ");
1069 for (Map.Entry<String, Vertex> entry : ghostNodeHash
1072 String vid = entry.getKey();
1073 bw.write("\n ==> Phantom Vid = " + vid + "\n");
1074 ArrayList<String> retArr = showPropertiesForNode(
1075 TRANSID, FROMAPPID, entry.getValue());
1076 for (String info : retArr) {
1077 bw.write(info + "\n");
1080 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1082 for (String info : retArr) {
1083 bw.write(info + "\n");
1085 } catch (Exception dex) {
1086 LOGGER.error("error trying to print detail info for a ghost-node: ", dex);
1090 bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
1091 for (Map.Entry<String, Vertex> entry : orphanNodeHash
1094 String vid = entry.getKey();
1095 bw.write("\n> Orphan Node Vid = " + vid + "\n");
1096 ArrayList<String> retArr = showPropertiesForNode(
1097 TRANSID, FROMAPPID, entry.getValue());
1098 for (String info : retArr) {
1099 bw.write(info + "\n");
1102 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1104 for (String info : retArr) {
1105 bw.write(info + "\n");
1107 } catch (Exception dex) {
1108 LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex);
1112 bw.write("\n ------------- Missing Dependent Edge (but not orphan) NODES: ");
1113 for (Map.Entry<String, Vertex> entry : missingDepNodeHash
1116 String vid = entry.getKey();
1117 bw.write("\n> Missing edge to Dependent Node (but has edges) Vid = "
1119 ArrayList<String> retArr = showPropertiesForNode(
1120 TRANSID, FROMAPPID, entry.getValue());
1121 for (String info : retArr) {
1122 bw.write(info + "\n");
1125 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1127 for (String info : retArr) {
1128 bw.write(info + "\n");
1130 } catch (Exception dex) {
1131 LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex);
1135 bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
1136 for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1138 String eid = entry.getKey();
1139 Edge thisE = entry.getValue();
1140 String badVid = emptyVertexHash.get(eid);
1141 bw.write("\n> Edge pointing to bad vertex (Vid = "
1142 + badVid + ") EdgeId = " + eid + "\n");
1143 bw.write("Label: [" + thisE.label() + "]\n");
1144 Iterator<Property<Object>> pI = thisE.properties();
1145 while (pI.hasNext()) {
1146 Property<Object> propKey = pI.next();
1147 bw.write("Prop: [" + propKey + "], val = ["
1148 + propKey.value() + "]\n");
1150 } catch (Exception pex) {
1151 LOGGER.error("error trying to print empty/bad vertex data: ", pex);
1155 bw.write("\n ------------- Duplicates: ");
1156 Iterator<String> dupeIter = dupeGroups.iterator();
1157 int dupeSetCounter = 0;
1158 while (dupeIter.hasNext()) {
1160 String dset = (String) dupeIter.next();
1162 bw.write("\n --- Duplicate Group # " + dupeSetCounter
1163 + " Detail -----------\n");
1165 // We expect each line to have at least two vid's, followed
1166 // by the preferred one to KEEP
1167 String[] dupeArr = dset.split("\\|");
1168 ArrayList<String> idArr = new ArrayList<>();
1169 int lastIndex = dupeArr.length - 1;
1170 for (int i = 0; i <= lastIndex; i++) {
1171 if (i < lastIndex) {
1172 // This is not the last entry, it is one of the
1173 // dupes, so we want to show all its info
1174 bw.write(" >> Duplicate Group # "
1175 + dupeSetCounter + " Node # " + i
1177 String vidString = dupeArr[i];
1178 idArr.add(vidString);
1179 long longVertId = Long.parseLong(vidString);
1180 Iterator<Vertex> vtxIterator = g.vertices(longVertId);
1182 if (vtxIterator.hasNext()) {
1183 vtx = vtxIterator.next();
1185 ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
1186 for (String info : retArr) {
1187 bw.write(info + "\n");
1190 retArr = showAllEdgesForNode(TRANSID,
1192 for (String info : retArr) {
1193 bw.write(info + "\n");
1196 // This is the last entry which should tell us if we
1197 // have a preferred keeper
1198 String prefString = dupeArr[i];
1199 if (prefString.equals("KeepVid=UNDETERMINED")) {
1200 bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
1201 bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
1203 // If we know which to keep, then the prefString
1204 // should look like, "KeepVid=12345"
1205 String[] prefArr = prefString.split("=");
1206 if (prefArr.length != 2
1207 || (!prefArr[0].equals("KeepVid"))) {
1208 throw new Exception("Bad format. Expecting KeepVid=999999");
1210 String keepVidStr = prefArr[1];
1211 if (idArr.contains(keepVidStr)) {
1212 bw.write("\n The vertex we want to KEEP has vertexId = "
1214 bw.write("\n The others become delete candidates: \n");
1215 idArr.remove(keepVidStr);
1216 for (int x = 0; x < idArr.size(); x++) {
1217 cleanupCandidateCount++;
1218 bw.write("DeleteCandidate: Duplicate Vid = ["
1219 + idArr.get(x) + "]\n");
1222 throw new Exception("ERROR - Vertex Id to keep not found in list of dupes. dset = ["
1226 }// else we know which one to keep
1228 }// for each vertex in a group
1229 } catch (Exception dex) {
1230 LOGGER.error("error trying to print duplicate vertex data", dex);
1233 }// while - work on each group of dupes
1235 bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
1236 for (Map.Entry<String, String> entry : misMatchedHash.entrySet()) {
1237 String msg = entry.getValue();
1238 bw.write("MixedMsg = " + msg + "\n");
1241 bw.write("\n ------------- Got these errors while processing: \n");
1242 Iterator<String> errIter = errArr.iterator();
1243 while (errIter.hasNext()) {
1244 String line = (String) errIter.next();
1245 bw.write(line + "\n");
1250 LOGGER.info("\n ------------- Done doing all the checks ------------ ");
1251 LOGGER.info("Output will be written to " + fullOutputFileName);
1253 if (cleanupCandidateCount > 0) {
1254 // Technically, this is not an error -- but we're throwing this
1255 // error so that hopefully a
1256 // monitoring system will pick it up and do something with it.
1257 throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
1258 + "] and investigate delete candidates. ");
1260 } catch (AAIException e) {
1261 LOGGER.error("Caught AAIException while grooming data", e);
1262 ErrorLogHelper.logException(e);
1263 } catch (Exception ex) {
1264 LOGGER.error("Caught exception while grooming data", ex);
1265 ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
1271 } catch (IOException iox) {
1272 LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
1276 if (g != null && g.tx().isOpen()) {
1277 // Any changes that worked correctly should have already done
1280 if (executeFinalCommit) {
1284 } catch (Exception ex) {
1285 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1286 LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
1290 if (g2 != null && g2.tx().isOpen()) {
1291 // Any changes that worked correctly should have already done
1295 } catch (Exception ex) {
1296 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1297 LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
1301 if( finalShutdownFlag ){
1303 if( graph != null && graph.isOpen() ){
1307 } catch (Exception ex) {
1308 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1309 LOGGER.warn("WARNING from final graph.shutdown()", ex);
1313 if( graph2 != null && graph2.isOpen() ){
1314 graph2.tx().close();
1317 } catch (Exception ex) {
1318 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1319 LOGGER.warn("WARNING from final graph2.shutdown()", ex);
1325 return cleanupCandidateCount;
1327 }// end of doTheGrooming()
1331 * Vertex has these keys.
1333 * @param tmpV the tmp V
1334 * @param propHashWithKeys the prop hash with keys
1335 * @return the boolean
1337 private static Boolean vertexHasTheseKeys( Vertex tmpV, HashMap <String, Object> propHashWithKeys) {
1338 Iterator <?> it = propHashWithKeys.entrySet().iterator();
1339 while( it.hasNext() ){
1340 String propName = "";
1341 String propVal = "";
1342 Map.Entry <?,?>propEntry = (Map.Entry<?,?>)it.next();
1343 Object propNameObj = propEntry.getKey();
1344 if( propNameObj != null ){
1345 propName = propNameObj.toString();
1347 Object propValObj = propEntry.getValue();
1348 if( propValObj != null ){
1349 propVal = propValObj.toString();
1351 Object checkValObj = tmpV.<Object>property(propName).orElse(null);
1352 if( checkValObj == null ) {
1355 else if( !propVal.equals(checkValObj.toString()) ){
1364 * Any key fields missing.
1366 * @param nType the n type
1368 * @return the boolean
1370 private static Boolean anyKeyFieldsMissing(String nType, Vertex v, Loader loader) {
1373 Introspector obj = null;
1375 obj = loader.introspectorFromName(nType);
1376 } catch (AAIUnknownObjectException e) {
1377 // They gave us a non-empty nodeType but our NodeKeyProps does
1378 // not have data for it. Since we do not know what the
1379 // key params are for this type of node, we will just
1381 String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
1382 + "]. We cannot determine required keys for this nType. ";
1383 // NOTE - this will be caught below and a "false" returned
1384 throw new AAIException("AAI_6121", emsg);
1387 // Determine what the key fields are for this nodeType
1388 Collection <String> keyPropNamesColl = obj.getKeys();
1389 Iterator<String> keyPropI = keyPropNamesColl.iterator();
1390 while (keyPropI.hasNext()) {
1391 String propName = keyPropI.next();
1392 Object ob = v.<Object>property(propName).orElse(null);
1393 if (ob == null || ob.toString().equals("")) {
1394 // It is missing a key property
1398 } catch (AAIException e) {
1399 // Something was wrong -- but since we weren't able to check
1400 // the keys, we will not declare that it is missing keys.
1408 * Gets the delete list.
1410 * @param targetDir the target dir
1411 * @param fileName the file name
1412 * @param edgesOnlyFlag the edges only flag
1413 * @param dontFixOrphans the dont fix orphans
1414 * @param dupeFixOn the dupe fix on
1415 * @return the delete list
1416 * @throws AAIException the AAI exception
1418 private static Set<String> getDeleteList(String targetDir,
1419 String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
1420 Boolean dupeFixOn) throws AAIException {
1422 // Look in the file for lines formated like we expect - pull out any
1423 // Vertex Id's to delete on this run
1424 Set<String> delList = new LinkedHashSet<>();
1425 String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
1427 try(BufferedReader br = new BufferedReader(new FileReader(fullFileName))) {
1428 String line = br.readLine();
1429 while (line != null) {
1430 if (!"".equals(line) && line.startsWith("DeleteCandidate")) {
1431 if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
1432 // We're not going to process edge guys
1433 } else if (dontFixOrphans && line.contains("Orphan")) {
1434 // We're not going to process orphans
1435 } else if (!dupeFixOn && line.contains("Duplicate")) {
1436 // We're not going to process Duplicates
1438 int begIndex = line.indexOf("id = ");
1439 int endIndex = line.indexOf("]");
1440 String vidVal = line.substring(begIndex + 6, endIndex);
1441 delList.add(vidVal);
1444 line = br.readLine();
1447 } catch (IOException e) {
1448 throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
1449 + "], exception= " + e.getMessage());
1454 }// end of getDeleteList
1457 * Gets the preferred dupe.
1459 * @param transId the trans id
1460 * @param fromAppId the from app id
1462 * @param dupeVertexList the dupe vertex list
1463 * @param ver the ver
1465 * @throws AAIException the AAI exception
1467 public static Vertex getPreferredDupe(String transId,
1468 String fromAppId, GraphTraversalSource g,
1469 ArrayList<Vertex> dupeVertexList, String ver, Loader loader)
1470 throws AAIException {
1472 // This method assumes that it is being passed a List of vertex objects
1474 // violate our uniqueness constraints.
1476 Vertex nullVtx = null;
1478 if (dupeVertexList == null) {
1481 int listSize = dupeVertexList.size();
1482 if (listSize == 0) {
1485 if (listSize == 1) {
1486 return (dupeVertexList.get(0));
1489 Vertex vtxPreferred = null;
1490 Vertex currentFaveVtx = dupeVertexList.get(0);
1491 for (int i = 1; i < listSize; i++) {
1492 Vertex vtxB = dupeVertexList.get(i);
1493 vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
1494 currentFaveVtx, vtxB, ver, loader);
1495 if (vtxPreferred == null) {
1496 // We couldn't choose one
1499 currentFaveVtx = vtxPreferred;
1503 return (currentFaveVtx);
1505 } // end of getPreferredDupe()
1508 * Pick one of two dupes.
1510 * @param transId the trans id
1511 * @param fromAppId the from app id
1513 * @param vtxA the vtx A
1514 * @param vtxB the vtx B
1515 * @param ver the ver
1517 * @throws AAIException the AAI exception
1519 public static Vertex pickOneOfTwoDupes(String transId,
1520 String fromAppId, GraphTraversalSource g, Vertex vtxA,
1521 Vertex vtxB, String ver, Loader loader) throws AAIException {
1523 Vertex nullVtx = null;
1524 Vertex preferredVtx = null;
1526 Long vidA = new Long(vtxA.id().toString());
1527 Long vidB = new Long(vtxB.id().toString());
1529 String vtxANodeType = "";
1530 String vtxBNodeType = "";
1531 Object objType = vtxA.<Object>property("aai-node-type").orElse(null);
1532 if (objType != null) {
1533 vtxANodeType = objType.toString();
1535 objType = vtxB.<Object>property("aai-node-type").orElse(null);
1536 if (objType != null) {
1537 vtxBNodeType = objType.toString();
1540 if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
1541 // Either they're not really dupes or there's some bad data - so
1546 // Check that node A and B both have the same key values (or else they
1548 // (We'll check dep-node later)
1549 // Determine what the key fields are for this nodeType
1550 Collection <String> keyProps = new ArrayList <>();
1552 keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
1553 } catch (AAIUnknownObjectException e) {
1554 LOGGER.warn("Required property not found", e);
1555 throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
1558 Iterator<String> keyPropI = keyProps.iterator();
1559 while (keyPropI.hasNext()) {
1560 String propName = keyPropI.next();
1561 String vtxAKeyPropVal = "";
1562 objType = vtxA.<Object>property(propName).orElse(null);
1563 if (objType != null) {
1564 vtxAKeyPropVal = objType.toString();
1566 String vtxBKeyPropVal = "";
1567 objType = vtxB.<Object>property(propName).orElse(null);
1568 if (objType != null) {
1569 vtxBKeyPropVal = objType.toString();
1572 if (vtxAKeyPropVal.equals("")
1573 || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
1574 // Either they're not really dupes or they are missing some key
1575 // data - so don't pick one
1580 // Collect the vid's and aai-node-types of the vertices that each vertex
1581 // (A and B) is connected to.
1582 ArrayList<String> vtxIdsConn2A = new ArrayList<>();
1583 ArrayList<String> vtxIdsConn2B = new ArrayList<>();
1584 HashMap<String, String> nodeTypesConn2A = new HashMap<>();
1585 HashMap<String, String> nodeTypesConn2B = new HashMap<>();
1587 ArrayList<Vertex> vertListA = getConnectedNodes( g, vtxA );
1588 if (vertListA != null) {
1589 Iterator<Vertex> iter = vertListA.iterator();
1590 while (iter.hasNext()) {
1591 Vertex tvCon = iter.next();
1592 String conVid = tvCon.id().toString();
1594 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1595 if (objType != null) {
1596 nt = objType.toString();
1598 nodeTypesConn2A.put(nt, conVid);
1599 vtxIdsConn2A.add(conVid);
1603 ArrayList<Vertex> vertListB = getConnectedNodes( g, vtxB );
1604 if (vertListB != null) {
1605 Iterator<Vertex> iter = vertListB.iterator();
1606 while (iter.hasNext()) {
1607 Vertex tvCon = iter.next();
1608 String conVid = tvCon.id().toString();
1610 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1611 if (objType != null) {
1612 nt = objType.toString();
1614 nodeTypesConn2B.put(nt, conVid);
1615 vtxIdsConn2B.add(conVid);
1619 // 1 - If this kind of node needs a dependent node for uniqueness, then
1620 // verify that they both nodes
1621 // point to the same dependent node (otherwise they're not really
1623 // Note - there are sometimes more than one dependent node type since
1624 // one nodeType can be used in
1625 // different ways. But for a particular node, it will only have one
1626 // dependent node that it's
1628 Collection <String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
1630 if (depNodeTypes.isEmpty()) {
1631 // This kind of node is not dependent on any other. That is ok.
1633 String depNodeVtxId4A = "";
1634 String depNodeVtxId4B = "";
1635 Iterator<String> iter = depNodeTypes.iterator();
1636 while (iter.hasNext()) {
1637 String depNodeType = iter.next();
1638 if (nodeTypesConn2A.containsKey(depNodeType)) {
1639 // This is the dependent node type that vertex A is using
1640 depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
1642 if (nodeTypesConn2B.containsKey(depNodeType)) {
1643 // This is the dependent node type that vertex B is using
1644 depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
1647 if (depNodeVtxId4A.equals("")
1648 || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
1649 // Either they're not really dupes or there's some bad data - so
1650 // don't pick either one
1655 if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
1656 // 2 - If they both have edges to all the same vertices, then return
1657 // the one with the lower vertexId.
1658 boolean allTheSame = true;
1659 Iterator<String> iter = vtxIdsConn2A.iterator();
1660 while (iter.hasNext()) {
1661 String vtxIdConn2A = iter.next();
1662 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1670 preferredVtx = vtxA;
1672 preferredVtx = vtxB;
1675 } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
1676 // 3 - VertexA is connected to more things than vtxB.
1677 // We'll pick VtxA if its edges are a superset of vtxB's edges.
1678 boolean missingOne = false;
1679 Iterator<String> iter = vtxIdsConn2B.iterator();
1680 while (iter.hasNext()) {
1681 String vtxIdConn2B = iter.next();
1682 if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
1688 preferredVtx = vtxA;
1690 } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
1691 // 4 - VertexB is connected to more things than vtxA.
1692 // We'll pick VtxB if its edges are a superset of vtxA's edges.
1693 boolean missingOne = false;
1694 Iterator<String> iter = vtxIdsConn2A.iterator();
1695 while (iter.hasNext()) {
1696 String vtxIdConn2A = iter.next();
1697 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1703 preferredVtx = vtxB;
1706 preferredVtx = nullVtx;
1709 return (preferredVtx);
1711 } // end of pickOneOfTwoDupes()
1714 * Check and process dupes.
1716 * @param transId the trans id
1717 * @param fromAppId the from app id
1719 * @param version the version
1720 * @param nType the n type
1721 * @param passedVertList the passed vert list
1722 * @param dupeFixOn the dupe fix on
1723 * @param deleteCandidateList the delete candidate list
1724 * @param singleCommits the single commits
1725 * @param alreadyFoundDupeGroups the already found dupe groups
1726 * @param dbMaps the db maps
1727 * @return the array list
1729 private static List<String> checkAndProcessDupes(String transId,
1730 String fromAppId, Graph g, GraphTraversalSource source, String version, String nType,
1731 List<Vertex> passedVertList, Boolean dupeFixOn,
1732 Set<String> deleteCandidateList, Boolean singleCommits,
1733 ArrayList<String> alreadyFoundDupeGroups, Loader loader ) {
1735 ArrayList<String> returnList = new ArrayList<>();
1736 ArrayList<Vertex> checkVertList = new ArrayList<>();
1737 ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
1738 Boolean noFilterList = true;
1739 Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
1740 while (afItr.hasNext()) {
1741 String dupeGrpStr = afItr.next();
1742 String[] dupeArr = dupeGrpStr.split("\\|");
1743 int lastIndex = dupeArr.length - 1;
1744 for (int i = 0; i < lastIndex; i++) {
1745 // Note: we don't want the last one...
1746 String vidString = dupeArr[i];
1747 alreadyFoundDupeVidArr.add(vidString);
1748 noFilterList = false;
1752 // For a given set of Nodes that were found with a set of KEY
1753 // Parameters, (nodeType + key data) we will
1754 // see if we find any duplicate nodes that need to be cleaned up. Note -
1755 // it's legit to have more than one
1756 // node with the same key data if the nodes depend on a parent for
1757 // uniqueness -- as long as the two nodes
1758 // don't hang off the same Parent.
1759 // If we find duplicates, and we can figure out which of each set of
1760 // duplicates is the one that we
1761 // think should be preserved, we will record that. Whether we can tell
1762 // which one should be
1763 // preserved or not, we will return info about any sets of duplicates
1766 // Each element in the returned arrayList might look like this:
1767 // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
1768 // couldn't figure out which one to keep)
1769 // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
1770 // thought the third one was the one that should survive)
1772 // Because of the way the calling code loops over stuff, we can get the
1773 // same data multiple times - so we should
1774 // not process any vertices that we've already seen.
1777 Iterator<Vertex> pItr = passedVertList.iterator();
1778 while (pItr.hasNext()) {
1779 Vertex tvx = pItr.next();
1780 String passedId = tvx.id().toString();
1781 if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
1782 // We haven't seen this one before - so we should check it.
1783 checkVertList.add(tvx);
1787 if (checkVertList.size() < 2) {
1788 // Nothing new to check.
1792 if (loader.introspectorFromName(nType).isTopLevel()) {
1793 // If this was a node that does NOT depend on other nodes for
1794 // uniqueness, and we
1795 // found more than one node using its key -- record the found
1796 // vertices as duplicates.
1797 String dupesStr = "";
1798 for (int i = 0; i < checkVertList.size(); i++) {
1800 + ((checkVertList.get(i))).id()
1803 if (dupesStr != "") {
1804 Vertex prefV = getPreferredDupe(transId, fromAppId,
1805 source, checkVertList, version, loader);
1806 if (prefV == null) {
1807 // We could not determine which duplicate to keep
1808 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1809 returnList.add(dupesStr);
1811 dupesStr = dupesStr + "KeepVid=" + prefV.id();
1812 Boolean didRemove = false;
1814 didRemove = deleteNonKeepersIfAppropriate(g,
1815 dupesStr, prefV.id().toString(),
1816 deleteCandidateList, singleCommits);
1821 // keep them on our list
1822 returnList.add(dupesStr);
1827 // More than one node have the same key fields since they may
1828 // depend on a parent node for uniqueness. Since we're finding
1829 // more than one, we want to check to see if any of the
1830 // vertices that have this set of keys (and are the same nodeType)
1831 // are also pointing at the same 'parent' node.
1832 // Note: for a given set of key data, it is possible that there
1833 // could be more than one set of duplicates.
1834 HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
1835 transId, fromAppId, source, version, nType,
1836 checkVertList, loader);
1837 for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
1839 ArrayList<Vertex> thisParentsVertList = entry
1841 if (thisParentsVertList.size() > 1) {
1842 // More than one vertex found with the same key info
1843 // hanging off the same parent/dependent node
1844 String dupesStr = "";
1845 for (int i = 0; i < thisParentsVertList.size(); i++) {
1847 + ((thisParentsVertList
1848 .get(i))).id() + "|";
1850 if (dupesStr != "") {
1851 Vertex prefV = getPreferredDupe(transId,
1852 fromAppId, source, thisParentsVertList,
1855 if (prefV == null) {
1856 // We could not determine which duplicate to
1858 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1859 returnList.add(dupesStr);
1861 Boolean didRemove = false;
1862 dupesStr = dupesStr + "KeepVid="
1863 + prefV.id().toString();
1865 didRemove = deleteNonKeepersIfAppropriate(
1866 g, dupesStr, prefV.id()
1868 deleteCandidateList, singleCommits);
1873 // keep them on our list
1874 returnList.add(dupesStr);
1881 } catch (Exception e) {
1882 LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
1887 }// End of checkAndProcessDupes()
1890 * Group verts by dep nodes.
1892 * @param transId the trans id
1893 * @param fromAppId the from app id
1895 * @param version the version
1896 * @param nType the n type
1897 * @param passedVertList the passed vert list
1898 * @param dbMaps the db maps
1899 * @return the hash map
1900 * @throws AAIException the AAI exception
1902 private static HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
1903 String transId, String fromAppId, GraphTraversalSource g, String version,
1904 String nType, ArrayList<Vertex> passedVertList, Loader loader)
1905 throws AAIException {
1906 // Given a list of Titan Vertices of one nodeType (see AAI-8956), group
1907 // them together by the parent node they depend on.
1908 // Ie. if given a list of ip address nodes (assumed to all have the
1909 // same key info) they might sit under several different parent vertices.
1910 // Under Normal conditions, there would only be one per parent -- but
1911 // we're trying to find duplicates - so we
1912 // allow for the case where more than one is under the same parent node.
1914 HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
1915 if (loader.introspectorFromName(nType).isTopLevel()) {
1916 // This method really should not have been called if this is not the
1918 // that depends on a parent for uniqueness, so just return the empty
1923 // Find out what types of nodes the passed in nodes can depend on
1924 ArrayList<String> depNodeTypeL = new ArrayList<>();
1925 Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn();
1926 Iterator<String> ntItr = depNTColl.iterator();
1927 while (ntItr.hasNext()) {
1928 depNodeTypeL.add(ntItr.next());
1930 // For each vertex, we want find its depended-on/parent vertex so we
1931 // can track what other vertexes that are dependent on that same guy.
1932 if (passedVertList != null) {
1933 Iterator<Vertex> iter = passedVertList.iterator();
1934 while (iter.hasNext()) {
1935 Vertex thisVert = iter.next();
1936 Vertex tmpParentVtx = getConnectedParent( g, thisVert );
1937 if( tmpParentVtx != null ) {
1938 String parentNt = null;
1939 Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
1941 parentNt = obj.toString();
1943 if (depNTColl.contains(parentNt)) {
1944 // This must be the parent/dependent node
1945 String parentVid = tmpParentVtx.id().toString();
1946 if (retHash.containsKey(parentVid)) {
1947 // add this vert to the list for this parent key
1948 retHash.get(parentVid).add(thisVert);
1950 // This is the first one we found on this parent
1951 ArrayList<Vertex> vList = new ArrayList<>();
1952 vList.add(thisVert);
1953 retHash.put(parentVid, vList);
1962 }// end of groupVertsByDepNodes()
1965 * Delete non keepers if appropriate.
1968 * @param dupeInfoString the dupe info string
1969 * @param vidToKeep the vid to keep
1970 * @param deleteCandidateList the delete candidate list
1971 * @param singleCommits the single commits
1972 * @return the boolean
1974 private static Boolean deleteNonKeepersIfAppropriate(Graph g,
1975 String dupeInfoString, String vidToKeep,
1976 Set<String> deleteCandidateList, Boolean singleCommits) {
1978 Boolean deletedSomething = false;
1979 // This assumes that the dupeInfoString is in the format of
1980 // pipe-delimited vid's followed by
1981 // ie. "3456|9880|keepVid=3456"
1982 if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
1983 // No vid's on the candidate list -- so no deleting will happen on
1988 String[] dupeArr = dupeInfoString.split("\\|");
1989 ArrayList<String> idArr = new ArrayList<>();
1990 int lastIndex = dupeArr.length - 1;
1991 for (int i = 0; i <= lastIndex; i++) {
1992 if (i < lastIndex) {
1993 // This is not the last entry, it is one of the dupes,
1994 String vidString = dupeArr[i];
1995 idArr.add(vidString);
1997 // This is the last entry which should tell us if we have a
1999 String prefString = dupeArr[i];
2000 if (prefString.equals("KeepVid=UNDETERMINED")) {
2001 // They sent us a bad string -- nothing should be deleted if
2002 // no dupe could be tagged as preferred
2005 // If we know which to keep, then the prefString should look
2006 // like, "KeepVid=12345"
2007 String[] prefArr = prefString.split("=");
2008 if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
2009 LOGGER.error("Bad format. Expecting KeepVid=999999");
2012 String keepVidStr = prefArr[1];
2013 if (idArr.contains(keepVidStr)) {
2014 idArr.remove(keepVidStr);
2016 // So now, the idArr should just contain the vid's
2017 // that we want to remove.
2018 for (int x = 0; x < idArr.size(); x++) {
2019 boolean okFlag = true;
2020 String thisVid = idArr.get(x);
2021 if (deleteCandidateList.contains(thisVid)) {
2022 // This vid is a valid delete candidate from
2023 // a prev. run, so we can remove it.
2025 long longVertId = Long
2026 .parseLong(thisVid);
2028 .traversal().V(longVertId).next();
2030 if (singleCommits) {
2031 // NOTE - the singleCommits option is not used in normal processing
2033 g = AAIGraph.getInstance().getGraph().newTransaction();
2035 } catch (Exception e) {
2037 LOGGER.error("ERROR trying to delete VID = " + thisVid, e);
2040 LOGGER.info(" DELETED VID = " + thisVid);
2041 deletedSomething = true;
2046 LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
2047 + dupeInfoString + "]");
2051 }// else we know which one to keep
2053 }// for each vertex in a group
2055 return deletedSomething;
2057 }// end of deleteNonKeepersIfAppropriate()
2061 * Gets the node just using key params.
2063 * @param transId the trans id
2064 * @param fromAppId the from app id
2065 * @param graph the graph
2066 * @param nodeType the node type
2067 * @param keyPropsHash the key props hash
2068 * @param apiVersion the api version
2069 * @return the node just using key params
2070 * @throws AAIException the AAI exception
2072 public static List <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, GraphTraversalSource graph, String nodeType,
2073 HashMap<String,Object> keyPropsHash, String apiVersion ) throws AAIException{
2075 List <Vertex> retVertList = new ArrayList <> ();
2077 // We assume that all NodeTypes have at least one key-property defined.
2078 // Note - instead of key-properties (the primary key properties), a user could pass
2079 // alternate-key values if they are defined for the nodeType.
2080 List<String> kName = new ArrayList<>();
2081 List<Object> kVal = new ArrayList<>();
2082 if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
2083 throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. ");
2087 for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){
2089 kName.add(i, entry.getKey());
2090 kVal.add(i, entry.getValue());
2092 int topPropIndex = i;
2094 String propsAndValuesForMsg = "";
2095 Iterator <Vertex> verts = null;
2098 if( topPropIndex == 0 ){
2099 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
2100 verts= graph.V().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType);
2102 else if( topPropIndex == 1 ){
2103 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2104 + kName.get(1) + " = " + kVal.get(1) + ") ";
2105 verts = graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType);
2107 else if( topPropIndex == 2 ){
2108 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2109 + kName.get(1) + " = " + kVal.get(1) + ", "
2110 + kName.get(2) + " = " + kVal.get(2) + ") ";
2111 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType);
2113 else if( topPropIndex == 3 ){
2114 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2115 + kName.get(1) + " = " + kVal.get(1) + ", "
2116 + kName.get(2) + " = " + kVal.get(2) + ", "
2117 + kName.get(3) + " = " + kVal.get(3) + ") ";
2118 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType);
2121 throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
2124 catch( Exception ex ){
2125 LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex);
2128 if( verts != null ){
2129 while( verts.hasNext() ){
2131 retVertList.add(tiV);
2135 if( retVertList.size() == 0 ){
2136 LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
2137 "], propsAndVal = " + propsAndValuesForMsg );
2142 }// End of getNodeJustUsingKeyParams()
2145 * Show all edges for node.
2147 * @param transId the trans id
2148 * @param fromAppId the from app id
2149 * @param tVert the t vert
2150 * @return the array list
2152 private static ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, Vertex tVert ){
2154 ArrayList <String> retArr = new ArrayList <> ();
2155 Iterator <Edge> eI = tVert.edges(Direction.IN);
2156 if( ! eI.hasNext() ){
2157 retArr.add("No IN edges were found for this vertex. ");
2159 while( eI.hasNext() ){
2160 Edge ed = eI.next();
2161 String lab = ed.label();
2163 if (tVert.equals(ed.inVertex())) {
2164 vtx = ed.outVertex();
2166 vtx = ed.inVertex();
2169 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2172 String nType = vtx.<String>property("aai-node-type").orElse(null);
2173 String vid = vtx.id().toString();
2174 retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
2179 eI = tVert.edges(Direction.OUT);
2180 if( ! eI.hasNext() ){
2181 retArr.add("No OUT edges were found for this vertex. ");
2183 while( eI.hasNext() ){
2184 Edge ed = eI.next();
2185 String lab = ed.label();
2187 if (tVert.equals(ed.inVertex())) {
2188 vtx = ed.outVertex();
2190 vtx = ed.inVertex();
2193 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2196 String nType = vtx.<String>property("aai-node-type").orElse(null);
2197 String vid = vtx.id().toString();
2198 retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
2206 * Show properties for node.
2208 * @param transId the trans id
2209 * @param fromAppId the from app id
2210 * @param tVert the t vert
2211 * @return the array list
2213 private static ArrayList <String> showPropertiesForNode( String transId, String fromAppId, Vertex tVert ){
2215 ArrayList <String> retArr = new ArrayList <> ();
2216 if( tVert == null ){
2217 retArr.add("null Node object passed to showPropertiesForNode()\n");
2220 String nodeType = "";
2221 Object ob = tVert.<Object>property("aai-node-type").orElse(null);
2226 nodeType = ob.toString();
2229 retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
2230 retArr.add(" Property Detail: ");
2231 Iterator<VertexProperty<Object>> pI = tVert.properties();
2232 while( pI.hasNext() ){
2233 VertexProperty<Object> tp = pI.next();
2234 Object val = tp.value();
2235 retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
2242 private static ArrayList <Vertex> getConnectedNodes(GraphTraversalSource g, Vertex startVtx )
2243 throws AAIException {
2245 ArrayList <Vertex> retArr = new ArrayList <> ();
2246 if( startVtx == null ){
2250 GraphTraversal<Vertex, Vertex> modPipe = null;
2251 modPipe = g.V(startVtx).both();
2252 if( modPipe != null && modPipe.hasNext() ){
2253 while( modPipe.hasNext() ){
2254 Vertex conVert = modPipe.next();
2255 retArr.add(conVert);
2261 }// End of getConnectedNodes()
2264 private static ArrayList <Vertex> getConnectedChildrenOfOneType( GraphTraversalSource g,
2265 Vertex startVtx, String childNType ) throws AAIException{
2267 ArrayList <Vertex> childList = new ArrayList <> ();
2268 Iterator <Vertex> vertI = g.V(startVtx).union(__.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).inV(), __.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).outV());
2270 Vertex tmpVtx = null;
2271 while( vertI != null && vertI.hasNext() ){
2272 tmpVtx = vertI.next();
2273 Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
2275 String tmpNt = ob.toString();
2276 if( tmpNt.equals(childNType)){
2277 childList.add(tmpVtx);
2284 }// End of getConnectedChildrenOfOneType()
2287 private static Vertex getConnectedParent( GraphTraversalSource g,
2288 Vertex startVtx ) throws AAIException{
2290 Vertex parentVtx = null;
2291 Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
2293 while( vertI != null && vertI.hasNext() ){
2294 // Note - there better only be one!
2295 parentVtx = vertI.next();
2300 }// End of getConnectedParent()
2303 private static long figureWindowStartTime( int timeWindowMinutes ){
2304 // Given a window size, calculate what the start-timestamp would be.
2306 if( timeWindowMinutes <= 0 ){
2307 // This just means that there is no window...
2310 long unixTimeNow = System.currentTimeMillis();
2311 long windowInMillis = timeWindowMinutes * 60L * 1000;
2313 long startTimeStamp = unixTimeNow - windowInMillis;
2315 return startTimeStamp;
2316 } // End of figureWindowStartTime()