2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.aai.dbgen;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
26 import java.io.FileReader;
27 import java.io.FileWriter;
28 import java.io.IOException;
29 import java.text.SimpleDateFormat;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.Date;
33 import java.util.HashMap;
34 import java.util.Iterator;
35 import java.util.LinkedHashSet;
36 import java.util.List;
38 import java.util.Properties;
40 import java.util.TimeZone;
41 import java.util.UUID;
43 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
44 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
45 import org.apache.tinkerpop.gremlin.structure.Direction;
46 import org.apache.tinkerpop.gremlin.structure.Edge;
47 import org.apache.tinkerpop.gremlin.structure.Property;
48 import org.apache.tinkerpop.gremlin.structure.Vertex;
49 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
51 import org.openecomp.aai.dbmap.AAIGraph;
52 import org.openecomp.aai.exceptions.AAIException;
53 import org.openecomp.aai.ingestModel.DbMaps;
54 import org.openecomp.aai.ingestModel.IngestModelMoxyOxm;
55 import org.openecomp.aai.logging.ErrorLogHelper;
56 import org.openecomp.aai.util.AAIConfig;
57 import org.openecomp.aai.util.AAIConstants;
58 import com.att.eelf.configuration.Configuration;
59 import com.att.eelf.configuration.EELFLogger;
60 import com.att.eelf.configuration.EELFManager;
61 import com.thinkaurelius.titan.core.TitanEdge;
62 import com.thinkaurelius.titan.core.TitanFactory;
63 import com.thinkaurelius.titan.core.TitanGraph;
64 import com.thinkaurelius.titan.core.TitanTransaction;
65 import com.thinkaurelius.titan.core.TitanVertex;
68 public class DataGrooming {
70 private static EELFLogger LOGGER;
71 private static final String FROMAPPID = "AAI-DB";
72 private static final String TRANSID = UUID.randomUUID().toString();
73 private static int dupeGrpsDeleted = 0;
78 * @param args the arguments
80 public static void main(String[] args) {
82 // Set the logging file properties to be used by EELFManager
83 Properties props = System.getProperties();
84 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS);
85 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
86 LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
87 String ver = "version"; // Placeholder
88 Boolean doAutoFix = false;
89 Boolean edgesOnlyFlag = false;
90 Boolean dontFixOrphansFlag = false;
91 Boolean singleCommits = false;
92 Boolean dupeCheckOff = false;
93 Boolean dupeFixOn = false;
94 Boolean ghost2CheckOff = false;
95 Boolean ghost2FixOn = false;
96 Boolean neverUseCache = false;
98 int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
99 int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
101 String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
102 if( maxFixStr != null && !maxFixStr.equals("") ){
103 maxRecordsToFix = Integer.parseInt(maxFixStr);
105 String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
106 if( sleepStr != null && !sleepStr.equals("") ){
107 sleepMinutes = Integer.parseInt(sleepStr);
110 catch ( Exception e ){
111 // Don't worry, we'll just use the defaults that we got from AAIConstants
112 LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
115 String prevFileName = "";
117 SimpleDateFormat d = new SimpleDateFormat("yyyyMMddHHmm");
118 d.setTimeZone(TimeZone.getTimeZone("GMT"));
119 String dteStr = d.format(new Date()).toString();
120 String groomOutFileName = "dataGrooming." + dteStr + ".out";
122 if (args.length > 0) {
123 // They passed some arguments in that will affect processing
124 for (int i = 0; i < args.length; i++) {
125 String thisArg = args[i];
126 if (thisArg.equals("-edgesOnly")) {
127 edgesOnlyFlag = true;
128 } else if (thisArg.equals("-autoFix")) {
130 } else if (thisArg.equals("-dontFixOrphans")) {
131 dontFixOrphansFlag = true;
132 } else if (thisArg.equals("-singleCommits")) {
133 singleCommits = true;
134 } else if (thisArg.equals("-dupeCheckOff")) {
136 } else if (thisArg.equals("-dupeFixOn")) {
138 } else if (thisArg.equals("-ghost2CheckOff")) {
139 ghost2CheckOff = true;
140 } else if (thisArg.equals("-neverUseCache")) {
141 neverUseCache = true;
142 } else if (thisArg.equals("-ghost2FixOn")) {
144 } else if (thisArg.equals("-maxFix")) {
146 if (i >= args.length) {
147 LOGGER.error(" No value passed with -maxFix option. ");
150 String nextArg = args[i];
152 maxRecordsToFix = Integer.parseInt(nextArg);
153 } catch (Exception e) {
154 LOGGER.error("Bad value passed with -maxFix option: ["
158 } else if (thisArg.equals("-sleepMinutes")) {
160 if (i >= args.length) {
161 LOGGER.error("No value passed with -sleepMinutes option.");
164 String nextArg = args[i];
166 sleepMinutes = Integer.parseInt(nextArg);
167 } catch (Exception e) {
168 LOGGER.error("Bad value passed with -sleepMinutes option: ["
172 } else if (thisArg.equals("-f")) {
174 if (i >= args.length) {
175 LOGGER.error(" No value passed with -f option. ");
178 prevFileName = args[i];
180 LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
182 LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache");
189 IngestModelMoxyOxm moxyMod = new IngestModelMoxyOxm();
191 ArrayList <String> defaultVerLst = new ArrayList <> ();
192 defaultVerLst.add( AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP) );
193 moxyMod.init( defaultVerLst, false);
195 catch (Exception ex){
196 LOGGER.error("ERROR - Could not do the moxyMod.init()", ex);
201 if (!prevFileName.equals("")) {
202 // They are trying to fix some data based on a data in a
204 LOGGER.info(" Call doTheGrooming() with a previous fileName ["
205 + prevFileName + "] for cleanup. ");
206 Boolean finalShutdownFlag = true;
207 Boolean cacheDbOkFlag = false;
208 doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
209 maxRecordsToFix, groomOutFileName, ver, singleCommits,
210 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
211 finalShutdownFlag, cacheDbOkFlag);
212 } else if (doAutoFix) {
213 // They want us to run the processing twice -- first to look for
214 // delete candidates, then after
215 // napping for a while, run it again and delete any candidates
216 // that were found by the first run.
217 // Note: we will produce a separate output file for each of the
219 LOGGER.info(" Doing an auto-fix call to Grooming. ");
220 LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
221 Boolean finalShutdownFlag = false;
222 Boolean cacheDbOkFlag = true;
223 int fixCandCount = doTheGrooming("", edgesOnlyFlag,
224 dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
225 ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
226 finalShutdownFlag, cacheDbOkFlag);
227 if (fixCandCount == 0) {
228 LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
230 // We'll sleep a little and then run a fix-pass based on the
231 // first-run's output file.
233 LOGGER.info("About to sleep for " + sleepMinutes
235 int sleepMsec = sleepMinutes * 60 * 1000;
236 Thread.sleep(sleepMsec);
237 } catch (InterruptedException ie) {
238 LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
242 d = new SimpleDateFormat("yyyyMMddHHmm");
243 d.setTimeZone(TimeZone.getTimeZone("GMT"));
244 dteStr = d.format(new Date()).toString();
245 String secondGroomOutFileName = "dataGrooming." + dteStr
247 LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
248 + "generated by the first pass for fixing: ["
249 + groomOutFileName + "]");
250 finalShutdownFlag = true;
251 cacheDbOkFlag = false;
252 doTheGrooming(groomOutFileName, edgesOnlyFlag,
253 dontFixOrphansFlag, maxRecordsToFix,
254 secondGroomOutFileName, ver, singleCommits,
255 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
256 finalShutdownFlag, cacheDbOkFlag);
259 // Do the grooming - plain vanilla (no fix-it-file, no
261 Boolean finalShutdownFlag = true;
262 LOGGER.info(" Call doTheGrooming() ");
263 Boolean cacheDbOkFlag = true;
265 // They have forbidden us from using a cached db connection.
266 cacheDbOkFlag = false;
268 doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
269 maxRecordsToFix, groomOutFileName, ver, singleCommits,
270 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
271 finalShutdownFlag, cacheDbOkFlag);
273 } catch (Exception ex) {
274 LOGGER.error("Exception while grooming data", ex);
277 LOGGER.info(" Done! ");
285 * @param fileNameForFixing the file name for fixing
286 * @param edgesOnlyFlag the edges only flag
287 * @param dontFixOrphansFlag the dont fix orphans flag
288 * @param maxRecordsToFix the max records to fix
289 * @param groomOutFileName the groom out file name
290 * @param version the version
291 * @param singleCommits the single commits
292 * @param dupeCheckOff the dupe check off
293 * @param dupeFixOn the dupe fix on
294 * @param ghost2CheckOff the ghost 2 check off
295 * @param ghost2FixOn the ghost 2 fix on
296 * @param finalShutdownFlag the final shutdown flag
297 * @param cacheDbOkFlag the cacheDbOk flag
300 private static int doTheGrooming(String fileNameForFixing,
301 Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
302 int maxRecordsToFix, String groomOutFileName, String version,
303 Boolean singleCommits,
304 Boolean dupeCheckOff, Boolean dupeFixOn,
305 Boolean ghost2CheckOff, Boolean ghost2FixOn,
306 Boolean finalShutdownFlag, Boolean cacheDbOkFlag) {
308 LOGGER.debug(" Entering doTheGrooming \n");
310 int cleanupCandidateCount = 0;
311 BufferedWriter bw = null;
312 TitanGraph graph = null;
313 TitanGraph graph2 = null;
315 boolean executeFinalCommit = false;
316 Set<String> deleteCandidateList = new LinkedHashSet<>();
317 Set<String> processedVertices = new LinkedHashSet<>();
318 TitanTransaction g = null;
319 TitanTransaction g2 = null;
322 String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
323 + "logs" + AAIConstants.AAI_FILESEP + "data"
324 + AAIConstants.AAI_FILESEP + "dataGrooming";
326 // Make sure the target directory exists
327 new File(targetDir).mkdirs();
329 if (!fileNameForFixing.equals("")) {
330 deleteCandidateList = getDeleteList(targetDir,
331 fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
335 if (deleteCandidateList.size() > maxRecordsToFix) {
336 LOGGER.warn(" >> WARNING >> Delete candidate list size ("
337 + deleteCandidateList.size()
338 + ") is too big. The maxFix we are using is: "
340 + ". No candidates will be deleted. ");
341 // Clear out the list so it won't be processed below.
342 deleteCandidateList = new LinkedHashSet<>();
345 SimpleDateFormat d = new SimpleDateFormat("yyyyMMddHHmm");
346 d.setTimeZone(TimeZone.getTimeZone("GMT"));
348 String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
350 File groomOutFile = new File(fullOutputFileName);
352 groomOutFile.createNewFile();
353 } catch (IOException e) {
354 String emsg = " Problem creating output file ["
355 + fullOutputFileName + "], exception=" + e.getMessage();
356 throw new AAIException("AAI_6124", emsg);
359 LOGGER.info(" Will write to " + fullOutputFileName );
360 FileWriter fw = new FileWriter(groomOutFile.getAbsoluteFile());
361 bw = new BufferedWriter(fw);
362 ErrorLogHelper.loadProperties();
364 LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n");
367 // Since we're just reading (not deleting/fixing anything), we can use
368 // a cached connection to the DB
369 graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG);
372 graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
375 String emsg = "null graph object in DataGrooming\n";
376 throw new AAIException("AAI_6101", emsg);
379 LOGGER.debug(" Got the graph object. ");
381 g = graph.newTransaction();
383 String emsg = "null graphTransaction object in DataGrooming\n";
384 throw new AAIException("AAI_6101", emsg);
388 ArrayList<String> errArr = new ArrayList<>();
389 int totalNodeCount = 0;
390 HashMap<String, String> misMatchedHash = new HashMap<String, String>();
391 HashMap<String, TitanVertex> orphanNodeHash = new HashMap<String, TitanVertex>();
392 HashMap<String, TitanVertex> missingDepNodeHash = new HashMap<String, TitanVertex>();
393 HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
394 HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
395 HashMap<String, TitanVertex> ghostNodeHash = new HashMap<String, TitanVertex>();
396 ArrayList<String> dupeGroups = new ArrayList<>();
399 DbMaps dbMaps = IngestModelMoxyOxm.dbMapsContainer.get(AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP));
401 Iterator<String> nodeMapKPropsIterator = dbMaps.NodeKeyProps.keySet().iterator();
404 LOGGER.info(" Starting DataGrooming Processing ");
407 LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
410 while (nodeMapKPropsIterator.hasNext()) {
411 String nType = nodeMapKPropsIterator.next();
413 int thisNtDeleteCount = 0;
414 LOGGER.debug(" > Look at : [" + nType + "] ...");
415 ntList = ntList + "," + nType;
417 // Get a collection of the names of the key properties for this nodeType to use later
418 // Determine what the key fields are for this nodeType
419 Collection <String> keyProps = new ArrayList <>();
420 if( dbMaps.NodeKeyProps.containsKey(nType) ){
421 keyProps = dbMaps.NodeKeyProps.get(nType);
424 throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + nType + ")");
427 // Get the types of nodes that this nodetype depends on for uniqueness (if any)
428 Collection <String> depNodeTypes = new ArrayList <>();
429 if( dbMaps.NodeDependencies.containsKey(nType) ){
430 depNodeTypes = dbMaps.NodeDependencies.get(nType);
433 // Loop through all the nodes of this Node type
434 int lastShownForNt = 0;
435 ArrayList <TitanVertex> tmpList = new ArrayList <> ();
436 Iterable <?> verts = g.query().has("aai-node-type",nType).vertices();
437 Iterator<?> iterv = verts.iterator();
438 while (iterv.hasNext()) {
439 // We put the nodes into an ArrayList because the graph.query iterator can time out
440 tmpList.add((TitanVertex)iterv.next());
443 Iterator <?> iter = tmpList.iterator();
444 while (iter.hasNext()) {
447 if( thisNtCount == lastShownForNt + 250 ){
448 lastShownForNt = thisNtCount;
449 LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
451 TitanVertex thisVtx = (TitanVertex) iter.next();
452 String thisVid = thisVtx.id().toString();
453 if (processedVertices.contains(thisVid)) {
454 LOGGER.debug("skipping already processed vertex: " + thisVid);
458 List <TitanVertex> secondGetList = new ArrayList <> ();
459 // -----------------------------------------------------------------------
460 // For each vertex of this nodeType, we want to:
461 // a) make sure that it can be retrieved using it's AAI defined key
462 // b) make sure that it is not a duplicate
463 // -----------------------------------------------------------------------
465 // For this instance of this nodeType, get the key properties
466 HashMap<String, Object> propHashWithKeys = new HashMap<>();
467 Iterator<String> keyPropI = keyProps.iterator();
468 while (keyPropI.hasNext()) {
469 String propName = keyPropI.next();
471 //delete an already deleted vertex
472 Object obj = thisVtx.<Object>property(propName).orElse(null);
474 propVal = obj.toString();
476 propHashWithKeys.put(propName, propVal);
479 // If this node is dependent on another for uniqueness, then do the query from that parent node
480 // Note - all of our nodes that are dependent on others for uniqueness are
481 // "children" of that node.
482 boolean depNodeOk = true;
483 if( depNodeTypes.isEmpty() ){
484 // This kind of node is not dependent on any other.
485 // Make sure we can get it back using it's key properties and that we only get one.
486 secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, g, nType,
487 propHashWithKeys, version );
490 // This kind of node is dependent on another for uniqueness.
491 // Start at it's parent (the dependent vertex) and make sure we can get it
492 // back using it's key properties and that we only get one.
493 Iterable <?> verts2 = thisVtx.query().direction(Direction.IN).has("isParent",true).vertices();
494 Iterator <?> vertI2 = verts2.iterator();
495 TitanVertex parentVtx = null;
497 while( vertI2 != null && vertI2.hasNext() ){
498 parentVtx = (TitanVertex) vertI2.next();
504 //List<Vertex> vertI2 = g.traversal().V(thisVtx).union(__.outE().has("isParent-REV",true).outV(),__.inE().has("isParent",true).inV()).toList();
505 //if( vertI2.isEmpty()){
507 // It's Missing it's dependent/parent node
509 boolean zeroEdges = false;
511 Iterator<Edge> tmpEdgeIter = thisVtx.edges(Direction.BOTH);
513 while( tmpEdgeIter.hasNext() ){
517 if( edgeCount == 0 ){
520 } catch (Exception ex) {
521 LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex);
524 if (deleteCandidateList.contains(thisVid)) {
525 boolean okFlag = true;
527 processedVertices.add(thisVtx.id().toString());
531 } catch (Exception e) {
533 LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e);
536 LOGGER.info(" DELETED missing-dep-node VID = " + thisVid);
539 // We count nodes missing their depNodes two ways - the first if it has
540 // at least some edges, and the second if it has zero edges. Either
541 // way, they are effectively orphaned.
542 // NOTE - Only nodes that have dependent nodes are ever considered "orphaned".
544 missingDepNodeHash.put(thisVid, thisVtx);
547 orphanNodeHash.put(thisVid, thisVtx);
551 else if ( pCount > 1 ){
552 // Not sure how this could happen? Should we do something here?
556 // We found the parent - so use it to do the second-look.
557 // NOTE --- We're just going to do the same check from the other direction - because
558 // there could be duplicates or the pointer going the other way could be broken
559 ArrayList <TitanVertex> tmpListSec = new ArrayList <> ();
561 tmpListSec = getConnectedChildrenOfOneType( g, parentVtx, nType ) ;
562 Iterator<TitanVertex> vIter = tmpListSec.iterator();
563 while (vIter.hasNext()) {
564 TitanVertex tmpV = vIter.next();
565 if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
566 secondGetList.add(tmpV);
572 if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
573 // We could not get the node back using it's own key info.
574 // So, it's a PHANTOM
575 if (deleteCandidateList.contains(thisVid)) {
576 boolean okFlag = true;
581 } catch (Exception e) {
583 LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e);
586 LOGGER.info(" DELETED VID = " + thisVid);
589 ghostNodeHash.put(thisVid, thisVtx);
592 else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
593 // Found some DUPLICATES - need to process them
594 LOGGER.info(" - now check Dupes for this guy - ");
595 List<String> tmpDupeGroups = checkAndProcessDupes(
596 TRANSID, FROMAPPID, g, version,
597 nType, secondGetList, dupeFixOn,
598 deleteCandidateList, singleCommits, dupeGroups, dbMaps);
599 Iterator<String> dIter = tmpDupeGroups.iterator();
600 while (dIter.hasNext()) {
601 // Add in any newly found dupes to our running list
602 String tmpGrp = dIter.next();
603 LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
604 dupeGroups.add(tmpGrp);
608 catch (AAIException e1) {
609 LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
610 errArr.add(e1.getErrorObject().toString());
612 catch (Exception e2) {
613 LOGGER.warn(" For nodeType = " + nType
614 + " Caught exception", e2);
615 errArr.add(e2.getMessage());
617 }// try block to enclose looping of a single vertex
618 catch (Exception exx) {
619 LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
622 } // while loop for each record of a nodeType
624 if ( (thisNtDeleteCount > 0) && singleCommits ) {
625 // NOTE - the singleCommits option is not used in normal processing
627 g = AAIGraph.getInstance().getGraph().newTransaction();
630 thisNtDeleteCount = 0;
631 LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " );
633 }// While-loop for each node type
634 }// end of check to make sure we weren't only supposed to do edges
637 // --------------------------------------------------------------------------------------
638 // Now, we're going to look for one-armed-edges. Ie. an edge that
640 // been deleted (because a vertex on one side was deleted) but
641 // somehow was not deleted.
642 // So the one end of it points to a vertexId -- but that vertex is
644 // --------------------------------------------------------------------------------------
646 // To do some strange checking - we need a second graph object
647 LOGGER.debug(" ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
648 // Note - graph2 just reads - but we want it to use a fresh connection to
649 // the database, so we are NOT using the CACHED DB CONFIG here.
650 graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
651 if (graph2 == null) {
652 String emsg = "null graph2 object in DataGrooming\n";
653 throw new AAIException("AAI_6101", emsg);
655 LOGGER.debug("Got the graph2 object... \n");
657 g2 = graph2.newTransaction();
659 String emsg = "null graphTransaction2 object in DataGrooming\n";
660 throw new AAIException("AAI_6101", emsg);
663 ArrayList<Vertex> vertList = new ArrayList<>();
664 Iterable<? extends Vertex> vIt3 = g.query().vertices();
665 Iterator<? extends Vertex> vItor3 = vIt3.iterator();
666 // Gotta hold these in a List - or else HBase times out as you cycle
668 while (vItor3.hasNext()) {
669 Vertex v = vItor3.next();
674 Iterator<Vertex> vItor2 = vertList.iterator();
675 LOGGER.info(" Checking for bad edges --- ");
677 while (vItor2.hasNext()) {
682 } catch (Exception vex) {
683 LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
688 String thisVertId = "";
690 thisVertId = v.id().toString();
691 } catch (Exception ev) {
692 LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. ");
695 if (ghostNodeHash.containsKey(thisVertId)) {
696 // This is a phantom node, so don't try to use it
697 LOGGER.info(" >> Skipping edge check for edges from vertexId = "
699 + ", since that guy is a Phantom Node");
702 if (counter == lastShown + 250) {
704 LOGGER.info("... Checking edges for vertex # "
707 Iterator<Edge> eItor = v.edges(Direction.BOTH);
708 while (eItor.hasNext()) {
714 } catch (Exception iex) {
715 LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
721 } catch (Exception err) {
722 LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
726 TitanVertex ghost2 = null;
728 Boolean keysMissing = true;
729 Boolean cantGetUsingVid = false;
732 Object ob = vIn.<Object>property("aai-node-type").orElse(null);
734 vNtI = ob.toString();
735 keysMissing = anyKeyFieldsMissing(vNtI, vIn, dbMaps);
740 vIdI = ob.toString();
741 vIdLong = Long.parseLong(vIdI);
744 if( ! ghost2CheckOff ){
745 TitanVertex connectedVert = g2.getVertex(vIdLong);
746 if( connectedVert == null ) {
747 LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
748 cantGetUsingVid = true;
750 // If we can NOT get this ghost with the SECOND graph-object,
751 // it is still a ghost since even though we can get data about it using the FIRST graph
754 ghost2 = g.getVertex(vIdLong);
756 catch( Exception ex){
757 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
759 if( ghost2 != null ){
760 ghostNodeHash.put(vIdI, ghost2);
763 }// end of the ghost2 checking
765 catch (Exception err) {
766 LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
769 if (keysMissing || vIn == null || vNtI.equals("")
770 || cantGetUsingVid) {
771 // this is a bad edge because it points to a vertex
772 // that isn't there anymore or is corrupted
773 String thisEid = e.id().toString();
774 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
775 boolean okFlag = true;
776 if (!vIdI.equals("")) {
777 // try to get rid of the corrupted vertex
779 if( (ghost2 != null) && ghost2FixOn ){
786 // NOTE - the singleCommits option is not used in normal processing
788 g = AAIGraph.getInstance().getGraph().newTransaction();
791 } catch (Exception e1) {
793 LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
797 LOGGER.info(" DELETED vertex from bad edge = "
801 // remove the edge if we couldn't get the
806 // NOTE - the singleCommits option is not used in normal processing
808 g = AAIGraph.getInstance().getGraph().newTransaction();
811 } catch (Exception ex) {
812 // NOTE - often, the exception is just
813 // that this edge has already been
816 LOGGER.warn("WARNING when trying to delete edge = "
820 LOGGER.info(" DELETED edge = " + thisEid);
824 oneArmedEdgeHash.put(thisEid, e);
825 if ((vIn != null) && (vIn.id() != null)) {
826 emptyVertexHash.put(thisEid, vIn.id()
833 vOut = e.outVertex();
834 } catch (Exception err) {
835 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
841 cantGetUsingVid = false;
844 Object ob = vOut.<Object>property("aai-node-type").orElse(null);
846 vNtO = ob.toString();
847 keysMissing = anyKeyFieldsMissing(vNtO,
853 vIdO = ob.toString();
854 vIdLong = Long.parseLong(vIdO);
857 if( ! ghost2CheckOff ){
858 TitanVertex connectedVert = g2.getVertex(vIdLong);
859 if( connectedVert == null ) {
860 cantGetUsingVid = true;
861 LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
862 // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
864 ghost2 = g.getVertex(vIdLong);
866 catch( Exception ex){
867 LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
869 if( ghost2 != null ){
870 ghostNodeHash.put(vIdO, ghost2);
874 } catch (Exception err) {
875 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
878 if (keysMissing || vOut == null || vNtO.equals("")
879 || cantGetUsingVid) {
880 // this is a bad edge because it points to a vertex
881 // that isn't there anymore
882 String thisEid = e.id().toString();
883 if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
884 boolean okFlag = true;
885 if (!vIdO.equals("")) {
886 // try to get rid of the corrupted vertex
888 if( (ghost2 != null) && ghost2FixOn ){
895 // NOTE - the singleCommits option is not used in normal processing
897 g = AAIGraph.getInstance().getGraph().newTransaction();
900 } catch (Exception e1) {
902 LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
906 LOGGER.info(" DELETED vertex from bad edge = "
910 // remove the edge if we couldn't get the
915 // NOTE - the singleCommits option is not used in normal processing
917 g = AAIGraph.getInstance().getGraph().newTransaction();
920 } catch (Exception ex) {
921 // NOTE - often, the exception is just
922 // that this edge has already been
925 LOGGER.warn("WARNING when trying to delete edge = "
929 LOGGER.info(" DELETED edge = " + thisEid);
933 oneArmedEdgeHash.put(thisEid, e);
934 if ((vOut != null) && (vOut.id() != null)) {
935 emptyVertexHash.put(thisEid, vOut.id()
940 }// End of while-edges-loop
941 } catch (Exception exx) {
942 LOGGER.warn("WARNING from in the while-verts-loop ", exx);
944 }// End of while-vertices-loop
946 deleteCount = deleteCount + dupeGrpsDeleted;
947 if (!singleCommits && deleteCount > 0) {
949 LOGGER.info("About to do the commit for "
950 + deleteCount + " removes. ");
951 executeFinalCommit = true;
952 LOGGER.info("Commit was successful ");
953 } catch (Exception excom) {
954 LOGGER.error(" >>>> ERROR <<<< Could not commit changes. ", excom);
959 int ghostNodeCount = ghostNodeHash.size();
960 int orphanNodeCount = orphanNodeHash.size();
961 int missingDepNodeCount = missingDepNodeHash.size();
962 int oneArmedEdgeCount = oneArmedEdgeHash.size();
963 int dupeCount = dupeGroups.size();
965 deleteCount = deleteCount + dupeGrpsDeleted;
967 bw.write("\n\n ============ Summary ==============\n");
968 bw.write("Ran these nodeTypes: " + ntList + "\n\n");
969 bw.write("There were this many delete candidates from previous run = "
970 + deleteCandidateList.size() + "\n");
971 if (dontFixOrphansFlag) {
972 bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
974 bw.write("Deleted this many delete candidates = " + deleteCount
976 bw.write("Total number of nodes looked at = " + totalNodeCount
978 bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
979 bw.write("Orphan Nodes identified = " + orphanNodeCount + "\n");
980 bw.write("Bad Edges identified = " + oneArmedEdgeCount + "\n");
981 bw.write("Missing Dependent Edge (but not orphaned) node count = "
982 + missingDepNodeCount + "\n");
983 bw.write("Duplicate Groups count = " + dupeCount + "\n");
984 bw.write("MisMatching Label/aai-node-type count = "
985 + misMatchedHash.size() + "\n");
987 bw.write("\n ------------- Delete Candidates ---------\n");
988 for (Map.Entry<String, TitanVertex> entry : ghostNodeHash
990 String vid = entry.getKey();
991 bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
992 cleanupCandidateCount++;
994 for (Map.Entry<String, TitanVertex> entry : orphanNodeHash
996 String vid = entry.getKey();
997 bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
998 if (!dontFixOrphansFlag) {
999 cleanupCandidateCount++;
1002 for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1003 String eid = entry.getKey();
1004 bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
1005 cleanupCandidateCount++;
1007 for (Map.Entry<String, TitanVertex> entry : missingDepNodeHash
1009 String vid = entry.getKey();
1010 bw.write("DeleteCandidate: (maybe) missingDepNode Vid = ["
1012 cleanupCandidateCount++;
1014 bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
1016 bw.write("\n ------------- GHOST NODES - detail ");
1017 for (Map.Entry<String, TitanVertex> entry : ghostNodeHash
1020 String vid = entry.getKey();
1021 bw.write("\n ==> Phantom Vid = " + vid + "\n");
1022 ArrayList<String> retArr = showPropertiesForNode(
1023 TRANSID, FROMAPPID, entry.getValue());
1024 for (String info : retArr) {
1025 bw.write(info + "\n");
1028 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1030 for (String info : retArr) {
1031 bw.write(info + "\n");
1033 } catch (Exception dex) {
1034 LOGGER.error("error trying to print detail info for a ghost-node: ", dex);
1038 bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
1039 for (Map.Entry<String, TitanVertex> entry : orphanNodeHash
1042 String vid = entry.getKey();
1043 bw.write("\n> Orphan Node Vid = " + vid + "\n");
1044 ArrayList<String> retArr = showPropertiesForNode(
1045 TRANSID, FROMAPPID, entry.getValue());
1046 for (String info : retArr) {
1047 bw.write(info + "\n");
1050 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1052 for (String info : retArr) {
1053 bw.write(info + "\n");
1055 } catch (Exception dex) {
1056 LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex);
1060 bw.write("\n ------------- Missing Dependent Edge (but not orphan) NODES: ");
1061 for (Map.Entry<String, TitanVertex> entry : missingDepNodeHash
1064 String vid = entry.getKey();
1065 bw.write("\n> Missing edge to Dependent Node (but has edges) Vid = "
1067 ArrayList<String> retArr = showPropertiesForNode(
1068 TRANSID, FROMAPPID, entry.getValue());
1069 for (String info : retArr) {
1070 bw.write(info + "\n");
1073 retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1075 for (String info : retArr) {
1076 bw.write(info + "\n");
1078 } catch (Exception dex) {
1079 LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex);
1083 bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
1084 for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1086 String eid = entry.getKey();
1087 Edge thisE = entry.getValue();
1088 String badVid = emptyVertexHash.get(eid);
1089 bw.write("\n> Edge pointing to bad vertex (Vid = "
1090 + badVid + ") EdgeId = " + eid + "\n");
1091 bw.write("Label: [" + thisE.label() + "]\n");
1092 Iterator<Property<Object>> pI = thisE.properties();
1093 while (pI.hasNext()) {
1094 Property<Object> propKey = pI.next();
1095 bw.write("Prop: [" + propKey + "], val = ["
1096 + propKey.value() + "]\n");
1098 } catch (Exception pex) {
1099 LOGGER.error("error trying to print empty/bad vertex data: ", pex);
1103 bw.write("\n ------------- Duplicates: ");
1104 Iterator<String> dupeIter = dupeGroups.iterator();
1105 int dupeSetCounter = 0;
1106 while (dupeIter.hasNext()) {
1108 String dset = (String) dupeIter.next();
1110 bw.write("\n --- Duplicate Group # " + dupeSetCounter
1111 + " Detail -----------\n");
1113 // We expect each line to have at least two vid's, followed
1114 // by the preferred one to KEEP
1115 String[] dupeArr = dset.split("\\|");
1116 ArrayList<String> idArr = new ArrayList<>();
1117 int lastIndex = dupeArr.length - 1;
1118 for (int i = 0; i <= lastIndex; i++) {
1119 if (i < lastIndex) {
1120 // This is not the last entry, it is one of the
1121 // dupes, so we want to show all its info
1122 bw.write(" >> Duplicate Group # "
1123 + dupeSetCounter + " Node # " + i
1125 String vidString = dupeArr[i];
1126 idArr.add(vidString);
1127 long longVertId = Long.parseLong(vidString);
1128 Iterator<Vertex> vtxIterator = g.vertices(longVertId);
1129 TitanVertex vtx = null;
1130 if (vtxIterator.hasNext()) {
1131 vtx = (TitanVertex)vtxIterator.next();
1133 ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
1134 for (String info : retArr) {
1135 bw.write(info + "\n");
1138 retArr = showAllEdgesForNode(TRANSID,
1140 for (String info : retArr) {
1141 bw.write(info + "\n");
1144 // This is the last entry which should tell us if we
1145 // have a preferred keeper
1146 String prefString = dupeArr[i];
1147 if (prefString.equals("KeepVid=UNDETERMINED")) {
1148 bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
1149 bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
1151 // If we know which to keep, then the prefString
1152 // should look like, "KeepVid=12345"
1153 String[] prefArr = prefString.split("=");
1154 if (prefArr.length != 2
1155 || (!prefArr[0].equals("KeepVid"))) {
1156 throw new Exception("Bad format. Expecting KeepVid=999999");
1158 String keepVidStr = prefArr[1];
1159 if (idArr.contains(keepVidStr)) {
1160 bw.write("\n The vertex we want to KEEP has vertexId = "
1162 bw.write("\n The others become delete candidates: \n");
1163 idArr.remove(keepVidStr);
1164 for (int x = 0; x < idArr.size(); x++) {
1165 cleanupCandidateCount++;
1166 bw.write("DeleteCandidate: Duplicate Vid = ["
1167 + idArr.get(x) + "]\n");
1170 throw new Exception("ERROR - Vertex Id to keep not found in list of dupes. dset = ["
1174 }// else we know which one to keep
1176 }// for each vertex in a group
1177 } catch (Exception dex) {
1178 LOGGER.error("error trying to print duplicate vertex data", dex);
1181 }// while - work on each group of dupes
1183 bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
1184 for (Map.Entry<String, String> entry : misMatchedHash.entrySet()) {
1185 String msg = entry.getValue();
1186 bw.write("MixedMsg = " + msg + "\n");
1189 bw.write("\n ------------- Got these errors while processing: \n");
1190 Iterator<String> errIter = errArr.iterator();
1191 while (errIter.hasNext()) {
1192 String line = (String) errIter.next();
1193 bw.write(line + "\n");
1198 LOGGER.info("\n ------------- Done doing all the checks ------------ ");
1199 LOGGER.info("Output will be written to " + fullOutputFileName);
1201 if (cleanupCandidateCount > 0) {
1202 // Technically, this is not an error -- but we're throwing this
1203 // error so that hopefully a
1204 // monitoring system will pick it up and do something with it.
1205 throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
1206 + "] and investigate delete candidates. ");
1208 } catch (AAIException e) {
1209 LOGGER.error("Caught AAIException while grooming data", e);
1210 ErrorLogHelper.logException(e);
1211 } catch (Exception ex) {
1212 LOGGER.error("Caught exception while grooming data", ex);
1213 ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
1219 } catch (IOException iox) {
1220 LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
1224 if (g != null && !g.isClosed()) {
1225 // Any changes that worked correctly should have already done
1228 if (executeFinalCommit) {
1232 } catch (Exception ex) {
1233 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1234 LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
1238 if (g2 != null && !g2.isClosed()) {
1239 // Any changes that worked correctly should have already done
1243 } catch (Exception ex) {
1244 // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1245 LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
1249 if( finalShutdownFlag ){
1251 if( graph != null && graph.isOpen() ){
1255 } catch (Exception ex) {
1256 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1257 LOGGER.warn("WARNING from final graph.shutdown()", ex);
1261 if( graph2 != null && graph2.isOpen() ){
1262 graph2.tx().close();
1265 } catch (Exception ex) {
1266 // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1267 LOGGER.warn("WARNING from final graph2.shutdown()", ex);
1273 return cleanupCandidateCount;
1275 }// end of doTheGrooming()
1279 * Vertex has these keys.
1281 * @param tmpV the tmp V
1282 * @param propHashWithKeys the prop hash with keys
1283 * @return the boolean
1285 private static Boolean vertexHasTheseKeys( TitanVertex tmpV, HashMap <String, Object> propHashWithKeys) {
1286 Iterator <?> it = propHashWithKeys.entrySet().iterator();
1287 while( it.hasNext() ){
1288 String propName = "";
1289 String propVal = "";
1290 Map.Entry <?,?>propEntry = (Map.Entry<?,?>)it.next();
1291 Object propNameObj = propEntry.getKey();
1292 if( propNameObj != null ){
1293 propName = propNameObj.toString();
1295 Object propValObj = propEntry.getValue();
1296 if( propValObj != null ){
1297 propVal = propValObj.toString();
1299 Object checkValObj = tmpV.<Object>property(propName).orElse(null);
1300 if( checkValObj == null ) {
1303 else if( !propVal.equals(checkValObj.toString()) ){
1312 * Any key fields missing.
1314 * @param nType the n type
1316 * @return the boolean
1318 private static Boolean anyKeyFieldsMissing(String nType, Vertex v, DbMaps dbMaps) {
1321 if( nType != null && !nType.trim().equals("")
1322 && !dbMaps.NodeKeyProps.containsKey(nType) ){
1323 // They gave us a non-empty nodeType but our NodeKeyProps does
1324 // not have data for it. Since we do not know what the
1325 // key params are for this type of node, we will just
1327 String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
1328 + "]. We cannot determine required keys for this nType. ";
1329 // NOTE - this will be caught below and a "false" returned
1330 throw new AAIException("AAI_6121", emsg);
1333 // Determine what the key fields are for this nodeType
1334 Collection <String> keyPropNamesColl = new ArrayList <>();
1335 if( dbMaps.NodeKeyProps.containsKey(nType) ){
1336 keyPropNamesColl = dbMaps.NodeKeyProps.get(nType);
1339 // NOTE - this will be caught below and a "false" returned
1340 throw new AAIException("AAI_6121", "Definition of key props not found for nodeType = " + nType + ")");
1343 Iterator<String> keyPropI = keyPropNamesColl.iterator();
1344 while (keyPropI.hasNext()) {
1345 String propName = keyPropI.next();
1346 Object ob = v.<Object>property(propName).orElse(null);
1347 if (ob == null || ob.toString().equals("")) {
1348 // It is missing a key property
1352 } catch (AAIException e) {
1353 // Something was wrong -- but since we weren't able to check
1354 // the keys, we will not declare that it is missing keys.
1362 * Gets the delete list.
1364 * @param targetDir the target dir
1365 * @param fileName the file name
1366 * @param edgesOnlyFlag the edges only flag
1367 * @param dontFixOrphans the dont fix orphans
1368 * @param dupeFixOn the dupe fix on
1369 * @return the delete list
1370 * @throws AAIException the AAI exception
1372 private static Set<String> getDeleteList(String targetDir,
1373 String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
1374 Boolean dupeFixOn) throws AAIException {
1376 // Look in the file for lines formated like we expect - pull out any
1377 // Vertex Id's to delete on this run
1378 Set<String> delList = new LinkedHashSet<>();
1379 String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
1380 BufferedReader br = null;
1383 br = new BufferedReader(new FileReader(fullFileName));
1384 String line = br.readLine();
1385 while (line != null) {
1386 if (!line.equals("") && line.startsWith("DeleteCandidate")) {
1387 if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
1388 // We're not going to process edge guys
1389 } else if (dontFixOrphans && line.contains("Orphan")) {
1390 // We're not going to process orphans
1391 } else if (!dupeFixOn && line.contains("Duplicate")) {
1392 // We're not going to process Duplicates
1394 int begIndex = line.indexOf("id = ");
1395 int endIndex = line.indexOf("]");
1396 String vidVal = line.substring(begIndex + 6, endIndex);
1397 delList.add(vidVal);
1400 line = br.readLine();
1403 } catch (IOException e) {
1404 throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
1405 + "], exception= " + e.getMessage());
1410 }// end of getDeleteList
1413 * Gets the preferred dupe.
1415 * @param transId the trans id
1416 * @param fromAppId the from app id
1418 * @param dupeVertexList the dupe vertex list
1419 * @param ver the ver
1420 * @return TitanVertex
1421 * @throws AAIException the AAI exception
1423 public static TitanVertex getPreferredDupe(String transId,
1424 String fromAppId, TitanTransaction g,
1425 ArrayList<TitanVertex> dupeVertexList, String ver, DbMaps dbMaps)
1426 throws AAIException {
1428 // This method assumes that it is being passed a List of vertex objects
1430 // violate our uniqueness constraints.
1432 TitanVertex nullVtx = null;
1434 if (dupeVertexList == null) {
1437 int listSize = dupeVertexList.size();
1438 if (listSize == 0) {
1441 if (listSize == 1) {
1442 return ((TitanVertex) dupeVertexList.get(0));
1445 TitanVertex vtxPreferred = null;
1446 TitanVertex currentFaveVtx = (TitanVertex) dupeVertexList.get(0);
1447 for (int i = 1; i < listSize; i++) {
1448 TitanVertex vtxB = (TitanVertex) dupeVertexList.get(i);
1449 vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
1450 currentFaveVtx, vtxB, ver, dbMaps);
1451 if (vtxPreferred == null) {
1452 // We couldn't choose one
1455 currentFaveVtx = vtxPreferred;
1459 return (currentFaveVtx);
1461 } // end of getPreferredDupe()
1464 * Pick one of two dupes.
1466 * @param transId the trans id
1467 * @param fromAppId the from app id
1469 * @param vtxA the vtx A
1470 * @param vtxB the vtx B
1471 * @param ver the ver
1472 * @return TitanVertex
1473 * @throws AAIException the AAI exception
1475 public static TitanVertex pickOneOfTwoDupes(String transId,
1476 String fromAppId, TitanTransaction g, TitanVertex vtxA,
1477 TitanVertex vtxB, String ver, DbMaps dbMaps) throws AAIException {
1479 TitanVertex nullVtx = null;
1480 TitanVertex preferredVtx = null;
1482 Long vidA = new Long(vtxA.id().toString());
1483 Long vidB = new Long(vtxB.id().toString());
1485 String vtxANodeType = "";
1486 String vtxBNodeType = "";
1487 Object obj = vtxA.<Object>property("aai-node-type").orElse(null);
1489 vtxANodeType = obj.toString();
1491 obj = vtxB.<Object>property("aai-node-type").orElse(null);
1493 vtxBNodeType = obj.toString();
1496 if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
1497 // Either they're not really dupes or there's some bad data - so
1502 // Check that node A and B both have the same key values (or else they
1504 // (We'll check dep-node later)
1505 // Determine what the key fields are for this nodeType
1506 Collection <String> keyProps = new ArrayList <>();
1507 if( dbMaps.NodeKeyProps.containsKey(vtxANodeType) ){
1508 keyProps = dbMaps.NodeKeyProps.get(vtxANodeType);
1511 throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
1514 Iterator<String> keyPropI = keyProps.iterator();
1515 while (keyPropI.hasNext()) {
1516 String propName = keyPropI.next();
1517 String vtxAKeyPropVal = "";
1518 obj = vtxA.<Object>property(propName).orElse(null);
1520 vtxAKeyPropVal = obj.toString();
1522 String vtxBKeyPropVal = "";
1523 obj = vtxB.<Object>property(propName).orElse(null);
1525 vtxBKeyPropVal = obj.toString();
1528 if (vtxAKeyPropVal.equals("")
1529 || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
1530 // Either they're not really dupes or they are missing some key
1531 // data - so don't pick one
1536 // Collect the vid's and aai-node-types of the vertices that each vertex
1537 // (A and B) is connected to.
1538 ArrayList<String> vtxIdsConn2A = new ArrayList<>();
1539 ArrayList<String> vtxIdsConn2B = new ArrayList<>();
1540 HashMap<String, String> nodeTypesConn2A = new HashMap<>();
1541 HashMap<String, String> nodeTypesConn2B = new HashMap<>();
1543 ArrayList<TitanVertex> vertListA = getConnectedNodes( g, vtxA );
1544 if (vertListA != null) {
1545 Iterator<TitanVertex> iter = vertListA.iterator();
1546 while (iter.hasNext()) {
1547 TitanVertex tvCon = iter.next();
1548 String conVid = tvCon.id().toString();
1550 obj = tvCon.<Object>property("aai-node-type").orElse(null);
1552 nt = obj.toString();
1554 nodeTypesConn2A.put(nt, conVid);
1555 vtxIdsConn2A.add(conVid);
1559 ArrayList<TitanVertex> vertListB = getConnectedNodes( g, vtxB );
1560 if (vertListB != null) {
1561 Iterator<TitanVertex> iter = vertListB.iterator();
1562 while (iter.hasNext()) {
1563 TitanVertex tvCon = iter.next();
1564 String conVid = tvCon.id().toString();
1566 obj = tvCon.<Object>property("aai-node-type").orElse(null);
1568 nt = obj.toString();
1570 nodeTypesConn2B.put(nt, conVid);
1571 vtxIdsConn2B.add(conVid);
1575 // 1 - If this kind of node needs a dependent node for uniqueness, then
1576 // verify that they both nodes
1577 // point to the same dependent node (otherwise they're not really
1579 // Note - there are sometimes more than one dependent node type since
1580 // one nodeType can be used in
1581 // different ways. But for a particular node, it will only have one
1582 // dependent node that it's
1584 Collection <String> depNodeTypes = new ArrayList <>();
1585 if( dbMaps.NodeDependencies.containsKey(vtxANodeType) ){
1586 depNodeTypes = dbMaps.NodeDependencies.get(vtxANodeType);
1589 if (depNodeTypes.isEmpty()) {
1590 // This kind of node is not dependent on any other. That is ok.
1592 String depNodeVtxId4A = "";
1593 String depNodeVtxId4B = "";
1594 Iterator<String> iter = depNodeTypes.iterator();
1595 while (iter.hasNext()) {
1596 String depNodeType = iter.next();
1597 if (nodeTypesConn2A.containsKey(depNodeType)) {
1598 // This is the dependent node type that vertex A is using
1599 depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
1601 if (nodeTypesConn2B.containsKey(depNodeType)) {
1602 // This is the dependent node type that vertex B is using
1603 depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
1606 if (depNodeVtxId4A.equals("")
1607 || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
1608 // Either they're not really dupes or there's some bad data - so
1609 // don't pick either one
1614 if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
1615 // 2 - If they both have edges to all the same vertices, then return
1616 // the one with the lower vertexId.
1617 boolean allTheSame = true;
1618 Iterator<String> iter = vtxIdsConn2A.iterator();
1619 while (iter.hasNext()) {
1620 String vtxIdConn2A = iter.next();
1621 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1629 preferredVtx = vtxA;
1631 preferredVtx = vtxB;
1634 } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
1635 // 3 - VertexA is connected to more things than vtxB.
1636 // We'll pick VtxA if its edges are a superset of vtxB's edges.
1637 boolean missingOne = false;
1638 Iterator<String> iter = vtxIdsConn2B.iterator();
1639 while (iter.hasNext()) {
1640 String vtxIdConn2B = iter.next();
1641 if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
1647 preferredVtx = vtxA;
1649 } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
1650 // 4 - VertexB is connected to more things than vtxA.
1651 // We'll pick VtxB if its edges are a superset of vtxA's edges.
1652 boolean missingOne = false;
1653 Iterator<String> iter = vtxIdsConn2A.iterator();
1654 while (iter.hasNext()) {
1655 String vtxIdConn2A = iter.next();
1656 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1662 preferredVtx = vtxB;
1665 preferredVtx = nullVtx;
1668 return (preferredVtx);
1670 } // end of pickOneOfTwoDupes()
1673 * Check and process dupes.
1675 * @param transId the trans id
1676 * @param fromAppId the from app id
1678 * @param version the version
1679 * @param nType the n type
1680 * @param passedVertList the passed vert list
1681 * @param dupeFixOn the dupe fix on
1682 * @param deleteCandidateList the delete candidate list
1683 * @param singleCommits the single commits
1684 * @param alreadyFoundDupeGroups the already found dupe groups
1685 * @param dbMaps the db maps
1686 * @return the array list
1688 private static List<String> checkAndProcessDupes(String transId,
1689 String fromAppId, TitanTransaction g, String version, String nType,
1690 List<TitanVertex> passedVertList, Boolean dupeFixOn,
1691 Set<String> deleteCandidateList, Boolean singleCommits,
1692 ArrayList<String> alreadyFoundDupeGroups, DbMaps dbMaps ) {
1694 ArrayList<String> returnList = new ArrayList<>();
1695 ArrayList<TitanVertex> checkVertList = new ArrayList<>();
1696 ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
1697 Boolean noFilterList = true;
1698 Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
1699 while (afItr.hasNext()) {
1700 String dupeGrpStr = afItr.next();
1701 String[] dupeArr = dupeGrpStr.split("\\|");
1702 int lastIndex = dupeArr.length - 1;
1703 for (int i = 0; i < lastIndex; i++) {
1704 // Note: we don't want the last one...
1705 String vidString = dupeArr[i];
1706 alreadyFoundDupeVidArr.add(vidString);
1707 noFilterList = false;
1711 // For a given set of Nodes that were found with a set of KEY
1712 // Parameters, (nodeType + key data) we will
1713 // see if we find any duplicate nodes that need to be cleaned up. Note -
1714 // it's legit to have more than one
1715 // node with the same key data if the nodes depend on a parent for
1716 // uniqueness -- as long as the two nodes
1717 // don't hang off the same Parent.
1718 // If we find duplicates, and we can figure out which of each set of
1719 // duplicates is the one that we
1720 // think should be preserved, we will record that. Whether we can tell
1721 // which one should be
1722 // preserved or not, we will return info about any sets of duplicates
1725 // Each element in the returned arrayList might look like this:
1726 // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
1727 // couldn't figure out which one to keep)
1728 // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
1729 // thought the third one was the one that should survive)
1731 // Because of the way the calling code loops over stuff, we can get the
1732 // same data multiple times - so we should
1733 // not process any vertices that we've already seen.
1736 Iterator<TitanVertex> pItr = passedVertList.iterator();
1737 while (pItr.hasNext()) {
1738 TitanVertex tvx = (TitanVertex) pItr.next();
1739 String passedId = tvx.id().toString();
1740 if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
1741 // We haven't seen this one before - so we should check it.
1742 checkVertList.add(tvx);
1746 if (checkVertList.size() < 2) {
1747 // Nothing new to check.
1751 if (!dbMaps.NodeDependencies.containsKey(nType)) {
1752 // If this was a node that does NOT depend on other nodes for
1753 // uniqueness, and we
1754 // found more than one node using its key -- record the found
1755 // vertices as duplicates.
1756 String dupesStr = "";
1757 for (int i = 0; i < checkVertList.size(); i++) {
1759 + ((TitanVertex) (checkVertList.get(i))).id()
1762 if (dupesStr != "") {
1763 TitanVertex prefV = getPreferredDupe(transId, fromAppId,
1764 g, checkVertList, version, dbMaps);
1765 if (prefV == null) {
1766 // We could not determine which duplicate to keep
1767 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1768 returnList.add(dupesStr);
1770 dupesStr = dupesStr + "KeepVid=" + prefV.id();
1771 Boolean didRemove = false;
1773 didRemove = deleteNonKeepersIfAppropriate(g,
1774 dupesStr, prefV.id().toString(),
1775 deleteCandidateList, singleCommits);
1780 // keep them on our list
1781 returnList.add(dupesStr);
1786 // More than one node have the same key fields since they may
1787 // depend on a parent node for uniqueness. Since we're finding
1788 // more than one, we want to check to see if any of the
1789 // vertices that have this set of keys (and are the same nodeType)
1790 // are also pointing at the same 'parent' node.
1791 // Note: for a given set of key data, it is possible that there
1792 // could be more than one set of duplicates.
1793 HashMap<String, ArrayList<TitanVertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
1794 transId, fromAppId, g, version, nType,
1795 checkVertList, dbMaps);
1796 for (Map.Entry<String, ArrayList<TitanVertex>> entry : vertsGroupedByParentHash
1798 ArrayList<TitanVertex> thisParentsVertList = entry
1800 if (thisParentsVertList.size() > 1) {
1801 // More than one vertex found with the same key info
1802 // hanging off the same parent/dependent node
1803 String dupesStr = "";
1804 for (int i = 0; i < thisParentsVertList.size(); i++) {
1806 + ((TitanVertex) (thisParentsVertList
1807 .get(i))).id() + "|";
1809 if (dupesStr != "") {
1810 TitanVertex prefV = getPreferredDupe(transId,
1811 fromAppId, g, thisParentsVertList,
1814 if (prefV == null) {
1815 // We could not determine which duplicate to
1817 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1818 returnList.add(dupesStr);
1820 Boolean didRemove = false;
1821 dupesStr = dupesStr + "KeepVid="
1822 + prefV.id().toString();
1824 didRemove = deleteNonKeepersIfAppropriate(
1825 g, dupesStr, prefV.id()
1827 deleteCandidateList, singleCommits);
1832 // keep them on our list
1833 returnList.add(dupesStr);
1840 } catch (Exception e) {
1841 LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
1846 }// End of checkAndProcessDupes()
1849 * Group verts by dep nodes.
1851 * @param transId the trans id
1852 * @param fromAppId the from app id
1854 * @param version the version
1855 * @param nType the n type
1856 * @param passedVertList the passed vert list
1857 * @param dbMaps the db maps
1858 * @return the hash map
1859 * @throws AAIException the AAI exception
1861 private static HashMap<String, ArrayList<TitanVertex>> groupVertsByDepNodes(
1862 String transId, String fromAppId, TitanTransaction g, String version,
1863 String nType, ArrayList<TitanVertex> passedVertList, DbMaps dbMaps)
1864 throws AAIException {
1865 // Given a list of Titan Vertices of one nodeType (see AAI-8956), group
1866 // them together by the parent node they depend on.
1867 // Ie. if given a list of ip address nodes (assumed to all have the
1868 // same key info) they might sit under several different parent vertices.
1869 // Under Normal conditions, there would only be one per parent -- but
1870 // we're trying to find duplicates - so we
1871 // allow for the case where more than one is under the same parent node.
1873 HashMap<String, ArrayList<TitanVertex>> retHash = new HashMap<String, ArrayList<TitanVertex>>();
1874 if (!dbMaps.NodeDependencies.containsKey(nType)) {
1875 // This method really should not have been called if this is not the
1877 // that depends on a parent for uniqueness, so just return the empty
1882 // Find out what types of nodes the passed in nodes can depend on
1883 ArrayList<String> depNodeTypeL = new ArrayList<>();
1884 Collection<String> depNTColl = dbMaps.NodeDependencies.get(nType);
1885 Iterator<String> ntItr = depNTColl.iterator();
1886 while (ntItr.hasNext()) {
1887 depNodeTypeL.add(ntItr.next());
1889 // For each vertex, we want find its depended-on/parent vertex so we
1890 // can track what other vertexes that are dependent on that same guy.
1891 if (passedVertList != null) {
1892 Iterator<TitanVertex> iter = passedVertList.iterator();
1893 while (iter.hasNext()) {
1894 TitanVertex thisVert = iter.next();
1895 TitanVertex tmpParentVtx = getConnectedParent( g, thisVert );
1896 if( tmpParentVtx != null ) {
1897 String parentNt = null;
1898 Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
1900 parentNt = obj.toString();
1902 if (depNTColl.contains(parentNt)) {
1903 // This must be the parent/dependent node
1904 String parentVid = tmpParentVtx.id().toString();
1905 if (retHash.containsKey(parentVid)) {
1906 // add this vert to the list for this parent key
1907 retHash.get(parentVid).add(thisVert);
1909 // This is the first one we found on this parent
1910 ArrayList<TitanVertex> vList = new ArrayList<>();
1911 vList.add(thisVert);
1912 retHash.put(parentVid, vList);
1921 }// end of groupVertsByDepNodes()
1924 * Delete non keepers if appropriate.
1927 * @param dupeInfoString the dupe info string
1928 * @param vidToKeep the vid to keep
1929 * @param deleteCandidateList the delete candidate list
1930 * @param singleCommits the single commits
1931 * @return the boolean
1933 private static Boolean deleteNonKeepersIfAppropriate(TitanTransaction g,
1934 String dupeInfoString, String vidToKeep,
1935 Set<String> deleteCandidateList, Boolean singleCommits) {
1937 Boolean deletedSomething = false;
1938 // This assumes that the dupeInfoString is in the format of
1939 // pipe-delimited vid's followed by
1940 // ie. "3456|9880|keepVid=3456"
1941 if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
1942 // No vid's on the candidate list -- so no deleting will happen on
1947 String[] dupeArr = dupeInfoString.split("\\|");
1948 ArrayList<String> idArr = new ArrayList<>();
1949 int lastIndex = dupeArr.length - 1;
1950 for (int i = 0; i <= lastIndex; i++) {
1951 if (i < lastIndex) {
1952 // This is not the last entry, it is one of the dupes,
1953 String vidString = dupeArr[i];
1954 idArr.add(vidString);
1956 // This is the last entry which should tell us if we have a
1958 String prefString = dupeArr[i];
1959 if (prefString.equals("KeepVid=UNDETERMINED")) {
1960 // They sent us a bad string -- nothing should be deleted if
1961 // no dupe could be tagged as preferred
1964 // If we know which to keep, then the prefString should look
1965 // like, "KeepVid=12345"
1966 String[] prefArr = prefString.split("=");
1967 if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
1968 LOGGER.error("Bad format. Expecting KeepVid=999999");
1971 String keepVidStr = prefArr[1];
1972 if (idArr.contains(keepVidStr)) {
1973 idArr.remove(keepVidStr);
1975 // So now, the idArr should just contain the vid's
1976 // that we want to remove.
1977 for (int x = 0; x < idArr.size(); x++) {
1978 boolean okFlag = true;
1979 String thisVid = idArr.get(x);
1980 if (deleteCandidateList.contains(thisVid)) {
1981 // This vid is a valid delete candidate from
1982 // a prev. run, so we can remove it.
1984 long longVertId = Long
1985 .parseLong(thisVid);
1987 .getVertex(longVertId);
1989 if (singleCommits) {
1990 // NOTE - the singleCommits option is not used in normal processing
1992 g = AAIGraph.getInstance().getGraph().newTransaction();
1994 } catch (Exception e) {
1996 LOGGER.error("ERROR trying to delete VID = " + thisVid, e);
1999 LOGGER.info(" DELETED VID = " + thisVid);
2000 deletedSomething = true;
2005 LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
2006 + dupeInfoString + "]");
2010 }// else we know which one to keep
2012 }// for each vertex in a group
2014 return deletedSomething;
2016 }// end of deleteNonKeepersIfAppropriate()
2020 * Gets the node just using key params.
2022 * @param transId the trans id
2023 * @param fromAppId the from app id
2024 * @param graph the graph
2025 * @param nodeType the node type
2026 * @param keyPropsHash the key props hash
2027 * @param apiVersion the api version
2028 * @return the node just using key params
2029 * @throws AAIException the AAI exception
2031 public static List <TitanVertex> getNodeJustUsingKeyParams( String transId, String fromAppId, TitanTransaction graph, String nodeType,
2032 HashMap<String,Object> keyPropsHash, String apiVersion ) throws AAIException{
2034 List <TitanVertex> retVertList = new ArrayList <> ();
2036 // We assume that all NodeTypes have at least one key-property defined.
2037 // Note - instead of key-properties (the primary key properties), a user could pass
2038 // alternate-key values if they are defined for the nodeType.
2039 List<String> kName = new ArrayList<>();
2040 List<Object> kVal = new ArrayList<>();
2041 if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
2042 throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. ");
2046 for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){
2048 kName.add(i, entry.getKey());
2049 kVal.add(i, entry.getValue());
2051 int topPropIndex = i;
2052 TitanVertex tiV = null;
2053 String propsAndValuesForMsg = "";
2054 Iterable <?> verts = null;
2057 if( topPropIndex == 0 ){
2058 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
2059 verts= graph.query().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType).vertices();
2061 else if( topPropIndex == 1 ){
2062 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2063 + kName.get(1) + " = " + kVal.get(1) + ") ";
2064 verts = graph.query().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType).vertices();
2066 else if( topPropIndex == 2 ){
2067 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2068 + kName.get(1) + " = " + kVal.get(1) + ", "
2069 + kName.get(2) + " = " + kVal.get(2) + ") ";
2070 verts= graph.query().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType).vertices();
2072 else if( topPropIndex == 3 ){
2073 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2074 + kName.get(1) + " = " + kVal.get(1) + ", "
2075 + kName.get(2) + " = " + kVal.get(2) + ", "
2076 + kName.get(3) + " = " + kVal.get(3) + ") ";
2077 verts= graph.query().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType).vertices();
2080 throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
2083 catch( Exception ex ){
2084 LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex);
2087 if( verts != null ){
2088 Iterator <?> vertI = verts.iterator();
2089 while( vertI.hasNext() ){
2090 tiV = (TitanVertex) vertI.next();
2091 retVertList.add(tiV);
2095 if( retVertList.size() == 0 ){
2096 LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
2097 "], propsAndVal = " + propsAndValuesForMsg );
2102 }// End of getNodeJustUsingKeyParams()
2105 * Show all edges for node.
2107 * @param transId the trans id
2108 * @param fromAppId the from app id
2109 * @param tVert the t vert
2110 * @return the array list
2112 private static ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, TitanVertex tVert ){
2114 ArrayList <String> retArr = new ArrayList <> ();
2115 Iterator <Edge> eI = tVert.edges(Direction.IN);
2116 if( ! eI.hasNext() ){
2117 retArr.add("No IN edges were found for this vertex. ");
2119 while( eI.hasNext() ){
2120 TitanEdge ed = (TitanEdge) eI.next();
2121 String lab = ed.label();
2122 TitanVertex vtx = (TitanVertex) ed.otherVertex(tVert);
2124 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2127 String nType = vtx.<String>property("aai-node-type").orElse(null);
2128 String vid = vtx.id().toString();
2129 retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
2134 eI = tVert.edges(Direction.OUT);
2135 if( ! eI.hasNext() ){
2136 retArr.add("No OUT edges were found for this vertex. ");
2138 while( eI.hasNext() ){
2139 TitanEdge ed = (TitanEdge) eI.next();
2140 String lab = ed.label();
2141 TitanVertex vtx = (TitanVertex) ed.otherVertex(tVert);
2143 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2146 String nType = vtx.<String>property("aai-node-type").orElse(null);
2147 String vid = vtx.id().toString();
2148 retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
2156 * Show properties for node.
2158 * @param transId the trans id
2159 * @param fromAppId the from app id
2160 * @param tVert the t vert
2161 * @return the array list
2163 private static ArrayList <String> showPropertiesForNode( String transId, String fromAppId, TitanVertex tVert ){
2165 ArrayList <String> retArr = new ArrayList <> ();
2166 if( tVert == null ){
2167 retArr.add("null Node object passed to showPropertiesForNode()\n");
2170 String nodeType = "";
2171 Object ob = tVert.<Object>property("aai-node-type").orElse(null);
2176 nodeType = ob.toString();
2179 retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
2180 retArr.add(" Property Detail: ");
2181 Iterator<VertexProperty<Object>> pI = tVert.properties();
2182 while( pI.hasNext() ){
2183 VertexProperty<Object> tp = pI.next();
2184 Object val = tp.value();
2185 retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
2192 private static ArrayList <TitanVertex> getConnectedNodes(TitanTransaction g, TitanVertex startVtx )
2193 throws AAIException {
2195 ArrayList <TitanVertex> retArr = new ArrayList <> ();
2196 if( startVtx == null ){
2200 GraphTraversal<Vertex, Vertex> modPipe = null;
2201 modPipe = g.traversal().V(startVtx).both();
2202 if( modPipe != null && modPipe.hasNext() ){
2203 while( modPipe.hasNext() ){
2204 TitanVertex conVert = (TitanVertex) modPipe.next();
2205 retArr.add(conVert);
2211 }// End of getConnectedNodes()
2214 private static ArrayList <TitanVertex> getConnectedChildrenOfOneType( TitanTransaction graph,
2215 TitanVertex startVtx, String childNType ) throws AAIException{
2217 ArrayList <TitanVertex> childList = new ArrayList <> ();
2218 Iterable <?> verts = startVtx.query().direction(Direction.OUT).has("isParent",true).vertices();
2219 Iterator <?> vertI = verts.iterator();
2220 TitanVertex tmpVtx = null;
2221 while( vertI != null && vertI.hasNext() ){
2222 tmpVtx = (TitanVertex) vertI.next();
2223 Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
2225 String tmpNt = ob.toString();
2226 if( tmpNt.equals(childNType)){
2227 childList.add(tmpVtx);
2234 }// End of getConnectedChildrenOfOneType()
2237 private static TitanVertex getConnectedParent( TitanTransaction graph,
2238 TitanVertex startVtx ) throws AAIException{
2240 TitanVertex parentVtx = null;
2241 Iterable <?> verts = startVtx.query().direction(Direction.IN).has("isParent",true).vertices();
2242 Iterator <?> vertI = verts.iterator();
2243 while( vertI != null && vertI.hasNext() ){
2244 // Note - there better only be one!
2245 parentVtx = (TitanVertex) vertI.next();
2250 }// End of getConnectedParent()