aai-core/src/main/java/org/openecomp/aai/dbgen/DataGrooming.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * org.openecomp.aai
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.aai.dbgen;
  22
  23 import java.io.BufferedReader;
  24 import java.io.BufferedWriter;
  25 import java.io.File;
  26 import java.io.FileReader;
  27 import java.io.FileWriter;
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.Collection;
  31 import java.util.HashMap;
  32 import java.util.Iterator;
  33 import java.util.LinkedHashSet;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.Map.Entry;
  37 import java.util.Properties;
  38 import java.util.Set;
  39 import java.util.UUID;
  40
  41 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
  42 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
  43 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
  44 import org.apache.tinkerpop.gremlin.structure.Direction;
  45 import org.apache.tinkerpop.gremlin.structure.Edge;
  46 import org.apache.tinkerpop.gremlin.structure.Property;
  47 import org.apache.tinkerpop.gremlin.structure.Vertex;
  48 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
  49 import org.openecomp.aai.db.props.AAIProperties;
  50 import org.openecomp.aai.dbmap.AAIGraph;
  51 import org.openecomp.aai.exceptions.AAIException;
  52 import org.openecomp.aai.introspection.Introspector;
  53 import org.openecomp.aai.introspection.Loader;
  54 import org.openecomp.aai.introspection.LoaderFactory;
  55 import org.openecomp.aai.introspection.ModelType;
  56 import org.openecomp.aai.introspection.exceptions.AAIUnknownObjectException;
  57 import org.openecomp.aai.logging.ErrorLogHelper;
  58 import org.openecomp.aai.serialization.db.EdgeProperties;
  59 import org.openecomp.aai.serialization.db.EdgeProperty;
  60 import org.openecomp.aai.util.AAIConfig;
  61 import org.openecomp.aai.util.AAIConstants;
  62 import org.openecomp.aai.util.FormatDate;
  63
  64 import com.att.eelf.configuration.Configuration;
  65 import com.att.eelf.configuration.EELFLogger;
  66 import com.att.eelf.configuration.EELFManager;
  67 import com.thinkaurelius.titan.core.TitanEdge;
  68 import com.thinkaurelius.titan.core.TitanFactory;
  69 import com.thinkaurelius.titan.core.TitanGraph;
  70 import com.thinkaurelius.titan.core.TitanTransaction;
  71
  72
  73 public class DataGrooming {
  74
  75         private static EELFLogger LOGGER;
  76         private static final String FROMAPPID = "AAI-DB";
  77         private static final String TRANSID = UUID.randomUUID().toString();
  78         private static int dupeGrpsDeleted = 0;
  79
  80         /**
  81          * The main method.
  82          *
  83          * @param args the arguments
  84          */
  85         public static void main(String[] args) {
  86
  87                 // Set the logging file properties to be used by EELFManager
  88                 Properties props = System.getProperties();
  89                 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS);
  90                 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
  91                 LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
  92                 String ver = "version"; // Placeholder
  93                 Boolean doAutoFix = false;
  94                 Boolean edgesOnlyFlag = false;
  95                 Boolean dontFixOrphansFlag = false;
  96                 Boolean skipHostCheck = false;
  97                 Boolean singleCommits = false;
  98                 Boolean dupeCheckOff = false;
  99                 Boolean dupeFixOn = false;
 100                 Boolean ghost2CheckOff = false;
 101                 Boolean ghost2FixOn = false;
 102                 Boolean neverUseCache = false;
 103
 104                 int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
 105                 int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
 106                 try {
 107                         String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
 108                         if( maxFixStr != null &&  !maxFixStr.equals("") ){
 109                                 maxRecordsToFix = Integer.parseInt(maxFixStr);
 110                         }
 111                         String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
 112                         if( sleepStr != null &&  !sleepStr.equals("") ){
 113                                 sleepMinutes = Integer.parseInt(sleepStr);
 114                         }
 115                 }
 116                 catch ( Exception e ){
 117                         // Don't worry, we'll just use the defaults that we got from AAIConstants
 118                         LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
 119                 }
 120
 121                 String prevFileName = "";
 122                 dupeGrpsDeleted = 0;
 123                 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
 124                 String dteStr = fd.getDateTime();
 125                 String groomOutFileName = "dataGrooming." + dteStr + ".out";
 126
 127                 if (args.length > 0) {
 128                         // They passed some arguments in that will affect processing
 129                         for (int i = 0; i < args.length; i++) {
 130                                 String thisArg = args[i];
 131                                 if (thisArg.equals("-edgesOnly")) {
 132                                         edgesOnlyFlag = true;
 133                                 } else if (thisArg.equals("-autoFix")) {
 134                                         doAutoFix = true;
 135                                 } else if (thisArg.equals("-skipHostCheck")) {
 136                                         skipHostCheck = true;
 137                                 } else if (thisArg.equals("-dontFixOrphans")) {
 138                                         dontFixOrphansFlag = true;
 139                                 } else if (thisArg.equals("-singleCommits")) {
 140                                         singleCommits = true;
 141                                 } else if (thisArg.equals("-dupeCheckOff")) {
 142                                         dupeCheckOff = true;
 143                                 } else if (thisArg.equals("-dupeFixOn")) {
 144                                         dupeFixOn = true;
 145                                 } else if (thisArg.equals("-ghost2CheckOff")) {
 146                                         ghost2CheckOff = true;
 147                                 } else if (thisArg.equals("-neverUseCache")) {
 148                                         neverUseCache = true;
 149                                 } else if (thisArg.equals("-ghost2FixOn")) {
 150                                         ghost2FixOn = true;
 151                                 } else if (thisArg.equals("-maxFix")) {
 152                                         i++;
 153                                         if (i >= args.length) {
 154                                                 LOGGER.error(" No value passed with -maxFix option.  ");
 155                                                 System.exit(0);
 156                                         }
 157                                         String nextArg = args[i];
 158                                         try {
 159                                                 maxRecordsToFix = Integer.parseInt(nextArg);
 160                                         } catch (Exception e) {
 161                                                 LOGGER.error("Bad value passed with -maxFix option: ["
 162                                                                                 + nextArg + "]");
 163                                                 System.exit(0);
 164                                         }
 165                                 } else if (thisArg.equals("-sleepMinutes")) {
 166                                         i++;
 167                                         if (i >= args.length) {
 168                                                 LOGGER.error("No value passed with -sleepMinutes option.");
 169                                                 System.exit(0);
 170                                         }
 171                                         String nextArg = args[i];
 172                                         try {
 173                                                 sleepMinutes = Integer.parseInt(nextArg);
 174                                         } catch (Exception e) {
 175                                                 LOGGER.error("Bad value passed with -sleepMinutes option: ["
 176                                                                                 + nextArg + "]");
 177                                                 System.exit(0);
 178                                         }
 179                                 } else if (thisArg.equals("-f")) {
 180                                         i++;
 181                                         if (i >= args.length) {
 182                                                 LOGGER.error(" No value passed with -f option. ");
 183                                                 System.exit(0);
 184                                         }
 185                                         prevFileName = args[i];
 186                                 } else {
 187                                         LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
 188                                                                         + thisArg + "]. ");
 189                                         LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache");
 190                                         System.exit(0);
 191                                 }
 192                         }
 193                 }
 194
 195
 196                 try {
 197                         LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
 198
 199                 }
 200                 catch (Exception ex){
 201                         LOGGER.error("ERROR - Could not create loader", ex);
 202                         System.exit(1);
 203                 }
 204
 205
 206                 try {
 207                         if (!prevFileName.equals("")) {
 208                                 // They are trying to fix some data based on a data in a
 209                                 // previous file.
 210                                 LOGGER.info(" Call doTheGrooming() with a previous fileName ["
 211                                                                 + prevFileName + "] for cleanup. ");
 212                                 Boolean finalShutdownFlag = true;
 213                                 Boolean cacheDbOkFlag = false;
 214                                 doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
 215                                                 maxRecordsToFix, groomOutFileName, ver, singleCommits,
 216                                                 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 217                                                 finalShutdownFlag, cacheDbOkFlag);
 218                         } else if (doAutoFix) {
 219                                 // They want us to run the processing twice -- first to look for
 220                                 // delete candidates, then after
 221                                 // napping for a while, run it again and delete any candidates
 222                                 // that were found by the first run.
 223                                 // Note: we will produce a separate output file for each of the
 224                                 // two runs.
 225                                 LOGGER.info(" Doing an auto-fix call to Grooming. ");
 226                                 LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
 227                                 Boolean finalShutdownFlag = false;
 228                                 Boolean cacheDbOkFlag = true;
 229                                 int fixCandCount = doTheGrooming("", edgesOnlyFlag,
 230                                                 dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
 231                                                 ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 232                                                 finalShutdownFlag, cacheDbOkFlag);
 233                                 if (fixCandCount == 0) {
 234                                         LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
 235                                 } else {
 236                                         // We'll sleep a little and then run a fix-pass based on the
 237                                         // first-run's output file.
 238                                         try {
 239                                                 LOGGER.info("About to sleep for " + sleepMinutes
 240                                                                 + " minutes.");
 241                                                 int sleepMsec = sleepMinutes * 60 * 1000;
 242                                                 Thread.sleep(sleepMsec);
 243                                         } catch (InterruptedException ie) {
 244                                                 LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
 245                                                 System.exit(0);
 246                                         }
 247
 248                                         dteStr = fd.getDateTime();
 249                                         String secondGroomOutFileName = "dataGrooming." + dteStr
 250                                                         + ".out";
 251                                         LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
 252                                                                         + "generated by the first pass for fixing: ["
 253                                                                         + groomOutFileName + "]");
 254                                         finalShutdownFlag = true;
 255                                         cacheDbOkFlag = false;
 256                                         doTheGrooming(groomOutFileName, edgesOnlyFlag,
 257                                                         dontFixOrphansFlag, maxRecordsToFix,
 258                                                         secondGroomOutFileName, ver, singleCommits,
 259                                                         dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 260                                                         finalShutdownFlag, cacheDbOkFlag);
 261                                 }
 262                         } else {
 263                                 // Do the grooming - plain vanilla (no fix-it-file, no
 264                                 // auto-fixing)
 265                                 Boolean finalShutdownFlag = true;
 266                                 LOGGER.info(" Call doTheGrooming() ");
 267                                 Boolean cacheDbOkFlag = true;
 268                                 if( neverUseCache ){
 269                                         // They have forbidden us from using a cached db connection.
 270                                         cacheDbOkFlag = false;
 271                                 }
 272                                 doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
 273                                                 maxRecordsToFix, groomOutFileName, ver, singleCommits,
 274                                                 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 275                                                 finalShutdownFlag, cacheDbOkFlag);
 276                         }
 277                 } catch (Exception ex) {
 278                         LOGGER.error("Exception while grooming data", ex);
 279                 }
 280
 281                 LOGGER.info(" Done! ");
 282                 System.exit(0);
 283
 284         }// End of main()
 285
 286         /**
 287          * Do the grooming.
 288          *
 289          * @param fileNameForFixing the file name for fixing
 290          * @param edgesOnlyFlag the edges only flag
 291          * @param dontFixOrphansFlag the dont fix orphans flag
 292          * @param maxRecordsToFix the max records to fix
 293          * @param groomOutFileName the groom out file name
 294          * @param version the version
 295          * @param singleCommits the single commits
 296          * @param dupeCheckOff the dupe check off
 297          * @param dupeFixOn the dupe fix on
 298          * @param ghost2CheckOff the ghost 2 check off
 299          * @param ghost2FixOn the ghost 2 fix on
 300          * @param finalShutdownFlag the final shutdown flag
 301          * @param cacheDbOkFlag the cacheDbOk flag
 302          * @return the int
 303          */
 304         private static int doTheGrooming(String fileNameForFixing,
 305                         Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
 306                         int maxRecordsToFix, String groomOutFileName, String version,
 307                         Boolean singleCommits,
 308                         Boolean dupeCheckOff, Boolean dupeFixOn,
 309                         Boolean ghost2CheckOff, Boolean ghost2FixOn,
 310                         Boolean finalShutdownFlag, Boolean cacheDbOkFlag) {
 311
 312                 LOGGER.debug(" Entering doTheGrooming \n");
 313
 314                 int cleanupCandidateCount = 0;
 315                 BufferedWriter bw = null;
 316                 TitanGraph graph = null;
 317                 TitanGraph graph2 = null;
 318                 int deleteCount = 0;
 319                 boolean executeFinalCommit = false;
 320                 Set<String> deleteCandidateList = new LinkedHashSet<>();
 321                 Set<String> processedVertices = new LinkedHashSet<>();
 322                 TitanTransaction g = null;
 323                 TitanTransaction g2 = null;
 324                 try {
 325                         AAIConfig.init();
 326                         String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
 327                                         + "logs" + AAIConstants.AAI_FILESEP + "data"
 328                                         + AAIConstants.AAI_FILESEP + "dataGrooming";
 329
 330                         // Make sure the target directory exists
 331                         new File(targetDir).mkdirs();
 332
 333                         if (!fileNameForFixing.equals("")) {
 334                                 deleteCandidateList = getDeleteList(targetDir,
 335                                                 fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
 336                                                 dupeFixOn);
 337                         }
 338
 339                         if (deleteCandidateList.size() > maxRecordsToFix) {
 340                                 LOGGER.warn(" >> WARNING >>  Delete candidate list size ("
 341                                                 + deleteCandidateList.size()
 342                                                 + ") is too big.  The maxFix we are using is: "
 343                                                 + maxRecordsToFix
 344                                                 + ".  No candidates will be deleted. ");
 345                                 // Clear out the list so it won't be processed below.
 346                                 deleteCandidateList = new LinkedHashSet<>();
 347                         }
 348
 349                         String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
 350                                         + groomOutFileName;
 351                         File groomOutFile = new File(fullOutputFileName);
 352                         try {
 353                                 groomOutFile.createNewFile();
 354                         } catch (IOException e) {
 355                                 String emsg = " Problem creating output file ["
 356                                                 + fullOutputFileName + "], exception=" + e.getMessage();
 357                                 throw new AAIException("AAI_6124", emsg);
 358                         }
 359
 360                         LOGGER.info(" Will write to " + fullOutputFileName );
 361                         FileWriter fw = new FileWriter(groomOutFile.getAbsoluteFile());
 362                         bw = new BufferedWriter(fw);
 363                         ErrorLogHelper.loadProperties();
 364
 365                         LOGGER.info("    ---- NOTE --- about to open graph (takes a little while)--------\n");
 366
 367                         if( cacheDbOkFlag ){
 368                                 // Since we're just reading (not deleting/fixing anything), we can use
 369                                 // a cached connection to the DB
 370                                 graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG);
 371                         }
 372                         else {
 373                                 graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
 374                         }
 375                         if (graph == null) {
 376                                 String emsg = "null graph object in DataGrooming\n";
 377                                 throw new AAIException("AAI_6101", emsg);
 378                         }
 379
 380                         LOGGER.debug(" Got the graph object. ");
 381
 382                         g = graph.newTransaction();
 383                         if (g == null) {
 384                                 String emsg = "null graphTransaction object in DataGrooming\n";
 385                                 throw new AAIException("AAI_6101", emsg);
 386                         }
 387                         GraphTraversalSource source1 = g.traversal();
 388
 389                         ArrayList<String> errArr = new ArrayList<>();
 390                         int totalNodeCount = 0;
 391                         HashMap<String, String> misMatchedHash = new HashMap<String, String>();
 392                         HashMap<String, Vertex> orphanNodeHash = new HashMap<String, Vertex>();
 393                         HashMap<String, Vertex> missingDepNodeHash = new HashMap<String, Vertex>();
 394                         HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
 395                         HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
 396                         HashMap<String, Vertex> ghostNodeHash = new HashMap<String, Vertex>();
 397                         ArrayList<String> dupeGroups = new ArrayList<>();
 398
 399                         Loader loader = LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
 400
 401                         Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
 402                         String ntList = "";
 403
 404                         LOGGER.info("  Starting DataGrooming Processing ");
 405
 406                         if (edgesOnlyFlag) {
 407                                 LOGGER.info(" NOTE >> Skipping Node processing as requested.  Will only process Edges. << ");
 408                         }
 409                         else {
 410                                 for (Entry<String, Introspector> entry : entrySet) {
 411                                         String nType = entry.getKey();
 412                                         int thisNtCount = 0;
 413                                         int thisNtDeleteCount = 0;
 414                                         LOGGER.debug(" >  Look at : [" + nType + "] ...");
 415                                         ntList = ntList + "," + nType;
 416
 417                                         // Get a collection of the names of the key properties for this nodeType to use later
 418                                         // Determine what the key fields are for this nodeType
 419                                         Collection <String> keyProps = entry.getValue().getKeys();
 420
 421                                         // Get the types of nodes that this nodetype depends on for uniqueness (if any)
 422                                         Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn();
 423
 424                                         // Loop through all the nodes of this Node type
 425                                         int lastShownForNt = 0;
 426                                         ArrayList <Vertex> tmpList = new ArrayList <> ();
 427                                         Iterator <Vertex> iterv =  source1.V().has("aai-node-type",nType);
 428                                         while (iterv.hasNext()) {
 429                                                 // We put the nodes into an ArrayList because the graph.query iterator can time out
 430                                                 tmpList.add(iterv.next());
 431                                         }
 432
 433                                         Iterator <Vertex> iter = tmpList.iterator();
 434                                         while (iter.hasNext()) {
 435                                                 try {
 436                                                         thisNtCount++;
 437                                                         if( thisNtCount == lastShownForNt + 250 ){
 438                                                                 lastShownForNt = thisNtCount;
 439                                                                 LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
 440                                                         }
 441                                                         Vertex thisVtx = iter.next();
 442                                                         String thisVid = thisVtx.id().toString();
 443                                                         if (processedVertices.contains(thisVid)) {
 444                                                                 LOGGER.debug("skipping already processed vertex: " + thisVid);
 445                                                                 continue;
 446                                                         }
 447                                                         totalNodeCount++;
 448                                                         List <Vertex> secondGetList = new ArrayList <> ();
 449                                                         // -----------------------------------------------------------------------
 450                                                         // For each vertex of this nodeType, we want to:
 451                                                         //              a) make sure that it can be retrieved using it's AAI defined key
 452                                                         //      b) make sure that it is not a duplicate
 453                                                         // -----------------------------------------------------------------------
 454
 455                                                         // For this instance of this nodeType, get the key properties
 456                                                         HashMap<String, Object> propHashWithKeys = new HashMap<>();
 457                                                         Iterator<String> keyPropI = keyProps.iterator();
 458                                                         while (keyPropI.hasNext()) {
 459                                                                 String propName = keyPropI.next();
 460                                                                 String propVal = "";
 461                                                                 //delete an already deleted vertex
 462                                                                 Object obj = thisVtx.<Object>property(propName).orElse(null);
 463                                                                 if (obj != null) {
 464                                                                         propVal = obj.toString();
 465                                                                 }
 466                                                                 propHashWithKeys.put(propName, propVal);
 467                                                         }
 468                                                         try {
 469                                                                 // If this node is dependent on another for uniqueness, then do the query from that parent node
 470                                                                 // Note - all of our nodes that are dependent on others for uniqueness are
 471                                                                 //              "children" of that node.
 472                                                                 boolean depNodeOk = true;
 473                                                                 if( depNodeTypes.isEmpty() ){
 474                                                                         // This kind of node is not dependent on any other.
 475                                                                         // Make sure we can get it back using it's key properties and that we only get one.
 476                                                                         secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType,
 477                                                                                         propHashWithKeys, version );
 478                                                                 }
 479                                                                 else {
 480                                                                         // This kind of node is dependent on another for uniqueness.
 481                                                                         // Start at it's parent (the dependent vertex) and make sure we can get it
 482                                                                         // back using it's key properties and that we only get one.
 483                                                                         Iterator <Vertex> vertI2 = source1.V(thisVtx).union(__.inE().has(EdgeProperties.out(EdgeProperty.IS_PARENT), true).outV(), __.outE().has(EdgeProperties.in(EdgeProperty.IS_PARENT)).inV());
 484                                                                         Vertex parentVtx = null;
 485                                                                         int pCount = 0;
 486                                                                         while( vertI2 != null && vertI2.hasNext() ){
 487                                                                                 parentVtx = vertI2.next();
 488                                                                                 pCount++;
 489                                                                         }
 490                                                                         if( pCount <= 0 ){
 491
 492
 493                                                                         //List<Vertex> vertI2 = g.traversal().V(thisVtx).union(__.outE().has("isParent-REV",true).outV(),__.inE().has("isParent",true).inV()).toList();
 494                                                                         //if( vertI2.isEmpty()){
 495
 496                                                                                 // It's Missing it's dependent/parent node
 497                                                                                 depNodeOk = false;
 498                                                                                 boolean zeroEdges = false;
 499                                                                                 try {
 500                                                                                         Iterator<Edge> tmpEdgeIter = thisVtx.edges(Direction.BOTH);
 501                                                                                         int edgeCount = 0;
 502                                                                                         while( tmpEdgeIter.hasNext() ){
 503                                                                                                 edgeCount++;
 504                                                                                                 tmpEdgeIter.next();
 505                                                                                         }
 506                                                                                         if( edgeCount == 0 ){
 507                                                                                                 zeroEdges = true;
 508                                                                                         }
 509                                                                                 } catch (Exception ex) {
 510                                                                                         LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex);
 511                                                                                 }
 512
 513                                                                                 if (deleteCandidateList.contains(thisVid)) {
 514                                                                                         boolean okFlag = true;
 515                                                                                         try {
 516                                                                                                 processedVertices.add(thisVtx.id().toString());
 517                                                                                                 thisVtx.remove();
 518                                                                                                 deleteCount++;
 519                                                                                                 thisNtDeleteCount++;
 520                                                                                         } catch (Exception e) {
 521                                                                                                 okFlag = false;
 522                                                                                                 LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e);
 523                                                                                         }
 524                                                                                         if (okFlag) {
 525                                                                                                 LOGGER.info(" DELETED missing-dep-node VID = " + thisVid);
 526                                                                                         }
 527                                                                                 } else {
 528                                                                                         // We count nodes missing their depNodes two ways - the first if it has
 529                                                                                         //    at least some edges, and the second if it has zero edges.  Either
 530                                                                                         //    way, they are effectively orphaned.
 531                                                                                         // NOTE - Only nodes that have dependent nodes are ever considered "orphaned".
 532                                                                                         if( zeroEdges ){
 533                                                                                                 missingDepNodeHash.put(thisVid, thisVtx);
 534                                                                                         }
 535                                                                                         else {
 536                                                                                                 orphanNodeHash.put(thisVid, thisVtx);
 537                                                                                         }
 538                                                                                 }
 539                                                                         }
 540                                                                         else if ( pCount > 1 ){
 541                                                                                 // Not sure how this could happen?  Should we do something here?
 542                                                                                 depNodeOk = false;
 543                                                                         }
 544                                                                         else {
 545                                                                                 // We found the parent - so use it to do the second-look.
 546                                                                                 // NOTE --- We're just going to do the same check from the other direction - because
 547                                                                                 //  there could be duplicates or the pointer going the other way could be broken
 548                                                                                 ArrayList <Vertex> tmpListSec = new ArrayList <> ();
 549
 550                                                                                 tmpListSec = getConnectedChildrenOfOneType( source1, parentVtx, nType ) ;
 551                                                                                 Iterator<Vertex> vIter = tmpListSec.iterator();
 552                                                                                 while (vIter.hasNext()) {
 553                                                                                         Vertex tmpV = vIter.next();
 554                                                                                         if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
 555                                                                                                 secondGetList.add(tmpV);
 556                                                                                         }
 557                                                                                 }
 558                                                                         }
 559                                                                 }
 560
 561                                                                 if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
 562                                                                         // We could not get the node back using it's own key info.
 563                                                                         // So, it's a PHANTOM
 564                                                                         if (deleteCandidateList.contains(thisVid)) {
 565                                                                                 boolean okFlag = true;
 566                                                                                 try {
 567                                                                                         thisVtx.remove();
 568                                                                                         deleteCount++;
 569                                                                                         thisNtDeleteCount++;
 570                                                                                 } catch (Exception e) {
 571                                                                                         okFlag = false;
 572                                                                                         LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e);
 573                                                                                 }
 574                                                                                 if (okFlag) {
 575                                                                                         LOGGER.info(" DELETED VID = " + thisVid);
 576                                                                                 }
 577                                                                         } else {
 578                                                                                 ghostNodeHash.put(thisVid, thisVtx);
 579                                                                         }
 580                                                                 }
 581                                                                 else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
 582                                                                         // Found some DUPLICATES - need to process them
 583                                                                         LOGGER.info(" - now check Dupes for this guy - ");
 584                                                                         List<String> tmpDupeGroups = checkAndProcessDupes(
 585                                                                                                 TRANSID, FROMAPPID, g, source1, version,
 586                                                                                                 nType, secondGetList, dupeFixOn,
 587                                                                                                 deleteCandidateList, singleCommits,     dupeGroups, loader);
 588                                                                         Iterator<String> dIter = tmpDupeGroups.iterator();
 589                                                                         while (dIter.hasNext()) {
 590                                                                                 // Add in any newly found dupes to our running list
 591                                                                                 String tmpGrp = dIter.next();
 592                                                                                 LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
 593                                                                                 dupeGroups.add(tmpGrp);
 594                                                                         }
 595                                                                 }
 596                                                         }
 597                                                         catch (AAIException e1) {
 598                                                                 LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
 599                                                                 errArr.add(e1.getErrorObject().toString());
 600                                                         }
 601                                                         catch (Exception e2) {
 602                                                                 LOGGER.warn(" For nodeType = " + nType
 603                                                                                 + " Caught exception", e2);
 604                                                                 errArr.add(e2.getMessage());
 605                                                         }
 606                                                 }// try block to enclose looping of a single vertex
 607                                                 catch (Exception exx) {
 608                                                         LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
 609                                                 }
 610
 611                                         } // while loop for each record of a nodeType
 612
 613                                         if ( (thisNtDeleteCount > 0) && singleCommits ) {
 614                                                 // NOTE - the singleCommits option is not used in normal processing
 615                                                 g.commit();
 616                                                 g = AAIGraph.getInstance().getGraph().newTransaction();
 617
 618                                         }
 619                                         thisNtDeleteCount = 0;
 620                                         LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " );
 621
 622                                 }// While-loop for each node type
 623                         }// end of check to make sure we weren't only supposed to do edges
 624
 625
 626                         // --------------------------------------------------------------------------------------
 627                         // Now, we're going to look for one-armed-edges. Ie. an edge that
 628                         // should have
 629                         // been deleted (because a vertex on one side was deleted) but
 630                         // somehow was not deleted.
 631                         // So the one end of it points to a vertexId -- but that vertex is
 632                         // empty.
 633                         // --------------------------------------------------------------------------------------
 634
 635                         // To do some strange checking - we need a second graph object
 636                         LOGGER.debug("    ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
 637                         // Note - graph2 just reads - but we want it to use a fresh connection to
 638                         //      the database, so we are NOT using the CACHED DB CONFIG here.
 639                         graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
 640                         if (graph2 == null) {
 641                                 String emsg = "null graph2 object in DataGrooming\n";
 642                                 throw new AAIException("AAI_6101", emsg);
 643                         } else {
 644                                 LOGGER.debug("Got the graph2 object... \n");
 645                         }
 646                         g2 = graph2.newTransaction();
 647                         if (g2 == null) {
 648                                 String emsg = "null graphTransaction2 object in DataGrooming\n";
 649                                 throw new AAIException("AAI_6101", emsg);
 650                         }
 651
 652                         ArrayList<Vertex> vertList = new ArrayList<>();
 653                         Iterable<? extends Vertex> vIt3 = g.query().vertices();
 654                         Iterator<? extends Vertex> vItor3 = vIt3.iterator();
 655                         // Gotta hold these in a List - or else HBase times out as you cycle
 656                         // through these
 657                         while (vItor3.hasNext()) {
 658                                 Vertex v = vItor3.next();
 659                                 vertList.add(v);
 660                         }
 661                         int counter = 0;
 662                         int lastShown = 0;
 663                         Iterator<Vertex> vItor2 = vertList.iterator();
 664                         LOGGER.info(" Checking for bad edges  --- ");
 665
 666                         while (vItor2.hasNext()) {
 667                                 Vertex v = null;
 668                                 try {
 669                                         try {
 670                                                 v = vItor2.next();
 671                                         } catch (Exception vex) {
 672                                                 LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
 673                                                 continue;
 674                                         }
 675
 676                                         counter++;
 677                                         String thisVertId = "";
 678                                         try {
 679                                                 thisVertId = v.id().toString();
 680                                         } catch (Exception ev) {
 681                                                 LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list.  ");
 682                                                 continue;
 683                                         }
 684                                         if (ghostNodeHash.containsKey(thisVertId)) {
 685                                                 // This is a phantom node, so don't try to use it
 686                                                 LOGGER.info(" >> Skipping edge check for edges from vertexId = "
 687                                                                                 + thisVertId
 688                                                                                 + ", since that guy is a Phantom Node");
 689                                                 continue;
 690                                         }
 691                                         if (counter == lastShown + 250) {
 692                                                 lastShown = counter;
 693                                                 LOGGER.info("... Checking edges for vertex # "
 694                                                                 + counter);
 695                                         }
 696                                         Iterator<Edge> eItor = v.edges(Direction.BOTH);
 697                                         while (eItor.hasNext()) {
 698                                                 Edge e = null;
 699                                                 Vertex vIn = null;
 700                                                 Vertex vOut = null;
 701                                                 try {
 702                                                         e = eItor.next();
 703                                                 } catch (Exception iex) {
 704                                                         LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
 705                                                         continue;
 706                                                 }
 707
 708                                                 try {
 709                                                         vIn = e.inVertex();
 710                                                 } catch (Exception err) {
 711                                                         LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
 712                                                 }
 713                                                 String vNtI = "";
 714                                                 String vIdI = "";
 715                                                 Vertex ghost2 = null;
 716
 717                                                 Boolean keysMissing = true;
 718                                                 Boolean cantGetUsingVid = false;
 719                                                 if (vIn != null) {
 720                                                         try {
 721                                                                 Object ob = vIn.<Object>property("aai-node-type").orElse(null);
 722                                                                 if (ob != null) {
 723                                                                         vNtI = ob.toString();
 724                                                                         keysMissing = anyKeyFieldsMissing(vNtI, vIn, loader);
 725                                                                 }
 726                                                                 ob = vIn.id();
 727                                                                 long vIdLong = 0L;
 728                                                                 if (ob != null) {
 729                                                                         vIdI = ob.toString();
 730                                                                         vIdLong = Long.parseLong(vIdI);
 731                                                                 }
 732
 733                                                                 if( ! ghost2CheckOff ){
 734                                                                         Vertex connectedVert = g2.getVertex(vIdLong);
 735                                                                         if( connectedVert == null ) {
 736                                                                                 LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
 737                                                                                 cantGetUsingVid = true;
 738
 739                                                                                 // If we can NOT get this ghost with the SECOND graph-object,
 740                                                                                 // it is still a ghost since even though we can get data about it using the FIRST graph
 741                                                                                 // object.
 742                                                                                 try {
 743                                                                                          ghost2 = g.getVertex(vIdLong);
 744                                                                                 }
 745                                                                                 catch( Exception ex){
 746                                                                                         LOGGER.warn( "GHOST2 --  Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
 747                                                                                 }
 748                                                                                 if( ghost2 != null ){
 749                                                                                         ghostNodeHash.put(vIdI, ghost2);
 750                                                                                 }
 751                                                                         }
 752                                                                 }// end of the ghost2 checking
 753                                                         }
 754                                                         catch (Exception err) {
 755                                                                 LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
 756                                                         }
 757                                                 }
 758                                                 if (keysMissing || vIn == null || vNtI.equals("")
 759                                                                 || cantGetUsingVid) {
 760                                                         // this is a bad edge because it points to a vertex
 761                                                         // that isn't there anymore or is corrupted
 762                                                         String thisEid = e.id().toString();
 763                                                         if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
 764                                                                 boolean okFlag = true;
 765                                                                 if (!vIdI.equals("")) {
 766                                                                         // try to get rid of the corrupted vertex
 767                                                                         try {
 768                                                                                 if( (ghost2 != null) && ghost2FixOn ){
 769                                                                                         ghost2.remove();
 770                                                                                 }
 771                                                                                 else {
 772                                                                                         vIn.remove();
 773                                                                                 }
 774                                                                                 if (singleCommits) {
 775                                                                                         // NOTE - the singleCommits option is not used in normal processing
 776                                                                                         g.commit();
 777                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 778                                                                                 }
 779                                                                                 deleteCount++;
 780                                                                         } catch (Exception e1) {
 781                                                                                 okFlag = false;
 782                                                                                 LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
 783                                                                                                 + vIdI, e1);
 784                                                                         }
 785                                                                         if (okFlag) {
 786                                                                                 LOGGER.info(" DELETED vertex from bad edge = "
 787                                                                                                                 + vIdI);
 788                                                                         }
 789                                                                 } else {
 790                                                                         // remove the edge if we couldn't get the
 791                                                                         // vertex
 792                                                                         try {
 793                                                                                 e.remove();
 794                                                                                 if (singleCommits) {
 795                                                                                         // NOTE - the singleCommits option is not used in normal processing
 796                                                                                         g.commit();
 797                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 798                                                                                 }
 799                                                                                 deleteCount++;
 800                                                                         } catch (Exception ex) {
 801                                                                                 // NOTE - often, the exception is just
 802                                                                                 // that this edge has already been
 803                                                                                 // removed
 804                                                                                 okFlag = false;
 805                                                                                 LOGGER.warn("WARNING when trying to delete edge = "
 806                                                                                                 + thisEid);
 807                                                                         }
 808                                                                         if (okFlag) {
 809                                                                                 LOGGER.info(" DELETED edge = " + thisEid);
 810                                                                         }
 811                                                                 }
 812                                                         } else {
 813                                                                 oneArmedEdgeHash.put(thisEid, e);
 814                                                                 if ((vIn != null) && (vIn.id() != null)) {
 815                                                                         emptyVertexHash.put(thisEid, vIn.id()
 816                                                                                         .toString());
 817                                                                 }
 818                                                         }
 819                                                 }
 820
 821                                                 try {
 822                                                         vOut = e.outVertex();
 823                                                 } catch (Exception err) {
 824                                                         LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
 825                                                 }
 826                                                 String vNtO = "";
 827                                                 String vIdO = "";
 828                                                 ghost2 = null;
 829                                                 keysMissing = true;
 830                                                 cantGetUsingVid = false;
 831                                                 if (vOut != null) {
 832                                                         try {
 833                                                                 Object ob = vOut.<Object>property("aai-node-type").orElse(null);
 834                                                                 if (ob != null) {
 835                                                                         vNtO = ob.toString();
 836                                                                         keysMissing = anyKeyFieldsMissing(vNtO,
 837                                                                                         vOut, loader);
 838                                                                 }
 839                                                                 ob = vOut.id();
 840                                                                 long vIdLong = 0L;
 841                                                                 if (ob != null) {
 842                                                                         vIdO = ob.toString();
 843                                                                         vIdLong = Long.parseLong(vIdO);
 844                                                                 }
 845
 846                                                                 if( ! ghost2CheckOff ){
 847                                                                         Vertex connectedVert = g2.getVertex(vIdLong);
 848                                                                         if( connectedVert == null ) {
 849                                                                                 cantGetUsingVid = true;
 850                                                                                 LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
 851                                                                                 // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
 852                                                                                 try {
 853                                                                                          ghost2 = g.getVertex(vIdLong);
 854                                                                                 }
 855                                                                                 catch( Exception ex){
 856                                                                                         LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
 857                                                                                 }
 858                                                                                 if( ghost2 != null ){
 859                                                                                         ghostNodeHash.put(vIdO, ghost2);
 860                                                                                 }
 861                                                                         }
 862                                                                 }
 863                                                         } catch (Exception err) {
 864                                                                 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
 865                                                         }
 866                                                 }
 867                                                 if (keysMissing || vOut == null || vNtO.equals("")
 868                                                                 || cantGetUsingVid) {
 869                                                         // this is a bad edge because it points to a vertex
 870                                                         // that isn't there anymore
 871                                                         String thisEid = e.id().toString();
 872                                                         if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
 873                                                                 boolean okFlag = true;
 874                                                                 if (!vIdO.equals("")) {
 875                                                                         // try to get rid of the corrupted vertex
 876                                                                         try {
 877                                                                                 if( (ghost2 != null) && ghost2FixOn ){
 878                                                                                         ghost2.remove();
 879                                                                                 }
 880                                                                                 else {
 881                                                                                         vOut.remove();
 882                                                                                 }
 883                                                                                 if (singleCommits) {
 884                                                                                         // NOTE - the singleCommits option is not used in normal processing
 885                                                                                         g.commit();
 886                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 887                                                                                 }
 888                                                                                 deleteCount++;
 889                                                                         } catch (Exception e1) {
 890                                                                                 okFlag = false;
 891                                                                                 LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
 892                                                                                                 + vIdO, e1);
 893                                                                         }
 894                                                                         if (okFlag) {
 895                                                                                 LOGGER.info(" DELETED vertex from bad edge = "
 896                                                                                                                 + vIdO);
 897                                                                         }
 898                                                                 } else {
 899                                                                         // remove the edge if we couldn't get the
 900                                                                         // vertex
 901                                                                         try {
 902                                                                                 e.remove();
 903                                                                                 if (singleCommits) {
 904                                                                                         // NOTE - the singleCommits option is not used in normal processing
 905                                                                                         g.commit();
 906                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 907                                                                                 }
 908                                                                                 deleteCount++;
 909                                                                         } catch (Exception ex) {
 910                                                                                 // NOTE - often, the exception is just
 911                                                                                 // that this edge has already been
 912                                                                                 // removed
 913                                                                                 okFlag = false;
 914                                                                                 LOGGER.warn("WARNING when trying to delete edge = "
 915                                                                                                 + thisEid, ex);
 916                                                                         }
 917                                                                         if (okFlag) {
 918                                                                                 LOGGER.info(" DELETED edge = " + thisEid);
 919                                                                         }
 920                                                                 }
 921                                                         } else {
 922                                                                 oneArmedEdgeHash.put(thisEid, e);
 923                                                                 if ((vOut != null) && (vOut.id() != null)) {
 924                                                                         emptyVertexHash.put(thisEid, vOut.id()
 925                                                                                         .toString());
 926                                                                 }
 927                                                         }
 928                                                 }
 929                                         }// End of while-edges-loop
 930                                 } catch (Exception exx) {
 931                                         LOGGER.warn("WARNING from in the while-verts-loop ", exx);
 932                                 }
 933                         }// End of while-vertices-loop
 934
 935                         deleteCount = deleteCount + dupeGrpsDeleted;
 936                         if (!singleCommits && deleteCount > 0) {
 937                                 try {
 938                                         LOGGER.info("About to do the commit for "
 939                                                         + deleteCount + " removes. ");
 940                                         executeFinalCommit = true;
 941                                         LOGGER.info("Commit was successful ");
 942                                 } catch (Exception excom) {
 943                                         LOGGER.error(" >>>> ERROR <<<<   Could not commit changes. ", excom);
 944                                         deleteCount = 0;
 945                                 }
 946                         }
 947
 948                         int ghostNodeCount = ghostNodeHash.size();
 949                         int orphanNodeCount = orphanNodeHash.size();
 950                         int missingDepNodeCount = missingDepNodeHash.size();
 951                         int oneArmedEdgeCount = oneArmedEdgeHash.size();
 952                         int dupeCount = dupeGroups.size();
 953
 954                         deleteCount = deleteCount + dupeGrpsDeleted;
 955
 956                         bw.write("\n\n ============ Summary ==============\n");
 957                         bw.write("Ran these nodeTypes: " + ntList + "\n\n");
 958                         bw.write("There were this many delete candidates from previous run =  "
 959                                         + deleteCandidateList.size() + "\n");
 960                         if (dontFixOrphansFlag) {
 961                                 bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
 962                         }
 963                         bw.write("Deleted this many delete candidates =  " + deleteCount
 964                                         + "\n");
 965                         bw.write("Total number of nodes looked at =  " + totalNodeCount
 966                                         + "\n");
 967                         bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
 968                         bw.write("Orphan Nodes identified =  " + orphanNodeCount + "\n");
 969                         bw.write("Bad Edges identified =  " + oneArmedEdgeCount + "\n");
 970                         bw.write("Missing Dependent Edge (but not orphaned) node count = "
 971                                         + missingDepNodeCount + "\n");
 972                         bw.write("Duplicate Groups count =  " + dupeCount + "\n");
 973                         bw.write("MisMatching Label/aai-node-type count =  "
 974                                         + misMatchedHash.size() + "\n");
 975
 976                         bw.write("\n ------------- Delete Candidates ---------\n");
 977                         for (Map.Entry<String, Vertex> entry : ghostNodeHash
 978                                         .entrySet()) {
 979                                 String vid = entry.getKey();
 980                                 bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
 981                                 cleanupCandidateCount++;
 982                         }
 983                         for (Map.Entry<String, Vertex> entry : orphanNodeHash
 984                                         .entrySet()) {
 985                                 String vid = entry.getKey();
 986                                 bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
 987                                 if (!dontFixOrphansFlag) {
 988                                         cleanupCandidateCount++;
 989                                 }
 990                         }
 991                         for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
 992                                 String eid = entry.getKey();
 993                                 bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
 994                                 cleanupCandidateCount++;
 995                         }
 996                         for (Map.Entry<String, Vertex> entry : missingDepNodeHash
 997                                         .entrySet()) {
 998                                 String vid = entry.getKey();
 999                                 bw.write("DeleteCandidate: (maybe) missingDepNode Vid = ["
1000                                                 + vid + "]\n");
1001                                 cleanupCandidateCount++;
1002                         }
1003                         bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
1004
1005                         bw.write("\n ------------- GHOST NODES - detail ");
1006                         for (Map.Entry<String, Vertex> entry : ghostNodeHash
1007                                         .entrySet()) {
1008                                 try {
1009                                         String vid = entry.getKey();
1010                                         bw.write("\n ==> Phantom Vid = " + vid + "\n");
1011                                         ArrayList<String> retArr = showPropertiesForNode(
1012                                                         TRANSID, FROMAPPID, entry.getValue());
1013                                         for (String info : retArr) {
1014                                                 bw.write(info + "\n");
1015                                         }
1016
1017                                         retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1018                                                         entry.getValue());
1019                                         for (String info : retArr) {
1020                                                 bw.write(info + "\n");
1021                                         }
1022                                 } catch (Exception dex) {
1023                                         LOGGER.error("error trying to print detail info for a ghost-node:  ", dex);
1024                                 }
1025                         }
1026
1027                         bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
1028                         for (Map.Entry<String, Vertex> entry : orphanNodeHash
1029                                         .entrySet()) {
1030                                 try {
1031                                         String vid = entry.getKey();
1032                                         bw.write("\n> Orphan Node Vid = " + vid + "\n");
1033                                         ArrayList<String> retArr = showPropertiesForNode(
1034                                                         TRANSID, FROMAPPID, entry.getValue());
1035                                         for (String info : retArr) {
1036                                                 bw.write(info + "\n");
1037                                         }
1038
1039                                         retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1040                                                         entry.getValue());
1041                                         for (String info : retArr) {
1042                                                 bw.write(info + "\n");
1043                                         }
1044                                 } catch (Exception dex) {
1045                                         LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex);
1046                                 }
1047                         }
1048
1049                         bw.write("\n ------------- Missing Dependent Edge (but not orphan) NODES: ");
1050                         for (Map.Entry<String, Vertex> entry : missingDepNodeHash
1051                                         .entrySet()) {
1052                                 try {
1053                                         String vid = entry.getKey();
1054                                         bw.write("\n>  Missing edge to Dependent Node (but has edges) Vid = "
1055                                                         + vid + "\n");
1056                                         ArrayList<String> retArr = showPropertiesForNode(
1057                                                         TRANSID, FROMAPPID, entry.getValue());
1058                                         for (String info : retArr) {
1059                                                 bw.write(info + "\n");
1060                                         }
1061
1062                                         retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1063                                                         entry.getValue());
1064                                         for (String info : retArr) {
1065                                                 bw.write(info + "\n");
1066                                         }
1067                                 } catch (Exception dex) {
1068                                         LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex);
1069                                 }
1070                         }
1071
1072                         bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
1073                         for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1074                                 try {
1075                                         String eid = entry.getKey();
1076                                         Edge thisE = entry.getValue();
1077                                         String badVid = emptyVertexHash.get(eid);
1078                                         bw.write("\n>  Edge pointing to bad vertex (Vid = "
1079                                                         + badVid + ") EdgeId = " + eid + "\n");
1080                                         bw.write("Label: [" + thisE.label() + "]\n");
1081                                         Iterator<Property<Object>> pI = thisE.properties();
1082                                         while (pI.hasNext()) {
1083                                                 Property<Object> propKey = pI.next();
1084                                                 bw.write("Prop: [" + propKey + "], val = ["
1085                                                                 + propKey.value() + "]\n");
1086                                         }
1087                                 } catch (Exception pex) {
1088                                         LOGGER.error("error trying to print empty/bad vertex data: ", pex);
1089                                 }
1090                         }
1091
1092                         bw.write("\n ------------- Duplicates: ");
1093                         Iterator<String> dupeIter = dupeGroups.iterator();
1094                         int dupeSetCounter = 0;
1095                         while (dupeIter.hasNext()) {
1096                                 dupeSetCounter++;
1097                                 String dset = (String) dupeIter.next();
1098
1099                                 bw.write("\n --- Duplicate Group # " + dupeSetCounter
1100                                                 + " Detail -----------\n");
1101                                 try {
1102                                         // We expect each line to have at least two vid's, followed
1103                                         // by the preferred one to KEEP
1104                                         String[] dupeArr = dset.split("\\|");
1105                                         ArrayList<String> idArr = new ArrayList<>();
1106                                         int lastIndex = dupeArr.length - 1;
1107                                         for (int i = 0; i <= lastIndex; i++) {
1108                                                 if (i < lastIndex) {
1109                                                         // This is not the last entry, it is one of the
1110                                                         // dupes, so we want to show all its info
1111                                                         bw.write("    >> Duplicate Group # "
1112                                                                         + dupeSetCounter + "  Node # " + i
1113                                                                         + " ----\n");
1114                                                         String vidString = dupeArr[i];
1115                                                         idArr.add(vidString);
1116                                                         long longVertId = Long.parseLong(vidString);
1117                                                         Iterator<Vertex> vtxIterator = g.vertices(longVertId);
1118                                                         Vertex vtx = null;
1119                                                         if (vtxIterator.hasNext()) {
1120                                                                 vtx = vtxIterator.next();
1121                                                         }
1122                                                         ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
1123                                                         for (String info : retArr) {
1124                                                                 bw.write(info + "\n");
1125                                                         }
1126
1127                                                         retArr = showAllEdgesForNode(TRANSID,
1128                                                                         FROMAPPID, vtx);
1129                                                         for (String info : retArr) {
1130                                                                 bw.write(info + "\n");
1131                                                         }
1132                                                 } else {
1133                                                         // This is the last entry which should tell us if we
1134                                                         // have a preferred keeper
1135                                                         String prefString = dupeArr[i];
1136                                                         if (prefString.equals("KeepVid=UNDETERMINED")) {
1137                                                                 bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
1138                                                                 bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
1139                                                         } else {
1140                                                                 // If we know which to keep, then the prefString
1141                                                                 // should look like, "KeepVid=12345"
1142                                                                 String[] prefArr = prefString.split("=");
1143                                                                 if (prefArr.length != 2
1144                                                                                 || (!prefArr[0].equals("KeepVid"))) {
1145                                                                         throw new Exception("Bad format. Expecting KeepVid=999999");
1146                                                                 } else {
1147                                                                         String keepVidStr = prefArr[1];
1148                                                                         if (idArr.contains(keepVidStr)) {
1149                                                                                 bw.write("\n The vertex we want to KEEP has vertexId = "
1150                                                                                                 + keepVidStr);
1151                                                                                 bw.write("\n The others become delete candidates: \n");
1152                                                                                 idArr.remove(keepVidStr);
1153                                                                                 for (int x = 0; x < idArr.size(); x++) {
1154                                                                                         cleanupCandidateCount++;
1155                                                                                         bw.write("DeleteCandidate: Duplicate Vid = ["
1156                                                                                                         + idArr.get(x) + "]\n");
1157                                                                                 }
1158                                                                         } else {
1159                                                                                 throw new Exception("ERROR - Vertex Id to keep not found in list of dupes.  dset = ["
1160                                                                                                 + dset + "]");
1161                                                                         }
1162                                                                 }
1163                                                         }// else we know which one to keep
1164                                                 }// else last entry
1165                                         }// for each vertex in a group
1166                                 } catch (Exception dex) {
1167                                         LOGGER.error("error trying to print duplicate vertex data", dex);
1168                                 }
1169
1170                         }// while - work on each group of dupes
1171
1172                         bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
1173                         for (Map.Entry<String, String> entry : misMatchedHash.entrySet()) {
1174                                 String msg = entry.getValue();
1175                                 bw.write("MixedMsg = " + msg + "\n");
1176                         }
1177
1178                         bw.write("\n ------------- Got these errors while processing: \n");
1179                         Iterator<String> errIter = errArr.iterator();
1180                         while (errIter.hasNext()) {
1181                                 String line = (String) errIter.next();
1182                                 bw.write(line + "\n");
1183                         }
1184
1185                         bw.close();
1186
1187                         LOGGER.info("\n ------------- Done doing all the checks ------------ ");
1188                         LOGGER.info("Output will be written to " + fullOutputFileName);
1189
1190                         if (cleanupCandidateCount > 0) {
1191                                 // Technically, this is not an error -- but we're throwing this
1192                                 // error so that hopefully a
1193                                 // monitoring system will pick it up and do something with it.
1194                                 throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
1195                                                 + "] and investigate delete candidates. ");
1196                         }
1197                 } catch (AAIException e) {
1198                         LOGGER.error("Caught AAIException while grooming data", e);
1199                         ErrorLogHelper.logException(e);
1200                 } catch (Exception ex) {
1201                         LOGGER.error("Caught exception while grooming data", ex);
1202                         ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
1203                 } finally {
1204
1205                         if (bw != null) {
1206                                 try {
1207                                         bw.close();
1208                                 } catch (IOException iox) {
1209                                         LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
1210                                 }
1211                         }
1212
1213                         if (g != null && !g.isClosed()) {
1214                                 // Any changes that worked correctly should have already done
1215                                 // their commits.
1216                                 try {
1217                                         if (executeFinalCommit) {
1218                                                 g.commit();
1219                                         }
1220                                         g.rollback();
1221                                 } catch (Exception ex) {
1222                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1223                                         LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
1224                                 }
1225                         }
1226
1227                         if (g2 != null && !g2.isClosed()) {
1228                                 // Any changes that worked correctly should have already done
1229                                 // their commits.
1230                                 try {
1231                                         g2.rollback();
1232                                 } catch (Exception ex) {
1233                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1234                                         LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
1235                                 }
1236                         }
1237
1238                         if( finalShutdownFlag ){
1239                                 try {
1240                                         if( graph != null && graph.isOpen() ){
1241                                                 graph.tx().close();
1242                                                 graph.close();
1243                                         }
1244                                 } catch (Exception ex) {
1245                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1246                                         LOGGER.warn("WARNING from final graph.shutdown()", ex);
1247                                 }
1248
1249                                 try {
1250                                         if( graph2 != null && graph2.isOpen() ){
1251                                                 graph2.tx().close();
1252                                                 graph2.close();
1253                                         }
1254                                 } catch (Exception ex) {
1255                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1256                                         LOGGER.warn("WARNING from final graph2.shutdown()", ex);
1257                                 }
1258                         }
1259
1260                 }
1261
1262                 return cleanupCandidateCount;
1263
1264         }// end of doTheGrooming()
1265
1266
1267         /**
1268          * Vertex has these keys.
1269          *
1270          * @param tmpV the tmp V
1271          * @param propHashWithKeys the prop hash with keys
1272          * @return the boolean
1273          */
1274         private static Boolean vertexHasTheseKeys( Vertex tmpV, HashMap <String, Object> propHashWithKeys) {
1275                 Iterator <?> it = propHashWithKeys.entrySet().iterator();
1276                 while( it.hasNext() ){
1277                         String propName = "";
1278                         String propVal = "";
1279                         Map.Entry <?,?>propEntry = (Map.Entry<?,?>)it.next();
1280                         Object propNameObj = propEntry.getKey();
1281                         if( propNameObj != null ){
1282                                 propName = propNameObj.toString();
1283                         }
1284                         Object propValObj = propEntry.getValue();
1285                         if( propValObj != null ){
1286                                 propVal = propValObj.toString();
1287                         }
1288                         Object checkValObj = tmpV.<Object>property(propName).orElse(null);
1289                         if( checkValObj == null ) {
1290                                 return false;
1291                         }
1292                         else if( !propVal.equals(checkValObj.toString()) ){
1293                                 return false;
1294                         }
1295                 }
1296                 return true;
1297         }
1298
1299
1300         /**
1301          * Any key fields missing.
1302          *
1303          * @param nType the n type
1304          * @param v the v
1305          * @return the boolean
1306          */
1307         private static Boolean anyKeyFieldsMissing(String nType, Vertex v, Loader loader) {
1308
1309                 try {
1310                         Introspector obj = null;
1311                         try {
1312                                 obj = loader.introspectorFromName(nType);
1313                         } catch (AAIUnknownObjectException e) {
1314                                 // They gave us a non-empty nodeType but our NodeKeyProps does
1315                                 //   not have data for it.  Since we do not know what the
1316                                 //   key params are for this type of node, we will just
1317                                 //   return "false".
1318                                 String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
1319                                                 + "].  We cannot determine required keys for this nType. ";
1320                                 // NOTE - this will be caught below and a "false" returned
1321                                 throw new AAIException("AAI_6121", emsg);
1322                         }
1323
1324                         // Determine what the key fields are for this nodeType
1325                         Collection <String> keyPropNamesColl = obj.getKeys();
1326                         Iterator<String> keyPropI = keyPropNamesColl.iterator();
1327                         while (keyPropI.hasNext()) {
1328                                 String propName = keyPropI.next();
1329                                 Object ob = v.<Object>property(propName).orElse(null);
1330                                 if (ob == null || ob.toString().equals("")) {
1331                                         // It is missing a key property
1332                                         return true;
1333                                 }
1334                         }
1335                 } catch (AAIException e) {
1336                         // Something was wrong -- but since we weren't able to check
1337                         // the keys, we will not declare that it is missing keys.
1338                         return false;
1339                 }
1340                 return false;
1341         }
1342
1343
1344         /**
1345          * Gets the delete list.
1346          *
1347          * @param targetDir the target dir
1348          * @param fileName the file name
1349          * @param edgesOnlyFlag the edges only flag
1350          * @param dontFixOrphans the dont fix orphans
1351          * @param dupeFixOn the dupe fix on
1352          * @return the delete list
1353          * @throws AAIException the AAI exception
1354          */
1355         private static Set<String> getDeleteList(String targetDir,
1356                         String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
1357                         Boolean dupeFixOn) throws AAIException {
1358
1359                 // Look in the file for lines formated like we expect - pull out any
1360                 // Vertex Id's to delete on this run
1361                 Set<String> delList = new LinkedHashSet<>();
1362                 String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
1363                 BufferedReader br = null;
1364
1365                 try {
1366                         br = new BufferedReader(new FileReader(fullFileName));
1367                         String line = br.readLine();
1368                         while (line != null) {
1369                                 if (!line.equals("") && line.startsWith("DeleteCandidate")) {
1370                                         if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
1371                                                 // We're not going to process edge guys
1372                                         } else if (dontFixOrphans && line.contains("Orphan")) {
1373                                                 // We're not going to process orphans
1374                                         } else if (!dupeFixOn && line.contains("Duplicate")) {
1375                                                 // We're not going to process Duplicates
1376                                         } else {
1377                                                 int begIndex = line.indexOf("id = ");
1378                                                 int endIndex = line.indexOf("]");
1379                                                 String vidVal = line.substring(begIndex + 6, endIndex);
1380                                                 delList.add(vidVal);
1381                                         }
1382                                 }
1383                                 line = br.readLine();
1384                         }
1385                         br.close();
1386                 } catch (IOException e) {
1387                         throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
1388                                         + "], exception= " + e.getMessage());
1389                 }
1390
1391                 return delList;
1392
1393         }// end of getDeleteList
1394
1395         /**
1396          * Gets the preferred dupe.
1397          *
1398          * @param transId the trans id
1399          * @param fromAppId the from app id
1400          * @param g the g
1401          * @param dupeVertexList the dupe vertex list
1402          * @param ver the ver
1403          * @return Vertex
1404          * @throws AAIException the AAI exception
1405          */
1406         public static Vertex getPreferredDupe(String transId,
1407                         String fromAppId, GraphTraversalSource g,
1408                         ArrayList<Vertex> dupeVertexList, String ver, Loader loader)
1409                         throws AAIException {
1410
1411                 // This method assumes that it is being passed a List of vertex objects
1412                 // which
1413                 // violate our uniqueness constraints.
1414
1415                 Vertex nullVtx = null;
1416
1417                 if (dupeVertexList == null) {
1418                         return nullVtx;
1419                 }
1420                 int listSize = dupeVertexList.size();
1421                 if (listSize == 0) {
1422                         return nullVtx;
1423                 }
1424                 if (listSize == 1) {
1425                         return (dupeVertexList.get(0));
1426                 }
1427
1428                 Vertex vtxPreferred = null;
1429                 Vertex currentFaveVtx = dupeVertexList.get(0);
1430                 for (int i = 1; i < listSize; i++) {
1431                         Vertex vtxB = dupeVertexList.get(i);
1432                         vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
1433                                         currentFaveVtx, vtxB, ver, loader);
1434                         if (vtxPreferred == null) {
1435                                 // We couldn't choose one
1436                                 return nullVtx;
1437                         } else {
1438                                 currentFaveVtx = vtxPreferred;
1439                         }
1440                 }
1441
1442                 return (currentFaveVtx);
1443
1444         } // end of getPreferredDupe()
1445
1446         /**
1447          * Pick one of two dupes.
1448          *
1449          * @param transId the trans id
1450          * @param fromAppId the from app id
1451          * @param g the g
1452          * @param vtxA the vtx A
1453          * @param vtxB the vtx B
1454          * @param ver the ver
1455          * @return Vertex
1456          * @throws AAIException the AAI exception
1457          */
1458         public static Vertex pickOneOfTwoDupes(String transId,
1459                         String fromAppId, GraphTraversalSource g, Vertex vtxA,
1460                         Vertex vtxB, String ver, Loader loader) throws AAIException {
1461
1462                 Vertex nullVtx = null;
1463                 Vertex preferredVtx = null;
1464
1465                 Long vidA = new Long(vtxA.id().toString());
1466                 Long vidB = new Long(vtxB.id().toString());
1467
1468                 String vtxANodeType = "";
1469                 String vtxBNodeType = "";
1470                 Object objType = vtxA.<Object>property("aai-node-type").orElse(null);
1471                 if (objType != null) {
1472                         vtxANodeType = objType.toString();
1473                 }
1474                 objType = vtxB.<Object>property("aai-node-type").orElse(null);
1475                 if (objType != null) {
1476                         vtxBNodeType = objType.toString();
1477                 }
1478
1479                 if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
1480                         // Either they're not really dupes or there's some bad data - so
1481                         // don't pick one
1482                         return nullVtx;
1483                 }
1484
1485                 // Check that node A and B both have the same key values (or else they
1486                 // are not dupes)
1487                 // (We'll check dep-node later)
1488                 // Determine what the key fields are for this nodeType
1489                 Collection <String> keyProps = new ArrayList <>();
1490                 try {
1491                         keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
1492                 } catch (AAIUnknownObjectException e) {
1493                         throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
1494                 }
1495
1496                 Iterator<String> keyPropI = keyProps.iterator();
1497                 while (keyPropI.hasNext()) {
1498                         String propName = keyPropI.next();
1499                         String vtxAKeyPropVal = "";
1500                         objType = vtxA.<Object>property(propName).orElse(null);
1501                         if (objType != null) {
1502                                 vtxAKeyPropVal = objType.toString();
1503                         }
1504                         String vtxBKeyPropVal = "";
1505                         objType = vtxB.<Object>property(propName).orElse(null);
1506                         if (objType != null) {
1507                                 vtxBKeyPropVal = objType.toString();
1508                         }
1509
1510                         if (vtxAKeyPropVal.equals("")
1511                                         || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
1512                                 // Either they're not really dupes or they are missing some key
1513                                 // data - so don't pick one
1514                                 return nullVtx;
1515                         }
1516                 }
1517
1518                 // Collect the vid's and aai-node-types of the vertices that each vertex
1519                 // (A and B) is connected to.
1520                 ArrayList<String> vtxIdsConn2A = new ArrayList<>();
1521                 ArrayList<String> vtxIdsConn2B = new ArrayList<>();
1522                 HashMap<String, String> nodeTypesConn2A = new HashMap<>();
1523                 HashMap<String, String> nodeTypesConn2B = new HashMap<>();
1524
1525                 ArrayList<Vertex> vertListA = getConnectedNodes( g, vtxA );
1526                 if (vertListA != null) {
1527                         Iterator<Vertex> iter = vertListA.iterator();
1528                         while (iter.hasNext()) {
1529                                 Vertex tvCon = iter.next();
1530                                 String conVid = tvCon.id().toString();
1531                                 String nt = "";
1532                                 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1533                                 if (objType != null) {
1534                                         nt = objType.toString();
1535                                 }
1536                                 nodeTypesConn2A.put(nt, conVid);
1537                                 vtxIdsConn2A.add(conVid);
1538                         }
1539                 }
1540
1541                 ArrayList<Vertex> vertListB = getConnectedNodes( g, vtxB );
1542                 if (vertListB != null) {
1543                         Iterator<Vertex> iter = vertListB.iterator();
1544                         while (iter.hasNext()) {
1545                                 Vertex tvCon = iter.next();
1546                                 String conVid = tvCon.id().toString();
1547                                 String nt = "";
1548                                 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1549                                 if (objType != null) {
1550                                         nt = objType.toString();
1551                                 }
1552                                 nodeTypesConn2B.put(nt, conVid);
1553                                 vtxIdsConn2B.add(conVid);
1554                         }
1555                 }
1556
1557                 // 1 - If this kind of node needs a dependent node for uniqueness, then
1558                 // verify that they both nodes
1559                 // point to the same dependent node (otherwise they're not really
1560                 // duplicates)
1561                 // Note - there are sometimes more than one dependent node type since
1562                 // one nodeType can be used in
1563                 // different ways. But for a particular node, it will only have one
1564                 // dependent node that it's
1565                 // connected to.
1566                 Collection <String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
1567
1568                 if (depNodeTypes.isEmpty()) {
1569                         // This kind of node is not dependent on any other. That is ok.
1570                 } else {
1571                         String depNodeVtxId4A = "";
1572                         String depNodeVtxId4B = "";
1573                         Iterator<String> iter = depNodeTypes.iterator();
1574                         while (iter.hasNext()) {
1575                                 String depNodeType = iter.next();
1576                                 if (nodeTypesConn2A.containsKey(depNodeType)) {
1577                                         // This is the dependent node type that vertex A is using
1578                                         depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
1579                                 }
1580                                 if (nodeTypesConn2B.containsKey(depNodeType)) {
1581                                         // This is the dependent node type that vertex B is using
1582                                         depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
1583                                 }
1584                         }
1585                         if (depNodeVtxId4A.equals("")
1586                                         || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
1587                                 // Either they're not really dupes or there's some bad data - so
1588                                 // don't pick either one
1589                                 return nullVtx;
1590                         }
1591                 }
1592
1593                 if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
1594                         // 2 - If they both have edges to all the same vertices, then return
1595                         // the one with the lower vertexId.
1596                         boolean allTheSame = true;
1597                         Iterator<String> iter = vtxIdsConn2A.iterator();
1598                         while (iter.hasNext()) {
1599                                 String vtxIdConn2A = iter.next();
1600                                 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1601                                         allTheSame = false;
1602                                         break;
1603                                 }
1604                         }
1605
1606                         if (allTheSame) {
1607                                 if (vidA < vidB) {
1608                                         preferredVtx = vtxA;
1609                                 } else {
1610                                         preferredVtx = vtxB;
1611                                 }
1612                         }
1613                 } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
1614                         // 3 - VertexA is connected to more things than vtxB.
1615                         // We'll pick VtxA if its edges are a superset of vtxB's edges.
1616                         boolean missingOne = false;
1617                         Iterator<String> iter = vtxIdsConn2B.iterator();
1618                         while (iter.hasNext()) {
1619                                 String vtxIdConn2B = iter.next();
1620                                 if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
1621                                         missingOne = true;
1622                                         break;
1623                                 }
1624                         }
1625                         if (!missingOne) {
1626                                 preferredVtx = vtxA;
1627                         }
1628                 } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
1629                         // 4 - VertexB is connected to more things than vtxA.
1630                         // We'll pick VtxB if its edges are a superset of vtxA's edges.
1631                         boolean missingOne = false;
1632                         Iterator<String> iter = vtxIdsConn2A.iterator();
1633                         while (iter.hasNext()) {
1634                                 String vtxIdConn2A = iter.next();
1635                                 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1636                                         missingOne = true;
1637                                         break;
1638                                 }
1639                         }
1640                         if (!missingOne) {
1641                                 preferredVtx = vtxB;
1642                         }
1643                 } else {
1644                         preferredVtx = nullVtx;
1645                 }
1646
1647                 return (preferredVtx);
1648
1649         } // end of pickOneOfTwoDupes()
1650
1651         /**
1652          * Check and process dupes.
1653          *
1654          * @param transId the trans id
1655          * @param fromAppId the from app id
1656          * @param g the g
1657          * @param version the version
1658          * @param nType the n type
1659          * @param passedVertList the passed vert list
1660          * @param dupeFixOn the dupe fix on
1661          * @param deleteCandidateList the delete candidate list
1662          * @param singleCommits the single commits
1663          * @param alreadyFoundDupeGroups the already found dupe groups
1664          * @param dbMaps the db maps
1665          * @return the array list
1666          */
1667         private static List<String> checkAndProcessDupes(String transId,
1668                         String fromAppId, TitanTransaction g, GraphTraversalSource source, String version, String nType,
1669                         List<Vertex> passedVertList, Boolean dupeFixOn,
1670                         Set<String> deleteCandidateList, Boolean singleCommits,
1671                         ArrayList<String> alreadyFoundDupeGroups, Loader loader ) {
1672
1673                 ArrayList<String> returnList = new ArrayList<>();
1674                 ArrayList<Vertex> checkVertList = new ArrayList<>();
1675                 ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
1676                 Boolean noFilterList = true;
1677                 Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
1678                 while (afItr.hasNext()) {
1679                         String dupeGrpStr = afItr.next();
1680                         String[] dupeArr = dupeGrpStr.split("\\|");
1681                         int lastIndex = dupeArr.length - 1;
1682                         for (int i = 0; i < lastIndex; i++) {
1683                                 // Note: we don't want the last one...
1684                                 String vidString = dupeArr[i];
1685                                 alreadyFoundDupeVidArr.add(vidString);
1686                                 noFilterList = false;
1687                         }
1688                 }
1689
1690                 // For a given set of Nodes that were found with a set of KEY
1691                 // Parameters, (nodeType + key data) we will
1692                 // see if we find any duplicate nodes that need to be cleaned up. Note -
1693                 // it's legit to have more than one
1694                 // node with the same key data if the nodes depend on a parent for
1695                 // uniqueness -- as long as the two nodes
1696                 // don't hang off the same Parent.
1697                 // If we find duplicates, and we can figure out which of each set of
1698                 // duplicates is the one that we
1699                 // think should be preserved, we will record that. Whether we can tell
1700                 // which one should be
1701                 // preserved or not, we will return info about any sets of duplicates
1702                 // found.
1703                 //
1704                 // Each element in the returned arrayList might look like this:
1705                 // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
1706                 // couldn't figure out which one to keep)
1707                 // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
1708                 // thought the third one was the one that should survive)
1709
1710                 // Because of the way the calling code loops over stuff, we can get the
1711                 // same data multiple times - so we should
1712                 // not process any vertices that we've already seen.
1713
1714                 try {
1715                         Iterator<Vertex> pItr = passedVertList.iterator();
1716                         while (pItr.hasNext()) {
1717                                 Vertex tvx = pItr.next();
1718                                 String passedId = tvx.id().toString();
1719                                 if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
1720                                         // We haven't seen this one before - so we should check it.
1721                                         checkVertList.add(tvx);
1722                                 }
1723                         }
1724
1725                         if (checkVertList.size() < 2) {
1726                                 // Nothing new to check.
1727                                 return returnList;
1728                         }
1729
1730                         if (loader.introspectorFromName(nType).isTopLevel()) {
1731                                 // If this was a node that does NOT depend on other nodes for
1732                                 // uniqueness, and we
1733                                 // found more than one node using its key -- record the found
1734                                 // vertices as duplicates.
1735                                 String dupesStr = "";
1736                                 for (int i = 0; i < checkVertList.size(); i++) {
1737                                         dupesStr = dupesStr
1738                                                         + ((checkVertList.get(i))).id()
1739                                                                         .toString() + "|";
1740                                 }
1741                                 if (dupesStr != "") {
1742                                         Vertex prefV = getPreferredDupe(transId, fromAppId,
1743                                                         source, checkVertList, version, loader);
1744                                         if (prefV == null) {
1745                                                 // We could not determine which duplicate to keep
1746                                                 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1747                                                 returnList.add(dupesStr);
1748                                         } else {
1749                                                 dupesStr = dupesStr + "KeepVid=" + prefV.id();
1750                                                 Boolean didRemove = false;
1751                                                 if (dupeFixOn) {
1752                                                         didRemove = deleteNonKeepersIfAppropriate(g,
1753                                                                         dupesStr, prefV.id().toString(),
1754                                                                         deleteCandidateList, singleCommits);
1755                                                 }
1756                                                 if (didRemove) {
1757                                                         dupeGrpsDeleted++;
1758                                                 } else {
1759                                                         // keep them on our list
1760                                                         returnList.add(dupesStr);
1761                                                 }
1762                                         }
1763                                 }
1764                         } else {
1765                                 // More than one node have the same key fields since they may
1766                                 // depend on a parent node for uniqueness. Since we're finding
1767                                 // more than one, we want to check to see if any of the
1768                                 // vertices that have this set of keys (and are the same nodeType)
1769                                 // are also pointing at the same 'parent' node.
1770                                 // Note: for a given set of key data, it is possible that there
1771                                 // could be more than one set of duplicates.
1772                                 HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
1773                                                 transId, fromAppId, source, version, nType,
1774                                                 checkVertList, loader);
1775                                 for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
1776                                                 .entrySet()) {
1777                                         ArrayList<Vertex> thisParentsVertList = entry
1778                                                         .getValue();
1779                                         if (thisParentsVertList.size() > 1) {
1780                                                 // More than one vertex found with the same key info
1781                                                 // hanging off the same parent/dependent node
1782                                                 String dupesStr = "";
1783                                                 for (int i = 0; i < thisParentsVertList.size(); i++) {
1784                                                         dupesStr = dupesStr
1785                                                                         + ((thisParentsVertList
1786                                                                                         .get(i))).id() + "|";
1787                                                 }
1788                                                 if (dupesStr != "") {
1789                                                         Vertex prefV = getPreferredDupe(transId,
1790                                                                         fromAppId, source, thisParentsVertList,
1791                                                                         version, loader);
1792
1793                                                         if (prefV == null) {
1794                                                                 // We could not determine which duplicate to
1795                                                                 // keep
1796                                                                 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1797                                                                 returnList.add(dupesStr);
1798                                                         } else {
1799                                                                 Boolean didRemove = false;
1800                                                                 dupesStr = dupesStr + "KeepVid="
1801                                                                                 + prefV.id().toString();
1802                                                                 if (dupeFixOn) {
1803                                                                         didRemove = deleteNonKeepersIfAppropriate(
1804                                                                                         g, dupesStr, prefV.id()
1805                                                                                                         .toString(),
1806                                                                                         deleteCandidateList, singleCommits);
1807                                                                 }
1808                                                                 if (didRemove) {
1809                                                                         dupeGrpsDeleted++;
1810                                                                 } else {
1811                                                                         // keep them on our list
1812                                                                         returnList.add(dupesStr);
1813                                                                 }
1814                                                         }
1815                                                 }
1816                                         }
1817                                 }
1818                         }
1819                 } catch (Exception e) {
1820                         LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
1821                 }
1822
1823                 return returnList;
1824
1825         }// End of checkAndProcessDupes()
1826
1827         /**
1828          * Group verts by dep nodes.
1829          *
1830          * @param transId the trans id
1831          * @param fromAppId the from app id
1832          * @param g the g
1833          * @param version the version
1834          * @param nType the n type
1835          * @param passedVertList the passed vert list
1836          * @param dbMaps the db maps
1837          * @return the hash map
1838          * @throws AAIException the AAI exception
1839          */
1840         private static HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
1841                         String transId, String fromAppId, GraphTraversalSource g, String version,
1842                         String nType, ArrayList<Vertex> passedVertList, Loader loader)
1843                         throws AAIException {
1844                 // Given a list of Titan Vertices of one nodeType (see AAI-8956), group
1845                 // them together by the parent node they depend on.
1846                 // Ie. if given a list of ip address nodes (assumed to all have the
1847                 // same key info) they might sit under several different parent vertices.
1848                 // Under Normal conditions, there would only be one per parent -- but
1849                 // we're trying to find duplicates - so we
1850                 // allow for the case where more than one is under the same parent node.
1851
1852                 HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
1853                 if (loader.introspectorFromName(nType).isTopLevel()) {
1854                         // This method really should not have been called if this is not the
1855                         // kind of node
1856                         // that depends on a parent for uniqueness, so just return the empty
1857                         // hash.
1858                         return retHash;
1859                 }
1860
1861                 // Find out what types of nodes the passed in nodes can depend on
1862                 ArrayList<String> depNodeTypeL = new ArrayList<>();
1863                 Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn();
1864                 Iterator<String> ntItr = depNTColl.iterator();
1865                 while (ntItr.hasNext()) {
1866                         depNodeTypeL.add(ntItr.next());
1867                 }
1868                 // For each vertex, we want find its depended-on/parent vertex so we
1869                 // can track what other vertexes that are dependent on that same guy.
1870                 if (passedVertList != null) {
1871                         Iterator<Vertex> iter = passedVertList.iterator();
1872                         while (iter.hasNext()) {
1873                                 Vertex thisVert = iter.next();
1874                                 Vertex tmpParentVtx = getConnectedParent( g, thisVert );
1875                                 if( tmpParentVtx != null ) {
1876                                         String parentNt = null;
1877                                         Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
1878                                         if (obj != null) {
1879                                                 parentNt = obj.toString();
1880                                         }
1881                                         if (depNTColl.contains(parentNt)) {
1882                                                 // This must be the parent/dependent node
1883                                                 String parentVid = tmpParentVtx.id().toString();
1884                                                 if (retHash.containsKey(parentVid)) {
1885                                                         // add this vert to the list for this parent key
1886                                                         retHash.get(parentVid).add(thisVert);
1887                                                 } else {
1888                                                         // This is the first one we found on this parent
1889                                                         ArrayList<Vertex> vList = new ArrayList<>();
1890                                                         vList.add(thisVert);
1891                                                         retHash.put(parentVid, vList);
1892                                                 }
1893                                         }
1894                                 }
1895                         }
1896                 }
1897
1898                 return retHash;
1899
1900         }// end of groupVertsByDepNodes()
1901
1902         /**
1903          * Delete non keepers if appropriate.
1904          *
1905          * @param g the g
1906          * @param dupeInfoString the dupe info string
1907          * @param vidToKeep the vid to keep
1908          * @param deleteCandidateList the delete candidate list
1909          * @param singleCommits the single commits
1910          * @return the boolean
1911          */
1912         private static Boolean deleteNonKeepersIfAppropriate(TitanTransaction g,
1913                         String dupeInfoString, String vidToKeep,
1914                         Set<String> deleteCandidateList, Boolean singleCommits) {
1915
1916                 Boolean deletedSomething = false;
1917                 // This assumes that the dupeInfoString is in the format of
1918                 // pipe-delimited vid's followed by
1919                 // ie. "3456|9880|keepVid=3456"
1920                 if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
1921                         // No vid's on the candidate list -- so no deleting will happen on
1922                         // this run
1923                         return false;
1924                 }
1925
1926                 String[] dupeArr = dupeInfoString.split("\\|");
1927                 ArrayList<String> idArr = new ArrayList<>();
1928                 int lastIndex = dupeArr.length - 1;
1929                 for (int i = 0; i <= lastIndex; i++) {
1930                         if (i < lastIndex) {
1931                                 // This is not the last entry, it is one of the dupes,
1932                                 String vidString = dupeArr[i];
1933                                 idArr.add(vidString);
1934                         } else {
1935                                 // This is the last entry which should tell us if we have a
1936                                 // preferred keeper
1937                                 String prefString = dupeArr[i];
1938                                 if (prefString.equals("KeepVid=UNDETERMINED")) {
1939                                         // They sent us a bad string -- nothing should be deleted if
1940                                         // no dupe could be tagged as preferred
1941                                         return false;
1942                                 } else {
1943                                         // If we know which to keep, then the prefString should look
1944                                         // like, "KeepVid=12345"
1945                                         String[] prefArr = prefString.split("=");
1946                                         if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
1947                                                 LOGGER.error("Bad format. Expecting KeepVid=999999");
1948                                                 return false;
1949                                         } else {
1950                                                 String keepVidStr = prefArr[1];
1951                                                 if (idArr.contains(keepVidStr)) {
1952                                                         idArr.remove(keepVidStr);
1953
1954                                                         // So now, the idArr should just contain the vid's
1955                                                         // that we want to remove.
1956                                                         for (int x = 0; x < idArr.size(); x++) {
1957                                                                 boolean okFlag = true;
1958                                                                 String thisVid = idArr.get(x);
1959                                                                 if (deleteCandidateList.contains(thisVid)) {
1960                                                                         // This vid is a valid delete candidate from
1961                                                                         // a prev. run, so we can remove it.
1962                                                                         try {
1963                                                                                 long longVertId = Long
1964                                                                                                 .parseLong(thisVid);
1965                                                                                 Vertex vtx = g
1966                                                                                                 .getVertex(longVertId);
1967                                                                                 vtx.remove();
1968                                                                                 if (singleCommits) {
1969                                                                                         // NOTE - the singleCommits option is not used in normal processing
1970                                                                                         g.commit();
1971                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
1972                                                                                 }
1973                                                                         } catch (Exception e) {
1974                                                                                 okFlag = false;
1975                                                                                 LOGGER.error("ERROR trying to delete VID = " + thisVid, e);
1976                                                                         }
1977                                                                         if (okFlag) {
1978                                                                                 LOGGER.info(" DELETED VID = " + thisVid);
1979                                                                                 deletedSomething = true;
1980                                                                         }
1981                                                                 }
1982                                                         }
1983                                                 } else {
1984                                                         LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes.  dupeInfoString = ["
1985                                                                         + dupeInfoString + "]");
1986                                                         return false;
1987                                                 }
1988                                         }
1989                                 }// else we know which one to keep
1990                         }// else last entry
1991                 }// for each vertex in a group
1992
1993                 return deletedSomething;
1994
1995         }// end of deleteNonKeepersIfAppropriate()
1996
1997
1998         /**
1999          * Gets the node just using key params.
2000          *
2001          * @param transId the trans id
2002          * @param fromAppId the from app id
2003          * @param graph the graph
2004          * @param nodeType the node type
2005          * @param keyPropsHash the key props hash
2006          * @param apiVersion the api version
2007          * @return the node just using key params
2008          * @throws AAIException the AAI exception
2009          */
2010         public static List <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, GraphTraversalSource graph, String nodeType,
2011                         HashMap<String,Object> keyPropsHash, String apiVersion )         throws AAIException{
2012
2013                 List <Vertex> retVertList = new ArrayList <> ();
2014
2015                 // We assume that all NodeTypes have at least one key-property defined.
2016                 // Note - instead of key-properties (the primary key properties), a user could pass
2017                 //        alternate-key values if they are defined for the nodeType.
2018                 List<String> kName = new ArrayList<>();
2019                 List<Object> kVal = new ArrayList<>();
2020                 if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
2021                         throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request.  NodeType = [" + nodeType + "]. ");
2022                 }
2023
2024                 int i = -1;
2025                 for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){
2026                         i++;
2027                         kName.add(i, entry.getKey());
2028                         kVal.add(i, entry.getValue());
2029                 }
2030                 int topPropIndex = i;
2031                 Vertex tiV = null;
2032                 String propsAndValuesForMsg = "";
2033                 Iterator <Vertex> verts = null;
2034
2035                 try {
2036                         if( topPropIndex == 0 ){
2037                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
2038                                 verts= graph.V().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType);
2039                         }
2040                         else if( topPropIndex == 1 ){
2041                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2042                                                 + kName.get(1) + " = " + kVal.get(1) + ") ";
2043                                 verts =  graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType);
2044                         }
2045                         else if( topPropIndex == 2 ){
2046                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2047                                                 + kName.get(1) + " = " + kVal.get(1) + ", "
2048                                                 + kName.get(2) + " = " + kVal.get(2) +  ") ";
2049                                 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType);
2050                         }
2051                         else if( topPropIndex == 3 ){
2052                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2053                                                 + kName.get(1) + " = " + kVal.get(1) + ", "
2054                                                 + kName.get(2) + " = " + kVal.get(2) + ", "
2055                                                 + kName.get(3) + " = " + kVal.get(3) +  ") ";
2056                                 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType);
2057                         }
2058                         else {
2059                                 throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
2060                         }
2061                 }
2062                 catch( Exception ex ){
2063                         LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex);
2064                 }
2065
2066                 if( verts != null ){
2067                         while( verts.hasNext() ){
2068                                 tiV = verts.next();
2069                                 retVertList.add(tiV);
2070                         }
2071                 }
2072
2073                 if( retVertList.size() == 0 ){
2074                         LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
2075                                         "], propsAndVal = " + propsAndValuesForMsg );
2076                 }
2077
2078                 return retVertList;
2079
2080         }// End of getNodeJustUsingKeyParams()
2081
2082         /**
2083          * Show all edges for node.
2084          *
2085          * @param transId the trans id
2086          * @param fromAppId the from app id
2087          * @param tVert the t vert
2088          * @return the array list
2089          */
2090         private static ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, Vertex tVert ){
2091
2092                 ArrayList <String> retArr = new ArrayList <> ();
2093                 Iterator <Edge> eI = tVert.edges(Direction.IN);
2094                 if( ! eI.hasNext() ){
2095                         retArr.add("No IN edges were found for this vertex. ");
2096                 }
2097                 while( eI.hasNext() ){
2098                         TitanEdge ed = (TitanEdge) eI.next();
2099                         String lab = ed.label();
2100                         Vertex vtx = ed.otherVertex(tVert);
2101                         if( vtx == null ){
2102                                 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2103                         }
2104                         else {
2105                                 String nType = vtx.<String>property("aai-node-type").orElse(null);
2106                                 String vid = vtx.id().toString();
2107                                 retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
2108
2109                         }
2110                 }
2111
2112                 eI = tVert.edges(Direction.OUT);
2113                 if( ! eI.hasNext() ){
2114                         retArr.add("No OUT edges were found for this vertex. ");
2115                 }
2116                 while( eI.hasNext() ){
2117                         TitanEdge ed = (TitanEdge) eI.next();
2118                         String lab = ed.label();
2119                         Vertex vtx = ed.otherVertex(tVert);
2120                         if( vtx == null ){
2121                                 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2122                         }
2123                         else {
2124                                 String nType = vtx.<String>property("aai-node-type").orElse(null);
2125                                 String vid = vtx.id().toString();
2126                                 retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
2127                         }
2128                 }
2129                 return retArr;
2130         }
2131
2132
2133         /**
2134          * Show properties for node.
2135          *
2136          * @param transId the trans id
2137          * @param fromAppId the from app id
2138          * @param tVert the t vert
2139          * @return the array list
2140          */
2141         private static ArrayList <String> showPropertiesForNode( String transId, String fromAppId, Vertex tVert ){
2142
2143                 ArrayList <String> retArr = new ArrayList <> ();
2144                 if( tVert == null ){
2145                         retArr.add("null Node object passed to showPropertiesForNode()\n");
2146                 }
2147                 else {
2148                         String nodeType = "";
2149                         Object ob = tVert.<Object>property("aai-node-type").orElse(null);
2150                         if( ob == null ){
2151                                 nodeType = "null";
2152                         }
2153                         else{
2154                                 nodeType = ob.toString();
2155                         }
2156
2157                         retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
2158                         retArr.add(" Property Detail: ");
2159                         Iterator<VertexProperty<Object>> pI = tVert.properties();
2160                         while( pI.hasNext() ){
2161                                 VertexProperty<Object> tp = pI.next();
2162                                 Object val = tp.value();
2163                                 retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
2164                         }
2165                 }
2166                 return retArr;
2167         }
2168
2169
2170         private static ArrayList <Vertex> getConnectedNodes(GraphTraversalSource g, Vertex startVtx )
2171                         throws AAIException {
2172
2173                 ArrayList <Vertex> retArr = new ArrayList <> ();
2174                 if( startVtx == null ){
2175                         return retArr;
2176                 }
2177                 else {
2178                          GraphTraversal<Vertex, Vertex> modPipe = null;
2179                          modPipe = g.V(startVtx).both();
2180                          if( modPipe != null && modPipe.hasNext() ){
2181                                 while( modPipe.hasNext() ){
2182                                         Vertex conVert = modPipe.next();
2183                                         retArr.add(conVert);
2184                                 }
2185                         }
2186                 }
2187                 return retArr;
2188
2189         }// End of getConnectedNodes()
2190
2191
2192         private static ArrayList <Vertex> getConnectedChildrenOfOneType( GraphTraversalSource g,
2193                         Vertex startVtx, String childNType ) throws AAIException{
2194
2195                 ArrayList <Vertex> childList = new ArrayList <> ();
2196                 Iterator <Vertex> vertI = g.V(startVtx).union(__.outE().has(EdgeProperties.out(EdgeProperty.IS_PARENT), true), __.inE().has(EdgeProperties.in(EdgeProperty.IS_PARENT), true)).bothV();
2197                 Vertex tmpVtx = null;
2198                 while( vertI != null && vertI.hasNext() ){
2199                         tmpVtx = vertI.next();
2200                         Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
2201                         if (ob != null) {
2202                                 String tmpNt = ob.toString();
2203                                 if( tmpNt.equals(childNType)){
2204                                         childList.add(tmpVtx);
2205                                 }
2206                         }
2207                 }
2208
2209                 return childList;
2210
2211         }// End of getConnectedChildrenOfOneType()
2212
2213
2214         private static Vertex getConnectedParent( GraphTraversalSource g,
2215                         Vertex startVtx ) throws AAIException{
2216
2217                 Vertex parentVtx = null;
2218                 Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperties.out(EdgeProperty.IS_PARENT), true), __.outE().has(EdgeProperties.in(EdgeProperty.IS_PARENT), true)).bothV();
2219                 while( vertI != null && vertI.hasNext() ){
2220                         // Note - there better only be one!
2221                         parentVtx = vertI.next();
2222                 }
2223
2224                 return parentVtx;
2225
2226         }// End of getConnectedParent()
2227
2228
2229 }