aai-core/src/main/java/org/openecomp/aai/dbgen/DataGrooming.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * org.openecomp.aai
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.aai.dbgen;
  22
  23 import java.io.BufferedReader;
  24 import java.io.BufferedWriter;
  25 import java.io.File;
  26 import java.io.FileReader;
  27 import java.io.FileWriter;
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.Collection;
  31 import java.util.HashMap;
  32 import java.util.Iterator;
  33 import java.util.LinkedHashSet;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.Map.Entry;
  37 import java.util.Properties;
  38 import java.util.Set;
  39 import java.util.UUID;
  40
  41 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
  42 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
  43 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
  44 import org.apache.tinkerpop.gremlin.structure.Direction;
  45 import org.apache.tinkerpop.gremlin.structure.Edge;
  46 import org.apache.tinkerpop.gremlin.structure.Graph;
  47 import org.apache.tinkerpop.gremlin.structure.Property;
  48 import org.apache.tinkerpop.gremlin.structure.Vertex;
  49 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
  50 import org.openecomp.aai.db.props.AAIProperties;
  51 import org.openecomp.aai.dbmap.AAIGraph;
  52 import org.openecomp.aai.exceptions.AAIException;
  53 import org.openecomp.aai.introspection.Introspector;
  54 import org.openecomp.aai.introspection.Loader;
  55 import org.openecomp.aai.introspection.LoaderFactory;
  56 import org.openecomp.aai.introspection.ModelType;
  57 import org.openecomp.aai.introspection.exceptions.AAIUnknownObjectException;
  58 import org.openecomp.aai.logging.ErrorLogHelper;
  59 import org.openecomp.aai.serialization.db.AAIDirection;
  60 import org.openecomp.aai.serialization.db.EdgeProperty;
  61 import org.openecomp.aai.util.AAIConfig;
  62 import org.openecomp.aai.util.AAIConstants;
  63 import org.openecomp.aai.util.FormatDate;
  64
  65 import com.att.eelf.configuration.Configuration;
  66 import com.att.eelf.configuration.EELFLogger;
  67 import com.att.eelf.configuration.EELFManager;
  68 import com.thinkaurelius.titan.core.TitanFactory;
  69 import com.thinkaurelius.titan.core.TitanGraph;
  70
  71
  72 public class DataGrooming {
  73
  74         private static EELFLogger LOGGER;
  75         private static final String FROMAPPID = "AAI-DB";
  76         private static final String TRANSID = UUID.randomUUID().toString();
  77         private static int dupeGrpsDeleted = 0;
  78
  79         /**
  80          * The main method.
  81          *
  82          * @param args the arguments
  83          */
  84         public static void main(String[] args) {
  85
  86                 // Set the logging file properties to be used by EELFManager
  87                 Properties props = System.getProperties();
  88                 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_DATA_GROOMING_LOGBACK_PROPS);
  89                 props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
  90                 LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
  91                 String ver = "version"; // Placeholder
  92                 Boolean doAutoFix = false;
  93                 Boolean edgesOnlyFlag = false;
  94                 Boolean dontFixOrphansFlag = false;
  95                 Boolean skipHostCheck = false;
  96                 Boolean singleCommits = false;
  97                 Boolean dupeCheckOff = false;
  98                 Boolean dupeFixOn = false;
  99                 Boolean ghost2CheckOff = false;
 100                 Boolean ghost2FixOn = false;
 101                 Boolean neverUseCache = false;
 102                 Boolean skipEdgeCheckFlag = false;
 103
 104                 int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look
 105                                    // at all nodes of the passed-in nodeType.
 106                 long windowStartTime = 0; // Translation of the window into a starting timestamp
 107
 108                 int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
 109                 int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
 110                 try {
 111                         String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
 112                         if( maxFixStr != null &&  !maxFixStr.equals("") ){
 113                                 maxRecordsToFix = Integer.parseInt(maxFixStr);
 114                         }
 115                         String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
 116                         if( sleepStr != null &&  !sleepStr.equals("") ){
 117                                 sleepMinutes = Integer.parseInt(sleepStr);
 118                         }
 119                 }
 120                 catch ( Exception e ){
 121                         // Don't worry, we'll just use the defaults that we got from AAIConstants
 122                         LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
 123                 }
 124
 125                 String prevFileName = "";
 126                 dupeGrpsDeleted = 0;
 127                 FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
 128                 String dteStr = fd.getDateTime();
 129                 String groomOutFileName = "dataGrooming." + dteStr + ".out";
 130
 131                 String argString = "";
 132                 for( int x = 0; x < args.length; x++ ) {
 133                         argString = argString + " " + args[x];
 134                 }
 135                 LOGGER.info(" DataGrooming called with these options: [" + argString + "]");
 136
 137                 if (args.length > 0) {
 138                         // They passed some arguments in that will affect processing
 139
 140                         for (int i = 0; i < args.length; i++) {
 141                                 String thisArg = args[i];
 142                                 if (thisArg.equals("-edgesOnly")) {
 143                                         edgesOnlyFlag = true;
 144                                 } else if (thisArg.equals("-autoFix")) {
 145                                         doAutoFix = true;
 146                                 } else if (thisArg.equals("-skipHostCheck")) {
 147                                         skipHostCheck = true;
 148                                 } else if (thisArg.equals("-dontFixOrphans")) {
 149                                         dontFixOrphansFlag = true;
 150                                 } else if (thisArg.equals("-singleCommits")) {
 151                                         singleCommits = true;
 152                                 } else if (thisArg.equals("-dupeCheckOff")) {
 153                                         dupeCheckOff = true;
 154                                 } else if (thisArg.equals("-dupeFixOn")) {
 155                                         dupeFixOn = true;
 156                                 } else if (thisArg.equals("-ghost2CheckOff")) {
 157                                         ghost2CheckOff = true;
 158                                 } else if (thisArg.equals("-neverUseCache")) {
 159                                         neverUseCache = true;
 160                                 } else if (thisArg.equals("-ghost2FixOn")) {
 161                                         ghost2FixOn = true;
 162                                 } else if (thisArg.equals("-skipEdgeChecks")) {
 163                                         skipEdgeCheckFlag = true;
 164                                 } else if (thisArg.equals("-maxFix")) {
 165                                         i++;
 166                                         if (i >= args.length) {
 167                                                 LOGGER.error(" No value passed with -maxFix option.  ");
 168                                                 System.exit(0);
 169                                         }
 170                                         String nextArg = args[i];
 171                                         try {
 172                                                 maxRecordsToFix = Integer.parseInt(nextArg);
 173                                         } catch (Exception e) {
 174                                                 LOGGER.error("Bad value passed with -maxFix option: ["
 175                                                                                 + nextArg + "]");
 176                                                 System.exit(0);
 177                                         }
 178                                 } else if (thisArg.equals("-sleepMinutes")) {
 179                                         i++;
 180                                         if (i >= args.length) {
 181                                                 LOGGER.error("No value passed with -sleepMinutes option.");
 182                                                 System.exit(0);
 183                                         }
 184                                         String nextArg = args[i];
 185                                         try {
 186                                                 sleepMinutes = Integer.parseInt(nextArg);
 187                                         } catch (Exception e) {
 188                                                 LOGGER.error("Bad value passed with -sleepMinutes option: ["
 189                                                                                 + nextArg + "]");
 190                                                 System.exit(0);
 191                                         }
 192                                 } else if (thisArg.equals("-timeWindowMinutes")) {
 193                                         i++;
 194                                         if (i >= args.length) {
 195                                                 LOGGER.error("No value passed with -timeWindowMinutes option.");
 196                                                 System.exit(0);
 197                                         }
 198                                         String nextArg = args[i];
 199                                         try {
 200                                                 timeWindowMinutes = Integer.parseInt(nextArg);
 201                                         } catch (Exception e) {
 202                                                 LOGGER.error("Bad value passed with -timeWindowMinutes option: ["
 203                                                                                 + nextArg + "]");
 204                                                 System.exit(0);
 205                                         }
 206                                         if( timeWindowMinutes > 0 ){
 207                                                 // Translate the window value (ie. 30 minutes) into a unix timestamp like
 208                                                 //    we use in the db - so we can select data created after that time.
 209                                                 windowStartTime = figureWindowStartTime( timeWindowMinutes );
 210                                         }
 211                                 } else if (thisArg.equals("-f")) {
 212                                         i++;
 213                                         if (i >= args.length) {
 214                                                 LOGGER.error(" No value passed with -f option. ");
 215                                                 System.exit(0);
 216                                         }
 217                                         prevFileName = args[i];
 218                                 } else {
 219                                         LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
 220                                                                         + thisArg + "]. ");
 221                                         LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -dupeFixOn -donFixOrphans -sleepMinutes -neverUseCache");
 222                                         System.exit(0);
 223                                 }
 224                         }
 225                 }
 226
 227
 228                 try {
 229                         LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
 230
 231                 }
 232                 catch (Exception ex){
 233                         LOGGER.error("ERROR - Could not create loader", ex);
 234                         System.exit(1);
 235                 }
 236
 237
 238                 try {
 239                         if (!prevFileName.equals("")) {
 240                                 // They are trying to fix some data based on a data in a
 241                                 // previous file.
 242                                 LOGGER.info(" Call doTheGrooming() with a previous fileName ["
 243                                                                 + prevFileName + "] for cleanup. ");
 244                                 Boolean finalShutdownFlag = true;
 245                                 Boolean cacheDbOkFlag = false;
 246                                 doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
 247                                                 maxRecordsToFix, groomOutFileName, ver, singleCommits,
 248                                                 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 249                                                 finalShutdownFlag, cacheDbOkFlag,
 250                                                 skipEdgeCheckFlag, windowStartTime);
 251                         } else if (doAutoFix) {
 252                                 // They want us to run the processing twice -- first to look for
 253                                 // delete candidates, then after
 254                                 // napping for a while, run it again and delete any candidates
 255                                 // that were found by the first run.
 256                                 // Note: we will produce a separate output file for each of the
 257                                 // two runs.
 258                                 LOGGER.info(" Doing an auto-fix call to Grooming. ");
 259                                 LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
 260                                 Boolean finalShutdownFlag = false;
 261                                 Boolean cacheDbOkFlag = true;
 262                                 int fixCandCount = doTheGrooming("", edgesOnlyFlag,
 263                                                 dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
 264                                                 ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 265                                                 finalShutdownFlag, cacheDbOkFlag,
 266                                                 skipEdgeCheckFlag, windowStartTime);
 267                                 if (fixCandCount == 0) {
 268                                         LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
 269                                 } else {
 270                                         // We'll sleep a little and then run a fix-pass based on the
 271                                         // first-run's output file.
 272                                         try {
 273                                                 LOGGER.info("About to sleep for " + sleepMinutes
 274                                                                 + " minutes.");
 275                                                 int sleepMsec = sleepMinutes * 60 * 1000;
 276                                                 Thread.sleep(sleepMsec);
 277                                         } catch (InterruptedException ie) {
 278                                                 LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
 279                                                 System.exit(0);
 280                                         }
 281
 282                                         dteStr = fd.getDateTime();
 283                                         String secondGroomOutFileName = "dataGrooming." + dteStr
 284                                                         + ".out";
 285                                         LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
 286                                                                         + "generated by the first pass for fixing: ["
 287                                                                         + groomOutFileName + "]");
 288                                         finalShutdownFlag = true;
 289                                         cacheDbOkFlag = false;
 290                                         doTheGrooming(groomOutFileName, edgesOnlyFlag,
 291                                                         dontFixOrphansFlag, maxRecordsToFix,
 292                                                         secondGroomOutFileName, ver, singleCommits,
 293                                                         dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 294                                                         finalShutdownFlag, cacheDbOkFlag,
 295                                                         skipEdgeCheckFlag, windowStartTime);
 296                                 }
 297                         } else {
 298                                 // Do the grooming - plain vanilla (no fix-it-file, no
 299                                 // auto-fixing)
 300                                 Boolean finalShutdownFlag = true;
 301                                 LOGGER.info(" Call doTheGrooming() ");
 302                                 Boolean cacheDbOkFlag = true;
 303                                 if( neverUseCache ){
 304                                         // They have forbidden us from using a cached db connection.
 305                                         cacheDbOkFlag = false;
 306                                 }
 307                                 doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
 308                                                 maxRecordsToFix, groomOutFileName, ver, singleCommits,
 309                                                 dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
 310                                                 finalShutdownFlag, cacheDbOkFlag,
 311                                                 skipEdgeCheckFlag, windowStartTime);
 312                         }
 313                 } catch (Exception ex) {
 314                         LOGGER.error("Exception while grooming data", ex);
 315                 }
 316
 317                 LOGGER.info(" Done! ");
 318                 System.exit(0);
 319
 320         }// End of main()
 321
 322         /**
 323          * Do the grooming.
 324          *
 325          * @param fileNameForFixing the file name for fixing
 326          * @param edgesOnlyFlag the edges only flag
 327          * @param dontFixOrphansFlag the dont fix orphans flag
 328          * @param maxRecordsToFix the max records to fix
 329          * @param groomOutFileName the groom out file name
 330          * @param version the version
 331          * @param singleCommits the single commits
 332          * @param dupeCheckOff the dupe check off
 333          * @param dupeFixOn the dupe fix on
 334          * @param ghost2CheckOff the ghost 2 check off
 335          * @param ghost2FixOn the ghost 2 fix on
 336          * @param finalShutdownFlag the final shutdown flag
 337          * @param cacheDbOkFlag the cacheDbOk flag
 338          * @return the int
 339          */
 340         private static int doTheGrooming(String fileNameForFixing,
 341                         Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
 342                         int maxRecordsToFix, String groomOutFileName, String version,
 343                         Boolean singleCommits,
 344                         Boolean dupeCheckOff, Boolean dupeFixOn,
 345                         Boolean ghost2CheckOff, Boolean ghost2FixOn,
 346                         Boolean finalShutdownFlag, Boolean cacheDbOkFlag,
 347                         Boolean skipEdgeCheckFlag, long windowStartTime) {
 348
 349                 LOGGER.debug(" Entering doTheGrooming \n");
 350
 351                 int cleanupCandidateCount = 0;
 352                 BufferedWriter bw = null;
 353                 TitanGraph graph = null;
 354                 TitanGraph graph2 = null;
 355                 int deleteCount = 0;
 356                 boolean executeFinalCommit = false;
 357                 Set<String> deleteCandidateList = new LinkedHashSet<>();
 358                 Set<String> processedVertices = new LinkedHashSet<>();
 359                 Graph g = null;
 360                 Graph g2 = null;
 361                 try {
 362                         AAIConfig.init();
 363                         String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
 364                                         + "logs" + AAIConstants.AAI_FILESEP + "data"
 365                                         + AAIConstants.AAI_FILESEP + "dataGrooming";
 366
 367                         // Make sure the target directory exists
 368                         new File(targetDir).mkdirs();
 369
 370                         if (!fileNameForFixing.equals("")) {
 371                                 deleteCandidateList = getDeleteList(targetDir,
 372                                                 fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
 373                                                 dupeFixOn);
 374                         }
 375
 376                         if (deleteCandidateList.size() > maxRecordsToFix) {
 377                                 LOGGER.warn(" >> WARNING >>  Delete candidate list size ("
 378                                                 + deleteCandidateList.size()
 379                                                 + ") is too big.  The maxFix we are using is: "
 380                                                 + maxRecordsToFix
 381                                                 + ".  No candidates will be deleted. ");
 382                                 // Clear out the list so it won't be processed below.
 383                                 deleteCandidateList = new LinkedHashSet<>();
 384                         }
 385
 386                         String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
 387                                         + groomOutFileName;
 388                         File groomOutFile = new File(fullOutputFileName);
 389                         try {
 390                                 groomOutFile.createNewFile();
 391                         } catch (IOException e) {
 392                                 String emsg = " Problem creating output file ["
 393                                                 + fullOutputFileName + "], exception=" + e.getMessage();
 394                                 throw new AAIException("AAI_6124", emsg);
 395                         }
 396
 397                         LOGGER.info(" Will write to " + fullOutputFileName );
 398                         bw = new BufferedWriter(new FileWriter(groomOutFile.getAbsoluteFile()));
 399                         ErrorLogHelper.loadProperties();
 400
 401                         LOGGER.info("    ---- NOTE --- about to open graph (takes a little while)--------\n");
 402
 403                         if( cacheDbOkFlag ){
 404                                 // Since we're just reading (not deleting/fixing anything), we can use
 405                                 // a cached connection to the DB
 406                                 graph = TitanFactory.open(AAIConstants.CACHED_DB_CONFIG);
 407                         }
 408                         else {
 409                                 graph = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
 410                         }
 411                         if (graph == null) {
 412                                 String emsg = "null graph object in DataGrooming\n";
 413                                 throw new AAIException("AAI_6101", emsg);
 414                         }
 415
 416                         LOGGER.debug(" Got the graph object. ");
 417
 418                         g = graph.newTransaction();
 419                         if (g == null) {
 420                                 String emsg = "null graphTransaction object in DataGrooming\n";
 421                                 throw new AAIException("AAI_6101", emsg);
 422                         }
 423                         GraphTraversalSource source1 = g.traversal();
 424
 425                         ArrayList<String> errArr = new ArrayList<>();
 426                         int totalNodeCount = 0;
 427                         HashMap<String, String> misMatchedHash = new HashMap<String, String>();
 428                         HashMap<String, Vertex> orphanNodeHash = new HashMap<String, Vertex>();
 429                         HashMap<String, Vertex> missingDepNodeHash = new HashMap<String, Vertex>();
 430                         HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
 431                         HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
 432                         HashMap<String, Vertex> ghostNodeHash = new HashMap<String, Vertex>();
 433                         ArrayList<String> dupeGroups = new ArrayList<>();
 434
 435                         Loader loader = LoaderFactory.createLoaderForVersion(ModelType.MOXY, AAIProperties.LATEST);
 436
 437                         Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
 438                         String ntList = "";
 439
 440                         LOGGER.info("  Starting DataGrooming Processing ");
 441
 442                         if (edgesOnlyFlag) {
 443                                 LOGGER.info(" NOTE >> Skipping Node processing as requested.  Will only process Edges. << ");
 444                         }
 445                         else {
 446                                 for (Entry<String, Introspector> entry : entrySet) {
 447                                         String nType = entry.getKey();
 448                                         int thisNtCount = 0;
 449                                         int thisNtDeleteCount = 0;
 450                                         LOGGER.debug(" >  Look at : [" + nType + "] ...");
 451                                         ntList = ntList + "," + nType;
 452
 453                                         // Get a collection of the names of the key properties for this nodeType to use later
 454                                         // Determine what the key fields are for this nodeType
 455                                         Collection <String> keyProps = entry.getValue().getKeys();
 456
 457                                         // Get the types of nodes that this nodetype depends on for uniqueness (if any)
 458                                         Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn();
 459
 460                                         // Loop through all the nodes of this Node type
 461                                         int lastShownForNt = 0;
 462                                         ArrayList <Vertex> tmpList = new ArrayList <> ();
 463                                         Iterator <Vertex> iterv =  source1.V().has("aai-node-type",nType);
 464                                         while (iterv.hasNext()) {
 465                                                 // We put the nodes into an ArrayList because the graph.query iterator can time out
 466                                                 tmpList.add(iterv.next());
 467                                         }
 468
 469                                         Iterator <Vertex> iter = tmpList.iterator();
 470                                         while (iter.hasNext()) {
 471                                                 try {
 472                                                         thisNtCount++;
 473                                                         if( thisNtCount == lastShownForNt + 250 ){
 474                                                                 lastShownForNt = thisNtCount;
 475                                                                 LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
 476                                                         }
 477                                                         Vertex thisVtx = iter.next();
 478                                                         if( windowStartTime > 0 ){
 479                                                                 // We only want nodes that are created after a passed-in timestamp
 480                                                                 Object objTimeStamp = thisVtx.property("aai-created-ts").orElse(null);
 481                                                                 if( objTimeStamp != null ){
 482                                                                         long thisNodeCreateTime = (long)objTimeStamp;
 483                                                                         if( thisNodeCreateTime < windowStartTime ){
 484                                                                                 // It is NOT in our window, so we can pass over it
 485                                                                                 continue;
 486                                                                         }
 487                                                                 }
 488                                                         }
 489
 490                                                         String thisVid = thisVtx.id().toString();
 491                                                         if (processedVertices.contains(thisVid)) {
 492                                                                 LOGGER.debug("skipping already processed vertex: " + thisVid);
 493                                                                 continue;
 494                                                         }
 495                                                         totalNodeCount++;
 496                                                         List <Vertex> secondGetList = new ArrayList <> ();
 497                                                         // -----------------------------------------------------------------------
 498                                                         // For each vertex of this nodeType, we want to:
 499                                                         //              a) make sure that it can be retrieved using it's AAI defined key
 500                                                         //      b) make sure that it is not a duplicate
 501                                                         // -----------------------------------------------------------------------
 502
 503                                                         // For this instance of this nodeType, get the key properties
 504                                                         HashMap<String, Object> propHashWithKeys = new HashMap<>();
 505                                                         Iterator<String> keyPropI = keyProps.iterator();
 506                                                         while (keyPropI.hasNext()) {
 507                                                                 String propName = keyPropI.next();
 508                                                                 String propVal = "";
 509                                                                 //delete an already deleted vertex
 510                                                                 Object obj = thisVtx.<Object>property(propName).orElse(null);
 511                                                                 if (obj != null) {
 512                                                                         propVal = obj.toString();
 513                                                                 }
 514                                                                 propHashWithKeys.put(propName, propVal);
 515                                                         }
 516                                                         try {
 517                                                                 // If this node is dependent on another for uniqueness, then do the query from that parent node
 518                                                                 // Note - all of our nodes that are dependent on others for uniqueness are
 519                                                                 //              "children" of that node.
 520                                                                 boolean depNodeOk = true;
 521                                                                 if( depNodeTypes.isEmpty() ){
 522                                                                         // This kind of node is not dependent on any other.
 523                                                                         // Make sure we can get it back using it's key properties and that we only get one.
 524                                                                         secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType,
 525                                                                                         propHashWithKeys, version );
 526                                                                 }
 527                                                                 else {
 528                                                                         // This kind of node is dependent on another for uniqueness.
 529                                                                         // Start at it's parent (the dependent vertex) and make sure we can get it
 530                                                                         // back using it's key properties and that we only get one.
 531                                                                         Iterator <Vertex> vertI2 = source1.V(thisVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
 532                                                                         Vertex parentVtx = null;
 533                                                                         int pCount = 0;
 534                                                                         while( vertI2 != null && vertI2.hasNext() ){
 535                                                                                 parentVtx = vertI2.next();
 536                                                                                 pCount++;
 537                                                                         }
 538                                                                         if( pCount <= 0 ){
 539
 540
 541                                                                         //List<Vertex> vertI2 = g.traversal().V(thisVtx).union(__.outE().has("isParent-REV",true).outV(),__.inE().has("isParent",true).inV()).toList();
 542                                                                         //if( vertI2.isEmpty()){
 543
 544                                                                                 // It's Missing it's dependent/parent node
 545                                                                                 depNodeOk = false;
 546                                                                                 boolean zeroEdges = false;
 547                                                                                 try {
 548                                                                                         Iterator<Edge> tmpEdgeIter = thisVtx.edges(Direction.BOTH);
 549                                                                                         int edgeCount = 0;
 550                                                                                         while( tmpEdgeIter.hasNext() ){
 551                                                                                                 edgeCount++;
 552                                                                                                 tmpEdgeIter.next();
 553                                                                                         }
 554                                                                                         if( edgeCount == 0 ){
 555                                                                                                 zeroEdges = true;
 556                                                                                         }
 557                                                                                 } catch (Exception ex) {
 558                                                                                         LOGGER.warn("WARNING from inside the for-each-vid-loop orphan-edges-check ", ex);
 559                                                                                 }
 560
 561                                                                                 if (deleteCandidateList.contains(thisVid)) {
 562                                                                                         boolean okFlag = true;
 563                                                                                         try {
 564                                                                                                 processedVertices.add(thisVtx.id().toString());
 565                                                                                                 thisVtx.remove();
 566                                                                                                 deleteCount++;
 567                                                                                                 thisNtDeleteCount++;
 568                                                                                         } catch (Exception e) {
 569                                                                                                 okFlag = false;
 570                                                                                                 LOGGER.error("ERROR trying to delete missing-dep-node VID = " + thisVid, e);
 571                                                                                         }
 572                                                                                         if (okFlag) {
 573                                                                                                 LOGGER.info(" DELETED missing-dep-node VID = " + thisVid);
 574                                                                                         }
 575                                                                                 } else {
 576                                                                                         // We count nodes missing their depNodes two ways - the first if it has
 577                                                                                         //    at least some edges, and the second if it has zero edges.  Either
 578                                                                                         //    way, they are effectively orphaned.
 579                                                                                         // NOTE - Only nodes that have dependent nodes are ever considered "orphaned".
 580                                                                                         if( zeroEdges ){
 581                                                                                                 missingDepNodeHash.put(thisVid, thisVtx);
 582                                                                                         }
 583                                                                                         else {
 584                                                                                                 orphanNodeHash.put(thisVid, thisVtx);
 585                                                                                         }
 586                                                                                 }
 587                                                                         }
 588                                                                         else if ( pCount > 1 ){
 589                                                                                 // Not sure how this could happen?  Should we do something here?
 590                                                                                 depNodeOk = false;
 591                                                                         }
 592                                                                         else {
 593                                                                                 // We found the parent - so use it to do the second-look.
 594                                                                                 // NOTE --- We're just going to do the same check from the other direction - because
 595                                                                                 //  there could be duplicates or the pointer going the other way could be broken
 596                                                                                 ArrayList <Vertex> tmpListSec = new ArrayList <> ();
 597
 598                                                                                 tmpListSec = getConnectedChildrenOfOneType( source1, parentVtx, nType ) ;
 599                                                                                 Iterator<Vertex> vIter = tmpListSec.iterator();
 600                                                                                 while (vIter.hasNext()) {
 601                                                                                         Vertex tmpV = vIter.next();
 602                                                                                         if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
 603                                                                                                 secondGetList.add(tmpV);
 604                                                                                         }
 605                                                                                 }
 606                                                                         }
 607                                                                 }
 608
 609                                                                 if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
 610                                                                         // We could not get the node back using it's own key info.
 611                                                                         // So, it's a PHANTOM
 612                                                                         if (deleteCandidateList.contains(thisVid)) {
 613                                                                                 boolean okFlag = true;
 614                                                                                 try {
 615                                                                                         thisVtx.remove();
 616                                                                                         deleteCount++;
 617                                                                                         thisNtDeleteCount++;
 618                                                                                 } catch (Exception e) {
 619                                                                                         okFlag = false;
 620                                                                                         LOGGER.error("ERROR trying to delete phantom VID = " + thisVid, e);
 621                                                                                 }
 622                                                                                 if (okFlag) {
 623                                                                                         LOGGER.info(" DELETED VID = " + thisVid);
 624                                                                                 }
 625                                                                         } else {
 626                                                                                 ghostNodeHash.put(thisVid, thisVtx);
 627                                                                         }
 628                                                                 }
 629                                                                 else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
 630                                                                         // Found some DUPLICATES - need to process them
 631                                                                         LOGGER.info(" - now check Dupes for this guy - ");
 632                                                                         List<String> tmpDupeGroups = checkAndProcessDupes(
 633                                                                                                 TRANSID, FROMAPPID, g, source1, version,
 634                                                                                                 nType, secondGetList, dupeFixOn,
 635                                                                                                 deleteCandidateList, singleCommits,     dupeGroups, loader);
 636                                                                         Iterator<String> dIter = tmpDupeGroups.iterator();
 637                                                                         while (dIter.hasNext()) {
 638                                                                                 // Add in any newly found dupes to our running list
 639                                                                                 String tmpGrp = dIter.next();
 640                                                                                 LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
 641                                                                                 dupeGroups.add(tmpGrp);
 642                                                                         }
 643                                                                 }
 644                                                         }
 645                                                         catch (AAIException e1) {
 646                                                                 LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
 647                                                                 errArr.add(e1.getErrorObject().toString());
 648                                                         }
 649                                                         catch (Exception e2) {
 650                                                                 LOGGER.warn(" For nodeType = " + nType
 651                                                                                 + " Caught exception", e2);
 652                                                                 errArr.add(e2.getMessage());
 653                                                         }
 654                                                 }// try block to enclose looping of a single vertex
 655                                                 catch (Exception exx) {
 656                                                         LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
 657                                                 }
 658
 659                                         } // while loop for each record of a nodeType
 660
 661                                         if ( (thisNtDeleteCount > 0) && singleCommits ) {
 662                                                 // NOTE - the singleCommits option is not used in normal processing
 663                                                 g.tx().commit();
 664                                                 g = AAIGraph.getInstance().getGraph().newTransaction();
 665
 666                                         }
 667                                         thisNtDeleteCount = 0;
 668                                         LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total overall. " );
 669
 670                                 }// While-loop for each node type
 671                         }// end of check to make sure we weren't only supposed to do edges
 672
 673
 674                   if( !skipEdgeCheckFlag ){
 675                         // --------------------------------------------------------------------------------------
 676                         // Now, we're going to look for one-armed-edges. Ie. an edge that
 677                         // should have
 678                         // been deleted (because a vertex on one side was deleted) but
 679                         // somehow was not deleted.
 680                         // So the one end of it points to a vertexId -- but that vertex is
 681                         // empty.
 682                         // --------------------------------------------------------------------------------------
 683
 684                         // To do some strange checking - we need a second graph object
 685                         LOGGER.debug("    ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
 686                         // Note - graph2 just reads - but we want it to use a fresh connection to
 687                         //      the database, so we are NOT using the CACHED DB CONFIG here.
 688                         graph2 = TitanFactory.open(AAIConstants.REALTIME_DB_CONFIG);
 689                         if (graph2 == null) {
 690                                 String emsg = "null graph2 object in DataGrooming\n";
 691                                 throw new AAIException("AAI_6101", emsg);
 692                         } else {
 693                                 LOGGER.debug("Got the graph2 object... \n");
 694                         }
 695                         g2 = graph2.newTransaction();
 696                         if (g2 == null) {
 697                                 String emsg = "null graphTransaction2 object in DataGrooming\n";
 698                                 throw new AAIException("AAI_6101", emsg);
 699                         }
 700
 701                         ArrayList<Vertex> vertList = new ArrayList<>();
 702                         Iterator<Vertex> vItor3 = g.traversal().V();
 703                         // Gotta hold these in a List - or else HBase times out as you cycle
 704                         // through these
 705                         while (vItor3.hasNext()) {
 706                                 Vertex v = vItor3.next();
 707                                 vertList.add(v);
 708                         }
 709                         int counter = 0;
 710                         int lastShown = 0;
 711                         Iterator<Vertex> vItor2 = vertList.iterator();
 712                         LOGGER.info(" Checking for bad edges  --- ");
 713
 714                         while (vItor2.hasNext()) {
 715                                 Vertex v = null;
 716                                 try {
 717                                         try {
 718                                                 v = vItor2.next();
 719                                         } catch (Exception vex) {
 720                                                 LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
 721                                                 continue;
 722                                         }
 723
 724                                         counter++;
 725                                         String thisVertId = "";
 726                                         try {
 727                                                 thisVertId = v.id().toString();
 728                                         } catch (Exception ev) {
 729                                                 LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list.  ");
 730                                                 continue;
 731                                         }
 732                                         if (ghostNodeHash.containsKey(thisVertId)) {
 733                                                 // This is a phantom node, so don't try to use it
 734                                                 LOGGER.info(" >> Skipping edge check for edges from vertexId = "
 735                                                                                 + thisVertId
 736                                                                                 + ", since that guy is a Phantom Node");
 737                                                 continue;
 738                                         }
 739
 740                                         if( windowStartTime > 0 ){
 741                                                 // We only want to look at nodes that are created after a passed-in timestamp
 742                                                 Object objTimeStamp = v.property("aai-created-ts").orElse(null);
 743                                                 if( objTimeStamp != null ){
 744                                                         long thisNodeCreateTime = (long)objTimeStamp;
 745                                                         if( thisNodeCreateTime < windowStartTime ){
 746                                                                 // It is NOT in our window, so we can pass over it
 747                                                                 continue;
 748                                                         }
 749                                                 }
 750                                         }
 751
 752                                         if (counter == lastShown + 250) {
 753                                                 lastShown = counter;
 754                                                 LOGGER.info("... Checking edges for vertex # "
 755                                                                 + counter);
 756                                         }
 757                                         Iterator<Edge> eItor = v.edges(Direction.BOTH);
 758                                         while (eItor.hasNext()) {
 759                                                 Edge e = null;
 760                                                 Vertex vIn = null;
 761                                                 Vertex vOut = null;
 762                                                 try {
 763                                                         e = eItor.next();
 764                                                 } catch (Exception iex) {
 765                                                         LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
 766                                                         continue;
 767                                                 }
 768
 769                                                 try {
 770                                                         vIn = e.inVertex();
 771                                                 } catch (Exception err) {
 772                                                         LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
 773                                                 }
 774                                                 String vNtI = "";
 775                                                 String vIdI = "";
 776                                                 Vertex ghost2 = null;
 777
 778                                                 Boolean keysMissing = true;
 779                                                 Boolean cantGetUsingVid = false;
 780                                                 if (vIn != null) {
 781                                                         try {
 782                                                                 Object ob = vIn.<Object>property("aai-node-type").orElse(null);
 783                                                                 if (ob != null) {
 784                                                                         vNtI = ob.toString();
 785                                                                         keysMissing = anyKeyFieldsMissing(vNtI, vIn, loader);
 786                                                                 }
 787                                                                 ob = vIn.id();
 788                                                                 long vIdLong = 0L;
 789                                                                 if (ob != null) {
 790                                                                         vIdI = ob.toString();
 791                                                                         vIdLong = Long.parseLong(vIdI);
 792                                                                 }
 793
 794                                                                 if( ! ghost2CheckOff ){
 795                                                                         Vertex connectedVert = g2.traversal().V(vIdLong).next();
 796                                                                         if( connectedVert == null ) {
 797                                                                                 LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
 798                                                                                 cantGetUsingVid = true;
 799
 800                                                                                 // If we can NOT get this ghost with the SECOND graph-object,
 801                                                                                 // it is still a ghost since even though we can get data about it using the FIRST graph
 802                                                                                 // object.
 803                                                                                 try {
 804                                                                                          ghost2 = g.traversal().V(vIdLong).next();
 805                                                                                 }
 806                                                                                 catch( Exception ex){
 807                                                                                         LOGGER.warn( "GHOST2 --  Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
 808                                                                                 }
 809                                                                                 if( ghost2 != null ){
 810                                                                                         ghostNodeHash.put(vIdI, ghost2);
 811                                                                                 }
 812                                                                         }
 813                                                                 }// end of the ghost2 checking
 814                                                         }
 815                                                         catch (Exception err) {
 816                                                                 LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
 817                                                         }
 818                                                 }
 819                                                 if (keysMissing || vIn == null || vNtI.equals("")
 820                                                                 || cantGetUsingVid) {
 821                                                         // this is a bad edge because it points to a vertex
 822                                                         // that isn't there anymore or is corrupted
 823                                                         String thisEid = e.id().toString();
 824                                                         if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
 825                                                                 boolean okFlag = true;
 826                                                                 if (!vIdI.equals("")) {
 827                                                                         // try to get rid of the corrupted vertex
 828                                                                         try {
 829                                                                                 if( (ghost2 != null) && ghost2FixOn ){
 830                                                                                         ghost2.remove();
 831                                                                                 }
 832                                                                                 else {
 833                                                                                         vIn.remove();
 834                                                                                 }
 835                                                                                 if (singleCommits) {
 836                                                                                         // NOTE - the singleCommits option is not used in normal processing
 837                                                                                         g.tx().commit();
 838                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 839                                                                                 }
 840                                                                                 deleteCount++;
 841                                                                         } catch (Exception e1) {
 842                                                                                 okFlag = false;
 843                                                                                 LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
 844                                                                                                 + vIdI, e1);
 845                                                                         }
 846                                                                         if (okFlag) {
 847                                                                                 LOGGER.info(" DELETED vertex from bad edge = "
 848                                                                                                                 + vIdI);
 849                                                                         }
 850                                                                 } else {
 851                                                                         // remove the edge if we couldn't get the
 852                                                                         // vertex
 853                                                                         try {
 854                                                                                 e.remove();
 855                                                                                 if (singleCommits) {
 856                                                                                         // NOTE - the singleCommits option is not used in normal processing
 857                                                                                         g.tx().commit();
 858                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 859                                                                                 }
 860                                                                                 deleteCount++;
 861                                                                         } catch (Exception ex) {
 862                                                                                 // NOTE - often, the exception is just
 863                                                                                 // that this edge has already been
 864                                                                                 // removed
 865                                                                                 okFlag = false;
 866                                                                                 LOGGER.warn("WARNING when trying to delete edge = "
 867                                                                                                 + thisEid);
 868                                                                         }
 869                                                                         if (okFlag) {
 870                                                                                 LOGGER.info(" DELETED edge = " + thisEid);
 871                                                                         }
 872                                                                 }
 873                                                         } else {
 874                                                                 oneArmedEdgeHash.put(thisEid, e);
 875                                                                 if ((vIn != null) && (vIn.id() != null)) {
 876                                                                         emptyVertexHash.put(thisEid, vIn.id()
 877                                                                                         .toString());
 878                                                                 }
 879                                                         }
 880                                                 }
 881
 882                                                 try {
 883                                                         vOut = e.outVertex();
 884                                                 } catch (Exception err) {
 885                                                         LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
 886                                                 }
 887                                                 String vNtO = "";
 888                                                 String vIdO = "";
 889                                                 ghost2 = null;
 890                                                 keysMissing = true;
 891                                                 cantGetUsingVid = false;
 892                                                 if (vOut != null) {
 893                                                         try {
 894                                                                 Object ob = vOut.<Object>property("aai-node-type").orElse(null);
 895                                                                 if (ob != null) {
 896                                                                         vNtO = ob.toString();
 897                                                                         keysMissing = anyKeyFieldsMissing(vNtO,
 898                                                                                         vOut, loader);
 899                                                                 }
 900                                                                 ob = vOut.id();
 901                                                                 long vIdLong = 0L;
 902                                                                 if (ob != null) {
 903                                                                         vIdO = ob.toString();
 904                                                                         vIdLong = Long.parseLong(vIdO);
 905                                                                 }
 906
 907                                                                 if( ! ghost2CheckOff ){
 908                                                                         Vertex connectedVert = g2.traversal().V(vIdLong).next();
 909                                                                         if( connectedVert == null ) {
 910                                                                                 cantGetUsingVid = true;
 911                                                                                 LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
 912                                                                                 // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
 913                                                                                 try {
 914                                                                                          ghost2 = g.traversal().V(vIdLong).next();
 915                                                                                 }
 916                                                                                 catch( Exception ex){
 917                                                                                         LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
 918                                                                                 }
 919                                                                                 if( ghost2 != null ){
 920                                                                                         ghostNodeHash.put(vIdO, ghost2);
 921                                                                                 }
 922                                                                         }
 923                                                                 }
 924                                                         } catch (Exception err) {
 925                                                                 LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
 926                                                         }
 927                                                 }
 928                                                 if (keysMissing || vOut == null || vNtO.equals("")
 929                                                                 || cantGetUsingVid) {
 930                                                         // this is a bad edge because it points to a vertex
 931                                                         // that isn't there anymore
 932                                                         String thisEid = e.id().toString();
 933                                                         if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
 934                                                                 boolean okFlag = true;
 935                                                                 if (!vIdO.equals("")) {
 936                                                                         // try to get rid of the corrupted vertex
 937                                                                         try {
 938                                                                                 if( (ghost2 != null) && ghost2FixOn ){
 939                                                                                         ghost2.remove();
 940                                                                                 }
 941                                                                                 else if (vOut != null) {
 942                                                                                         vOut.remove();
 943                                                                                 }
 944                                                                                 if (singleCommits) {
 945                                                                                         // NOTE - the singleCommits option is not used in normal processing
 946                                                                                         g.tx().commit();
 947                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 948                                                                                 }
 949                                                                                 deleteCount++;
 950                                                                         } catch (Exception e1) {
 951                                                                                 okFlag = false;
 952                                                                                 LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
 953                                                                                                 + vIdO, e1);
 954                                                                         }
 955                                                                         if (okFlag) {
 956                                                                                 LOGGER.info(" DELETED vertex from bad edge = "
 957                                                                                                                 + vIdO);
 958                                                                         }
 959                                                                 } else {
 960                                                                         // remove the edge if we couldn't get the
 961                                                                         // vertex
 962                                                                         try {
 963                                                                                 e.remove();
 964                                                                                 if (singleCommits) {
 965                                                                                         // NOTE - the singleCommits option is not used in normal processing
 966                                                                                         g.tx().commit();
 967                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
 968                                                                                 }
 969                                                                                 deleteCount++;
 970                                                                         } catch (Exception ex) {
 971                                                                                 // NOTE - often, the exception is just
 972                                                                                 // that this edge has already been
 973                                                                                 // removed
 974                                                                                 okFlag = false;
 975                                                                                 LOGGER.warn("WARNING when trying to delete edge = "
 976                                                                                                 + thisEid, ex);
 977                                                                         }
 978                                                                         if (okFlag) {
 979                                                                                 LOGGER.info(" DELETED edge = " + thisEid);
 980                                                                         }
 981                                                                 }
 982                                                         } else {
 983                                                                 oneArmedEdgeHash.put(thisEid, e);
 984                                                                 if ((vOut != null) && (vOut.id() != null)) {
 985                                                                         emptyVertexHash.put(thisEid, vOut.id()
 986                                                                                         .toString());
 987                                                                 }
 988                                                         }
 989                                                 }
 990                                         }// End of while-edges-loop
 991                                 } catch (Exception exx) {
 992                                         LOGGER.warn("WARNING from in the while-verts-loop ", exx);
 993                                 }
 994                         }// End of while-vertices-loop (the edge-checking)
 995                   }     // end of -- if we're not skipping the edge-checking
 996
 997
 998                         deleteCount = deleteCount + dupeGrpsDeleted;
 999                         if (!singleCommits && deleteCount > 0) {
1000                                 try {
1001                                         LOGGER.info("About to do the commit for "
1002                                                         + deleteCount + " removes. ");
1003                                         executeFinalCommit = true;
1004                                         LOGGER.info("Commit was successful ");
1005                                 } catch (Exception excom) {
1006                                         LOGGER.error(" >>>> ERROR <<<<   Could not commit changes. ", excom);
1007                                         deleteCount = 0;
1008                                 }
1009                         }
1010
1011                         int ghostNodeCount = ghostNodeHash.size();
1012                         int orphanNodeCount = orphanNodeHash.size();
1013                         int missingDepNodeCount = missingDepNodeHash.size();
1014                         int oneArmedEdgeCount = oneArmedEdgeHash.size();
1015                         int dupeCount = dupeGroups.size();
1016
1017                         deleteCount = deleteCount + dupeGrpsDeleted;
1018
1019                         bw.write("\n\n ============ Summary ==============\n");
1020                         bw.write("Ran these nodeTypes: " + ntList + "\n\n");
1021                         bw.write("There were this many delete candidates from previous run =  "
1022                                         + deleteCandidateList.size() + "\n");
1023                         if (dontFixOrphansFlag) {
1024                                 bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
1025                         }
1026                         bw.write("Deleted this many delete candidates =  " + deleteCount
1027                                         + "\n");
1028                         bw.write("Total number of nodes looked at =  " + totalNodeCount
1029                                         + "\n");
1030                         bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
1031                         bw.write("Orphan Nodes identified =  " + orphanNodeCount + "\n");
1032                         bw.write("Bad Edges identified =  " + oneArmedEdgeCount + "\n");
1033                         bw.write("Missing Dependent Edge (but not orphaned) node count = "
1034                                         + missingDepNodeCount + "\n");
1035                         bw.write("Duplicate Groups count =  " + dupeCount + "\n");
1036                         bw.write("MisMatching Label/aai-node-type count =  "
1037                                         + misMatchedHash.size() + "\n");
1038
1039                         bw.write("\n ------------- Delete Candidates ---------\n");
1040                         for (Map.Entry<String, Vertex> entry : ghostNodeHash
1041                                         .entrySet()) {
1042                                 String vid = entry.getKey();
1043                                 bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
1044                                 cleanupCandidateCount++;
1045                         }
1046                         for (Map.Entry<String, Vertex> entry : orphanNodeHash
1047                                         .entrySet()) {
1048                                 String vid = entry.getKey();
1049                                 bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
1050                                 if (!dontFixOrphansFlag) {
1051                                         cleanupCandidateCount++;
1052                                 }
1053                         }
1054                         for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1055                                 String eid = entry.getKey();
1056                                 bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
1057                                 cleanupCandidateCount++;
1058                         }
1059                         for (Map.Entry<String, Vertex> entry : missingDepNodeHash
1060                                         .entrySet()) {
1061                                 String vid = entry.getKey();
1062                                 bw.write("DeleteCandidate: (maybe) missingDepNode Vid = ["
1063                                                 + vid + "]\n");
1064                                 cleanupCandidateCount++;
1065                         }
1066                         bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
1067
1068                         bw.write("\n ------------- GHOST NODES - detail ");
1069                         for (Map.Entry<String, Vertex> entry : ghostNodeHash
1070                                         .entrySet()) {
1071                                 try {
1072                                         String vid = entry.getKey();
1073                                         bw.write("\n ==> Phantom Vid = " + vid + "\n");
1074                                         ArrayList<String> retArr = showPropertiesForNode(
1075                                                         TRANSID, FROMAPPID, entry.getValue());
1076                                         for (String info : retArr) {
1077                                                 bw.write(info + "\n");
1078                                         }
1079
1080                                         retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1081                                                         entry.getValue());
1082                                         for (String info : retArr) {
1083                                                 bw.write(info + "\n");
1084                                         }
1085                                 } catch (Exception dex) {
1086                                         LOGGER.error("error trying to print detail info for a ghost-node:  ", dex);
1087                                 }
1088                         }
1089
1090                         bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
1091                         for (Map.Entry<String, Vertex> entry : orphanNodeHash
1092                                         .entrySet()) {
1093                                 try {
1094                                         String vid = entry.getKey();
1095                                         bw.write("\n> Orphan Node Vid = " + vid + "\n");
1096                                         ArrayList<String> retArr = showPropertiesForNode(
1097                                                         TRANSID, FROMAPPID, entry.getValue());
1098                                         for (String info : retArr) {
1099                                                 bw.write(info + "\n");
1100                                         }
1101
1102                                         retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1103                                                         entry.getValue());
1104                                         for (String info : retArr) {
1105                                                 bw.write(info + "\n");
1106                                         }
1107                                 } catch (Exception dex) {
1108                                         LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge", dex);
1109                                 }
1110                         }
1111
1112                         bw.write("\n ------------- Missing Dependent Edge (but not orphan) NODES: ");
1113                         for (Map.Entry<String, Vertex> entry : missingDepNodeHash
1114                                         .entrySet()) {
1115                                 try {
1116                                         String vid = entry.getKey();
1117                                         bw.write("\n>  Missing edge to Dependent Node (but has edges) Vid = "
1118                                                         + vid + "\n");
1119                                         ArrayList<String> retArr = showPropertiesForNode(
1120                                                         TRANSID, FROMAPPID, entry.getValue());
1121                                         for (String info : retArr) {
1122                                                 bw.write(info + "\n");
1123                                         }
1124
1125                                         retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
1126                                                         entry.getValue());
1127                                         for (String info : retArr) {
1128                                                 bw.write(info + "\n");
1129                                         }
1130                                 } catch (Exception dex) {
1131                                         LOGGER.error("error trying to print detail info for a node missing its dependent edge but not an orphan", dex);
1132                                 }
1133                         }
1134
1135                         bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
1136                         for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
1137                                 try {
1138                                         String eid = entry.getKey();
1139                                         Edge thisE = entry.getValue();
1140                                         String badVid = emptyVertexHash.get(eid);
1141                                         bw.write("\n>  Edge pointing to bad vertex (Vid = "
1142                                                         + badVid + ") EdgeId = " + eid + "\n");
1143                                         bw.write("Label: [" + thisE.label() + "]\n");
1144                                         Iterator<Property<Object>> pI = thisE.properties();
1145                                         while (pI.hasNext()) {
1146                                                 Property<Object> propKey = pI.next();
1147                                                 bw.write("Prop: [" + propKey + "], val = ["
1148                                                                 + propKey.value() + "]\n");
1149                                         }
1150                                 } catch (Exception pex) {
1151                                         LOGGER.error("error trying to print empty/bad vertex data: ", pex);
1152                                 }
1153                         }
1154
1155                         bw.write("\n ------------- Duplicates: ");
1156                         Iterator<String> dupeIter = dupeGroups.iterator();
1157                         int dupeSetCounter = 0;
1158                         while (dupeIter.hasNext()) {
1159                                 dupeSetCounter++;
1160                                 String dset = (String) dupeIter.next();
1161
1162                                 bw.write("\n --- Duplicate Group # " + dupeSetCounter
1163                                                 + " Detail -----------\n");
1164                                 try {
1165                                         // We expect each line to have at least two vid's, followed
1166                                         // by the preferred one to KEEP
1167                                         String[] dupeArr = dset.split("\\|");
1168                                         ArrayList<String> idArr = new ArrayList<>();
1169                                         int lastIndex = dupeArr.length - 1;
1170                                         for (int i = 0; i <= lastIndex; i++) {
1171                                                 if (i < lastIndex) {
1172                                                         // This is not the last entry, it is one of the
1173                                                         // dupes, so we want to show all its info
1174                                                         bw.write("    >> Duplicate Group # "
1175                                                                         + dupeSetCounter + "  Node # " + i
1176                                                                         + " ----\n");
1177                                                         String vidString = dupeArr[i];
1178                                                         idArr.add(vidString);
1179                                                         long longVertId = Long.parseLong(vidString);
1180                                                         Iterator<Vertex> vtxIterator = g.vertices(longVertId);
1181                                                         Vertex vtx = null;
1182                                                         if (vtxIterator.hasNext()) {
1183                                                                 vtx = vtxIterator.next();
1184                                                         }
1185                                                         ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
1186                                                         for (String info : retArr) {
1187                                                                 bw.write(info + "\n");
1188                                                         }
1189
1190                                                         retArr = showAllEdgesForNode(TRANSID,
1191                                                                         FROMAPPID, vtx);
1192                                                         for (String info : retArr) {
1193                                                                 bw.write(info + "\n");
1194                                                         }
1195                                                 } else {
1196                                                         // This is the last entry which should tell us if we
1197                                                         // have a preferred keeper
1198                                                         String prefString = dupeArr[i];
1199                                                         if (prefString.equals("KeepVid=UNDETERMINED")) {
1200                                                                 bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
1201                                                                 bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
1202                                                         } else {
1203                                                                 // If we know which to keep, then the prefString
1204                                                                 // should look like, "KeepVid=12345"
1205                                                                 String[] prefArr = prefString.split("=");
1206                                                                 if (prefArr.length != 2
1207                                                                                 || (!prefArr[0].equals("KeepVid"))) {
1208                                                                         throw new Exception("Bad format. Expecting KeepVid=999999");
1209                                                                 } else {
1210                                                                         String keepVidStr = prefArr[1];
1211                                                                         if (idArr.contains(keepVidStr)) {
1212                                                                                 bw.write("\n The vertex we want to KEEP has vertexId = "
1213                                                                                                 + keepVidStr);
1214                                                                                 bw.write("\n The others become delete candidates: \n");
1215                                                                                 idArr.remove(keepVidStr);
1216                                                                                 for (int x = 0; x < idArr.size(); x++) {
1217                                                                                         cleanupCandidateCount++;
1218                                                                                         bw.write("DeleteCandidate: Duplicate Vid = ["
1219                                                                                                         + idArr.get(x) + "]\n");
1220                                                                                 }
1221                                                                         } else {
1222                                                                                 throw new Exception("ERROR - Vertex Id to keep not found in list of dupes.  dset = ["
1223                                                                                                 + dset + "]");
1224                                                                         }
1225                                                                 }
1226                                                         }// else we know which one to keep
1227                                                 }// else last entry
1228                                         }// for each vertex in a group
1229                                 } catch (Exception dex) {
1230                                         LOGGER.error("error trying to print duplicate vertex data", dex);
1231                                 }
1232
1233                         }// while - work on each group of dupes
1234
1235                         bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
1236                         for (Map.Entry<String, String> entry : misMatchedHash.entrySet()) {
1237                                 String msg = entry.getValue();
1238                                 bw.write("MixedMsg = " + msg + "\n");
1239                         }
1240
1241                         bw.write("\n ------------- Got these errors while processing: \n");
1242                         Iterator<String> errIter = errArr.iterator();
1243                         while (errIter.hasNext()) {
1244                                 String line = (String) errIter.next();
1245                                 bw.write(line + "\n");
1246                         }
1247
1248                         bw.close();
1249
1250                         LOGGER.info("\n ------------- Done doing all the checks ------------ ");
1251                         LOGGER.info("Output will be written to " + fullOutputFileName);
1252
1253                         if (cleanupCandidateCount > 0) {
1254                                 // Technically, this is not an error -- but we're throwing this
1255                                 // error so that hopefully a
1256                                 // monitoring system will pick it up and do something with it.
1257                                 throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
1258                                                 + "] and investigate delete candidates. ");
1259                         }
1260                 } catch (AAIException e) {
1261                         LOGGER.error("Caught AAIException while grooming data", e);
1262                         ErrorLogHelper.logException(e);
1263                 } catch (Exception ex) {
1264                         LOGGER.error("Caught exception while grooming data", ex);
1265                         ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
1266                 } finally {
1267
1268                         if (bw != null) {
1269                                 try {
1270                                         bw.close();
1271                                 } catch (IOException iox) {
1272                                         LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
1273                                 }
1274                         }
1275
1276                         if (g != null && g.tx().isOpen()) {
1277                                 // Any changes that worked correctly should have already done
1278                                 // their commits.
1279                                 try {
1280                                         if (executeFinalCommit) {
1281                                                 g.tx().commit();
1282                                         }
1283                                         g.tx().rollback();
1284                                 } catch (Exception ex) {
1285                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1286                                         LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
1287                                 }
1288                         }
1289
1290                         if (g2 != null && g2.tx().isOpen()) {
1291                                 // Any changes that worked correctly should have already done
1292                                 // their commits.
1293                                 try {
1294                                         g2.tx().rollback();
1295                                 } catch (Exception ex) {
1296                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed
1297                                         LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
1298                                 }
1299                         }
1300
1301                         if( finalShutdownFlag ){
1302                                 try {
1303                                         if( graph != null && graph.isOpen() ){
1304                                                 graph.tx().close();
1305                                                 graph.close();
1306                                         }
1307                                 } catch (Exception ex) {
1308                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1309                                         LOGGER.warn("WARNING from final graph.shutdown()", ex);
1310                                 }
1311
1312                                 try {
1313                                         if( graph2 != null && graph2.isOpen() ){
1314                                                 graph2.tx().close();
1315                                                 graph2.close();
1316                                         }
1317                                 } catch (Exception ex) {
1318                                         // Don't throw anything because Titan sometimes is just saying that the graph is already closed{
1319                                         LOGGER.warn("WARNING from final graph2.shutdown()", ex);
1320                                 }
1321                         }
1322
1323                 }
1324
1325                 return cleanupCandidateCount;
1326
1327         }// end of doTheGrooming()
1328
1329
1330         /**
1331          * Vertex has these keys.
1332          *
1333          * @param tmpV the tmp V
1334          * @param propHashWithKeys the prop hash with keys
1335          * @return the boolean
1336          */
1337         private static Boolean vertexHasTheseKeys( Vertex tmpV, HashMap <String, Object> propHashWithKeys) {
1338                 Iterator <?> it = propHashWithKeys.entrySet().iterator();
1339                 while( it.hasNext() ){
1340                         String propName = "";
1341                         String propVal = "";
1342                         Map.Entry <?,?>propEntry = (Map.Entry<?,?>)it.next();
1343                         Object propNameObj = propEntry.getKey();
1344                         if( propNameObj != null ){
1345                                 propName = propNameObj.toString();
1346                         }
1347                         Object propValObj = propEntry.getValue();
1348                         if( propValObj != null ){
1349                                 propVal = propValObj.toString();
1350                         }
1351                         Object checkValObj = tmpV.<Object>property(propName).orElse(null);
1352                         if( checkValObj == null ) {
1353                                 return false;
1354                         }
1355                         else if( !propVal.equals(checkValObj.toString()) ){
1356                                 return false;
1357                         }
1358                 }
1359                 return true;
1360         }
1361
1362
1363         /**
1364          * Any key fields missing.
1365          *
1366          * @param nType the n type
1367          * @param v the v
1368          * @return the boolean
1369          */
1370         private static Boolean anyKeyFieldsMissing(String nType, Vertex v, Loader loader) {
1371
1372                 try {
1373                         Introspector obj = null;
1374                         try {
1375                                 obj = loader.introspectorFromName(nType);
1376                         } catch (AAIUnknownObjectException e) {
1377                                 // They gave us a non-empty nodeType but our NodeKeyProps does
1378                                 //   not have data for it.  Since we do not know what the
1379                                 //   key params are for this type of node, we will just
1380                                 //   return "false".
1381                                 String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
1382                                                 + "].  We cannot determine required keys for this nType. ";
1383                                 // NOTE - this will be caught below and a "false" returned
1384                                 throw new AAIException("AAI_6121", emsg);
1385                         }
1386
1387                         // Determine what the key fields are for this nodeType
1388                         Collection <String> keyPropNamesColl = obj.getKeys();
1389                         Iterator<String> keyPropI = keyPropNamesColl.iterator();
1390                         while (keyPropI.hasNext()) {
1391                                 String propName = keyPropI.next();
1392                                 Object ob = v.<Object>property(propName).orElse(null);
1393                                 if (ob == null || ob.toString().equals("")) {
1394                                         // It is missing a key property
1395                                         return true;
1396                                 }
1397                         }
1398                 } catch (AAIException e) {
1399                         // Something was wrong -- but since we weren't able to check
1400                         // the keys, we will not declare that it is missing keys.
1401                         return false;
1402                 }
1403                 return false;
1404         }
1405
1406
1407         /**
1408          * Gets the delete list.
1409          *
1410          * @param targetDir the target dir
1411          * @param fileName the file name
1412          * @param edgesOnlyFlag the edges only flag
1413          * @param dontFixOrphans the dont fix orphans
1414          * @param dupeFixOn the dupe fix on
1415          * @return the delete list
1416          * @throws AAIException the AAI exception
1417          */
1418         private static Set<String> getDeleteList(String targetDir,
1419                         String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
1420                         Boolean dupeFixOn) throws AAIException {
1421
1422                 // Look in the file for lines formated like we expect - pull out any
1423                 // Vertex Id's to delete on this run
1424                 Set<String> delList = new LinkedHashSet<>();
1425                 String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
1426
1427                 try(BufferedReader br = new BufferedReader(new FileReader(fullFileName))) {
1428                         String line = br.readLine();
1429                         while (line != null) {
1430                                 if (!"".equals(line) && line.startsWith("DeleteCandidate")) {
1431                                         if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
1432                                                 // We're not going to process edge guys
1433                                         } else if (dontFixOrphans && line.contains("Orphan")) {
1434                                                 // We're not going to process orphans
1435                                         } else if (!dupeFixOn && line.contains("Duplicate")) {
1436                                                 // We're not going to process Duplicates
1437                                         } else {
1438                                                 int begIndex = line.indexOf("id = ");
1439                                                 int endIndex = line.indexOf("]");
1440                                                 String vidVal = line.substring(begIndex + 6, endIndex);
1441                                                 delList.add(vidVal);
1442                                         }
1443                                 }
1444                                 line = br.readLine();
1445                         }
1446                         br.close();
1447                 } catch (IOException e) {
1448                         throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
1449                                         + "], exception= " + e.getMessage());
1450                 }
1451
1452                 return delList;
1453
1454         }// end of getDeleteList
1455
1456         /**
1457          * Gets the preferred dupe.
1458          *
1459          * @param transId the trans id
1460          * @param fromAppId the from app id
1461          * @param g the g
1462          * @param dupeVertexList the dupe vertex list
1463          * @param ver the ver
1464          * @return Vertex
1465          * @throws AAIException the AAI exception
1466          */
1467         public static Vertex getPreferredDupe(String transId,
1468                         String fromAppId, GraphTraversalSource g,
1469                         ArrayList<Vertex> dupeVertexList, String ver, Loader loader)
1470                         throws AAIException {
1471
1472                 // This method assumes that it is being passed a List of vertex objects
1473                 // which
1474                 // violate our uniqueness constraints.
1475
1476                 Vertex nullVtx = null;
1477
1478                 if (dupeVertexList == null) {
1479                         return nullVtx;
1480                 }
1481                 int listSize = dupeVertexList.size();
1482                 if (listSize == 0) {
1483                         return nullVtx;
1484                 }
1485                 if (listSize == 1) {
1486                         return (dupeVertexList.get(0));
1487                 }
1488
1489                 Vertex vtxPreferred = null;
1490                 Vertex currentFaveVtx = dupeVertexList.get(0);
1491                 for (int i = 1; i < listSize; i++) {
1492                         Vertex vtxB = dupeVertexList.get(i);
1493                         vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
1494                                         currentFaveVtx, vtxB, ver, loader);
1495                         if (vtxPreferred == null) {
1496                                 // We couldn't choose one
1497                                 return nullVtx;
1498                         } else {
1499                                 currentFaveVtx = vtxPreferred;
1500                         }
1501                 }
1502
1503                 return (currentFaveVtx);
1504
1505         } // end of getPreferredDupe()
1506
1507         /**
1508          * Pick one of two dupes.
1509          *
1510          * @param transId the trans id
1511          * @param fromAppId the from app id
1512          * @param g the g
1513          * @param vtxA the vtx A
1514          * @param vtxB the vtx B
1515          * @param ver the ver
1516          * @return Vertex
1517          * @throws AAIException the AAI exception
1518          */
1519         public static Vertex pickOneOfTwoDupes(String transId,
1520                         String fromAppId, GraphTraversalSource g, Vertex vtxA,
1521                         Vertex vtxB, String ver, Loader loader) throws AAIException {
1522
1523                 Vertex nullVtx = null;
1524                 Vertex preferredVtx = null;
1525
1526                 Long vidA = new Long(vtxA.id().toString());
1527                 Long vidB = new Long(vtxB.id().toString());
1528
1529                 String vtxANodeType = "";
1530                 String vtxBNodeType = "";
1531                 Object objType = vtxA.<Object>property("aai-node-type").orElse(null);
1532                 if (objType != null) {
1533                         vtxANodeType = objType.toString();
1534                 }
1535                 objType = vtxB.<Object>property("aai-node-type").orElse(null);
1536                 if (objType != null) {
1537                         vtxBNodeType = objType.toString();
1538                 }
1539
1540                 if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
1541                         // Either they're not really dupes or there's some bad data - so
1542                         // don't pick one
1543                         return nullVtx;
1544                 }
1545
1546                 // Check that node A and B both have the same key values (or else they
1547                 // are not dupes)
1548                 // (We'll check dep-node later)
1549                 // Determine what the key fields are for this nodeType
1550                 Collection <String> keyProps = new ArrayList <>();
1551                 try {
1552                         keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
1553                 } catch (AAIUnknownObjectException e) {
1554                         LOGGER.warn("Required property not found", e);
1555                         throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
1556                 }
1557
1558                 Iterator<String> keyPropI = keyProps.iterator();
1559                 while (keyPropI.hasNext()) {
1560                         String propName = keyPropI.next();
1561                         String vtxAKeyPropVal = "";
1562                         objType = vtxA.<Object>property(propName).orElse(null);
1563                         if (objType != null) {
1564                                 vtxAKeyPropVal = objType.toString();
1565                         }
1566                         String vtxBKeyPropVal = "";
1567                         objType = vtxB.<Object>property(propName).orElse(null);
1568                         if (objType != null) {
1569                                 vtxBKeyPropVal = objType.toString();
1570                         }
1571
1572                         if (vtxAKeyPropVal.equals("")
1573                                         || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
1574                                 // Either they're not really dupes or they are missing some key
1575                                 // data - so don't pick one
1576                                 return nullVtx;
1577                         }
1578                 }
1579
1580                 // Collect the vid's and aai-node-types of the vertices that each vertex
1581                 // (A and B) is connected to.
1582                 ArrayList<String> vtxIdsConn2A = new ArrayList<>();
1583                 ArrayList<String> vtxIdsConn2B = new ArrayList<>();
1584                 HashMap<String, String> nodeTypesConn2A = new HashMap<>();
1585                 HashMap<String, String> nodeTypesConn2B = new HashMap<>();
1586
1587                 ArrayList<Vertex> vertListA = getConnectedNodes( g, vtxA );
1588                 if (vertListA != null) {
1589                         Iterator<Vertex> iter = vertListA.iterator();
1590                         while (iter.hasNext()) {
1591                                 Vertex tvCon = iter.next();
1592                                 String conVid = tvCon.id().toString();
1593                                 String nt = "";
1594                                 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1595                                 if (objType != null) {
1596                                         nt = objType.toString();
1597                                 }
1598                                 nodeTypesConn2A.put(nt, conVid);
1599                                 vtxIdsConn2A.add(conVid);
1600                         }
1601                 }
1602
1603                 ArrayList<Vertex> vertListB = getConnectedNodes( g, vtxB );
1604                 if (vertListB != null) {
1605                         Iterator<Vertex> iter = vertListB.iterator();
1606                         while (iter.hasNext()) {
1607                                 Vertex tvCon = iter.next();
1608                                 String conVid = tvCon.id().toString();
1609                                 String nt = "";
1610                                 objType = tvCon.<Object>property("aai-node-type").orElse(null);
1611                                 if (objType != null) {
1612                                         nt = objType.toString();
1613                                 }
1614                                 nodeTypesConn2B.put(nt, conVid);
1615                                 vtxIdsConn2B.add(conVid);
1616                         }
1617                 }
1618
1619                 // 1 - If this kind of node needs a dependent node for uniqueness, then
1620                 // verify that they both nodes
1621                 // point to the same dependent node (otherwise they're not really
1622                 // duplicates)
1623                 // Note - there are sometimes more than one dependent node type since
1624                 // one nodeType can be used in
1625                 // different ways. But for a particular node, it will only have one
1626                 // dependent node that it's
1627                 // connected to.
1628                 Collection <String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
1629
1630                 if (depNodeTypes.isEmpty()) {
1631                         // This kind of node is not dependent on any other. That is ok.
1632                 } else {
1633                         String depNodeVtxId4A = "";
1634                         String depNodeVtxId4B = "";
1635                         Iterator<String> iter = depNodeTypes.iterator();
1636                         while (iter.hasNext()) {
1637                                 String depNodeType = iter.next();
1638                                 if (nodeTypesConn2A.containsKey(depNodeType)) {
1639                                         // This is the dependent node type that vertex A is using
1640                                         depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
1641                                 }
1642                                 if (nodeTypesConn2B.containsKey(depNodeType)) {
1643                                         // This is the dependent node type that vertex B is using
1644                                         depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
1645                                 }
1646                         }
1647                         if (depNodeVtxId4A.equals("")
1648                                         || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
1649                                 // Either they're not really dupes or there's some bad data - so
1650                                 // don't pick either one
1651                                 return nullVtx;
1652                         }
1653                 }
1654
1655                 if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
1656                         // 2 - If they both have edges to all the same vertices, then return
1657                         // the one with the lower vertexId.
1658                         boolean allTheSame = true;
1659                         Iterator<String> iter = vtxIdsConn2A.iterator();
1660                         while (iter.hasNext()) {
1661                                 String vtxIdConn2A = iter.next();
1662                                 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1663                                         allTheSame = false;
1664                                         break;
1665                                 }
1666                         }
1667
1668                         if (allTheSame) {
1669                                 if (vidA < vidB) {
1670                                         preferredVtx = vtxA;
1671                                 } else {
1672                                         preferredVtx = vtxB;
1673                                 }
1674                         }
1675                 } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
1676                         // 3 - VertexA is connected to more things than vtxB.
1677                         // We'll pick VtxA if its edges are a superset of vtxB's edges.
1678                         boolean missingOne = false;
1679                         Iterator<String> iter = vtxIdsConn2B.iterator();
1680                         while (iter.hasNext()) {
1681                                 String vtxIdConn2B = iter.next();
1682                                 if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
1683                                         missingOne = true;
1684                                         break;
1685                                 }
1686                         }
1687                         if (!missingOne) {
1688                                 preferredVtx = vtxA;
1689                         }
1690                 } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
1691                         // 4 - VertexB is connected to more things than vtxA.
1692                         // We'll pick VtxB if its edges are a superset of vtxA's edges.
1693                         boolean missingOne = false;
1694                         Iterator<String> iter = vtxIdsConn2A.iterator();
1695                         while (iter.hasNext()) {
1696                                 String vtxIdConn2A = iter.next();
1697                                 if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
1698                                         missingOne = true;
1699                                         break;
1700                                 }
1701                         }
1702                         if (!missingOne) {
1703                                 preferredVtx = vtxB;
1704                         }
1705                 } else {
1706                         preferredVtx = nullVtx;
1707                 }
1708
1709                 return (preferredVtx);
1710
1711         } // end of pickOneOfTwoDupes()
1712
1713         /**
1714          * Check and process dupes.
1715          *
1716          * @param transId the trans id
1717          * @param fromAppId the from app id
1718          * @param g the g
1719          * @param version the version
1720          * @param nType the n type
1721          * @param passedVertList the passed vert list
1722          * @param dupeFixOn the dupe fix on
1723          * @param deleteCandidateList the delete candidate list
1724          * @param singleCommits the single commits
1725          * @param alreadyFoundDupeGroups the already found dupe groups
1726          * @param dbMaps the db maps
1727          * @return the array list
1728          */
1729         private static List<String> checkAndProcessDupes(String transId,
1730                         String fromAppId, Graph g, GraphTraversalSource source, String version, String nType,
1731                         List<Vertex> passedVertList, Boolean dupeFixOn,
1732                         Set<String> deleteCandidateList, Boolean singleCommits,
1733                         ArrayList<String> alreadyFoundDupeGroups, Loader loader ) {
1734
1735                 ArrayList<String> returnList = new ArrayList<>();
1736                 ArrayList<Vertex> checkVertList = new ArrayList<>();
1737                 ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
1738                 Boolean noFilterList = true;
1739                 Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
1740                 while (afItr.hasNext()) {
1741                         String dupeGrpStr = afItr.next();
1742                         String[] dupeArr = dupeGrpStr.split("\\|");
1743                         int lastIndex = dupeArr.length - 1;
1744                         for (int i = 0; i < lastIndex; i++) {
1745                                 // Note: we don't want the last one...
1746                                 String vidString = dupeArr[i];
1747                                 alreadyFoundDupeVidArr.add(vidString);
1748                                 noFilterList = false;
1749                         }
1750                 }
1751
1752                 // For a given set of Nodes that were found with a set of KEY
1753                 // Parameters, (nodeType + key data) we will
1754                 // see if we find any duplicate nodes that need to be cleaned up. Note -
1755                 // it's legit to have more than one
1756                 // node with the same key data if the nodes depend on a parent for
1757                 // uniqueness -- as long as the two nodes
1758                 // don't hang off the same Parent.
1759                 // If we find duplicates, and we can figure out which of each set of
1760                 // duplicates is the one that we
1761                 // think should be preserved, we will record that. Whether we can tell
1762                 // which one should be
1763                 // preserved or not, we will return info about any sets of duplicates
1764                 // found.
1765                 //
1766                 // Each element in the returned arrayList might look like this:
1767                 // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
1768                 // couldn't figure out which one to keep)
1769                 // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
1770                 // thought the third one was the one that should survive)
1771
1772                 // Because of the way the calling code loops over stuff, we can get the
1773                 // same data multiple times - so we should
1774                 // not process any vertices that we've already seen.
1775
1776                 try {
1777                         Iterator<Vertex> pItr = passedVertList.iterator();
1778                         while (pItr.hasNext()) {
1779                                 Vertex tvx = pItr.next();
1780                                 String passedId = tvx.id().toString();
1781                                 if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
1782                                         // We haven't seen this one before - so we should check it.
1783                                         checkVertList.add(tvx);
1784                                 }
1785                         }
1786
1787                         if (checkVertList.size() < 2) {
1788                                 // Nothing new to check.
1789                                 return returnList;
1790                         }
1791
1792                         if (loader.introspectorFromName(nType).isTopLevel()) {
1793                                 // If this was a node that does NOT depend on other nodes for
1794                                 // uniqueness, and we
1795                                 // found more than one node using its key -- record the found
1796                                 // vertices as duplicates.
1797                                 String dupesStr = "";
1798                                 for (int i = 0; i < checkVertList.size(); i++) {
1799                                         dupesStr = dupesStr
1800                                                         + ((checkVertList.get(i))).id()
1801                                                                         .toString() + "|";
1802                                 }
1803                                 if (dupesStr != "") {
1804                                         Vertex prefV = getPreferredDupe(transId, fromAppId,
1805                                                         source, checkVertList, version, loader);
1806                                         if (prefV == null) {
1807                                                 // We could not determine which duplicate to keep
1808                                                 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1809                                                 returnList.add(dupesStr);
1810                                         } else {
1811                                                 dupesStr = dupesStr + "KeepVid=" + prefV.id();
1812                                                 Boolean didRemove = false;
1813                                                 if (dupeFixOn) {
1814                                                         didRemove = deleteNonKeepersIfAppropriate(g,
1815                                                                         dupesStr, prefV.id().toString(),
1816                                                                         deleteCandidateList, singleCommits);
1817                                                 }
1818                                                 if (didRemove) {
1819                                                         dupeGrpsDeleted++;
1820                                                 } else {
1821                                                         // keep them on our list
1822                                                         returnList.add(dupesStr);
1823                                                 }
1824                                         }
1825                                 }
1826                         } else {
1827                                 // More than one node have the same key fields since they may
1828                                 // depend on a parent node for uniqueness. Since we're finding
1829                                 // more than one, we want to check to see if any of the
1830                                 // vertices that have this set of keys (and are the same nodeType)
1831                                 // are also pointing at the same 'parent' node.
1832                                 // Note: for a given set of key data, it is possible that there
1833                                 // could be more than one set of duplicates.
1834                                 HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
1835                                                 transId, fromAppId, source, version, nType,
1836                                                 checkVertList, loader);
1837                                 for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
1838                                                 .entrySet()) {
1839                                         ArrayList<Vertex> thisParentsVertList = entry
1840                                                         .getValue();
1841                                         if (thisParentsVertList.size() > 1) {
1842                                                 // More than one vertex found with the same key info
1843                                                 // hanging off the same parent/dependent node
1844                                                 String dupesStr = "";
1845                                                 for (int i = 0; i < thisParentsVertList.size(); i++) {
1846                                                         dupesStr = dupesStr
1847                                                                         + ((thisParentsVertList
1848                                                                                         .get(i))).id() + "|";
1849                                                 }
1850                                                 if (dupesStr != "") {
1851                                                         Vertex prefV = getPreferredDupe(transId,
1852                                                                         fromAppId, source, thisParentsVertList,
1853                                                                         version, loader);
1854
1855                                                         if (prefV == null) {
1856                                                                 // We could not determine which duplicate to
1857                                                                 // keep
1858                                                                 dupesStr = dupesStr + "KeepVid=UNDETERMINED";
1859                                                                 returnList.add(dupesStr);
1860                                                         } else {
1861                                                                 Boolean didRemove = false;
1862                                                                 dupesStr = dupesStr + "KeepVid="
1863                                                                                 + prefV.id().toString();
1864                                                                 if (dupeFixOn) {
1865                                                                         didRemove = deleteNonKeepersIfAppropriate(
1866                                                                                         g, dupesStr, prefV.id()
1867                                                                                                         .toString(),
1868                                                                                         deleteCandidateList, singleCommits);
1869                                                                 }
1870                                                                 if (didRemove) {
1871                                                                         dupeGrpsDeleted++;
1872                                                                 } else {
1873                                                                         // keep them on our list
1874                                                                         returnList.add(dupesStr);
1875                                                                 }
1876                                                         }
1877                                                 }
1878                                         }
1879                                 }
1880                         }
1881                 } catch (Exception e) {
1882                         LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
1883                 }
1884
1885                 return returnList;
1886
1887         }// End of checkAndProcessDupes()
1888
1889         /**
1890          * Group verts by dep nodes.
1891          *
1892          * @param transId the trans id
1893          * @param fromAppId the from app id
1894          * @param g the g
1895          * @param version the version
1896          * @param nType the n type
1897          * @param passedVertList the passed vert list
1898          * @param dbMaps the db maps
1899          * @return the hash map
1900          * @throws AAIException the AAI exception
1901          */
1902         private static HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
1903                         String transId, String fromAppId, GraphTraversalSource g, String version,
1904                         String nType, ArrayList<Vertex> passedVertList, Loader loader)
1905                         throws AAIException {
1906                 // Given a list of Titan Vertices of one nodeType (see AAI-8956), group
1907                 // them together by the parent node they depend on.
1908                 // Ie. if given a list of ip address nodes (assumed to all have the
1909                 // same key info) they might sit under several different parent vertices.
1910                 // Under Normal conditions, there would only be one per parent -- but
1911                 // we're trying to find duplicates - so we
1912                 // allow for the case where more than one is under the same parent node.
1913
1914                 HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
1915                 if (loader.introspectorFromName(nType).isTopLevel()) {
1916                         // This method really should not have been called if this is not the
1917                         // kind of node
1918                         // that depends on a parent for uniqueness, so just return the empty
1919                         // hash.
1920                         return retHash;
1921                 }
1922
1923                 // Find out what types of nodes the passed in nodes can depend on
1924                 ArrayList<String> depNodeTypeL = new ArrayList<>();
1925                 Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn();
1926                 Iterator<String> ntItr = depNTColl.iterator();
1927                 while (ntItr.hasNext()) {
1928                         depNodeTypeL.add(ntItr.next());
1929                 }
1930                 // For each vertex, we want find its depended-on/parent vertex so we
1931                 // can track what other vertexes that are dependent on that same guy.
1932                 if (passedVertList != null) {
1933                         Iterator<Vertex> iter = passedVertList.iterator();
1934                         while (iter.hasNext()) {
1935                                 Vertex thisVert = iter.next();
1936                                 Vertex tmpParentVtx = getConnectedParent( g, thisVert );
1937                                 if( tmpParentVtx != null ) {
1938                                         String parentNt = null;
1939                                         Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
1940                                         if (obj != null) {
1941                                                 parentNt = obj.toString();
1942                                         }
1943                                         if (depNTColl.contains(parentNt)) {
1944                                                 // This must be the parent/dependent node
1945                                                 String parentVid = tmpParentVtx.id().toString();
1946                                                 if (retHash.containsKey(parentVid)) {
1947                                                         // add this vert to the list for this parent key
1948                                                         retHash.get(parentVid).add(thisVert);
1949                                                 } else {
1950                                                         // This is the first one we found on this parent
1951                                                         ArrayList<Vertex> vList = new ArrayList<>();
1952                                                         vList.add(thisVert);
1953                                                         retHash.put(parentVid, vList);
1954                                                 }
1955                                         }
1956                                 }
1957                         }
1958                 }
1959
1960                 return retHash;
1961
1962         }// end of groupVertsByDepNodes()
1963
1964         /**
1965          * Delete non keepers if appropriate.
1966          *
1967          * @param g the g
1968          * @param dupeInfoString the dupe info string
1969          * @param vidToKeep the vid to keep
1970          * @param deleteCandidateList the delete candidate list
1971          * @param singleCommits the single commits
1972          * @return the boolean
1973          */
1974         private static Boolean deleteNonKeepersIfAppropriate(Graph g,
1975                         String dupeInfoString, String vidToKeep,
1976                         Set<String> deleteCandidateList, Boolean singleCommits) {
1977
1978                 Boolean deletedSomething = false;
1979                 // This assumes that the dupeInfoString is in the format of
1980                 // pipe-delimited vid's followed by
1981                 // ie. "3456|9880|keepVid=3456"
1982                 if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
1983                         // No vid's on the candidate list -- so no deleting will happen on
1984                         // this run
1985                         return false;
1986                 }
1987
1988                 String[] dupeArr = dupeInfoString.split("\\|");
1989                 ArrayList<String> idArr = new ArrayList<>();
1990                 int lastIndex = dupeArr.length - 1;
1991                 for (int i = 0; i <= lastIndex; i++) {
1992                         if (i < lastIndex) {
1993                                 // This is not the last entry, it is one of the dupes,
1994                                 String vidString = dupeArr[i];
1995                                 idArr.add(vidString);
1996                         } else {
1997                                 // This is the last entry which should tell us if we have a
1998                                 // preferred keeper
1999                                 String prefString = dupeArr[i];
2000                                 if (prefString.equals("KeepVid=UNDETERMINED")) {
2001                                         // They sent us a bad string -- nothing should be deleted if
2002                                         // no dupe could be tagged as preferred
2003                                         return false;
2004                                 } else {
2005                                         // If we know which to keep, then the prefString should look
2006                                         // like, "KeepVid=12345"
2007                                         String[] prefArr = prefString.split("=");
2008                                         if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
2009                                                 LOGGER.error("Bad format. Expecting KeepVid=999999");
2010                                                 return false;
2011                                         } else {
2012                                                 String keepVidStr = prefArr[1];
2013                                                 if (idArr.contains(keepVidStr)) {
2014                                                         idArr.remove(keepVidStr);
2015
2016                                                         // So now, the idArr should just contain the vid's
2017                                                         // that we want to remove.
2018                                                         for (int x = 0; x < idArr.size(); x++) {
2019                                                                 boolean okFlag = true;
2020                                                                 String thisVid = idArr.get(x);
2021                                                                 if (deleteCandidateList.contains(thisVid)) {
2022                                                                         // This vid is a valid delete candidate from
2023                                                                         // a prev. run, so we can remove it.
2024                                                                         try {
2025                                                                                 long longVertId = Long
2026                                                                                                 .parseLong(thisVid);
2027                                                                                 Vertex vtx = g
2028                                                                                                 .traversal().V(longVertId).next();
2029                                                                                 vtx.remove();
2030                                                                                 if (singleCommits) {
2031                                                                                         // NOTE - the singleCommits option is not used in normal processing
2032                                                                                         g.tx().commit();
2033                                                                                         g = AAIGraph.getInstance().getGraph().newTransaction();
2034                                                                                 }
2035                                                                         } catch (Exception e) {
2036                                                                                 okFlag = false;
2037                                                                                 LOGGER.error("ERROR trying to delete VID = " + thisVid, e);
2038                                                                         }
2039                                                                         if (okFlag) {
2040                                                                                 LOGGER.info(" DELETED VID = " + thisVid);
2041                                                                                 deletedSomething = true;
2042                                                                         }
2043                                                                 }
2044                                                         }
2045                                                 } else {
2046                                                         LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes.  dupeInfoString = ["
2047                                                                         + dupeInfoString + "]");
2048                                                         return false;
2049                                                 }
2050                                         }
2051                                 }// else we know which one to keep
2052                         }// else last entry
2053                 }// for each vertex in a group
2054
2055                 return deletedSomething;
2056
2057         }// end of deleteNonKeepersIfAppropriate()
2058
2059
2060         /**
2061          * Gets the node just using key params.
2062          *
2063          * @param transId the trans id
2064          * @param fromAppId the from app id
2065          * @param graph the graph
2066          * @param nodeType the node type
2067          * @param keyPropsHash the key props hash
2068          * @param apiVersion the api version
2069          * @return the node just using key params
2070          * @throws AAIException the AAI exception
2071          */
2072         public static List <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, GraphTraversalSource graph, String nodeType,
2073                         HashMap<String,Object> keyPropsHash, String apiVersion )         throws AAIException{
2074
2075                 List <Vertex> retVertList = new ArrayList <> ();
2076
2077                 // We assume that all NodeTypes have at least one key-property defined.
2078                 // Note - instead of key-properties (the primary key properties), a user could pass
2079                 //        alternate-key values if they are defined for the nodeType.
2080                 List<String> kName = new ArrayList<>();
2081                 List<Object> kVal = new ArrayList<>();
2082                 if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
2083                         throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request.  NodeType = [" + nodeType + "]. ");
2084                 }
2085
2086                 int i = -1;
2087                 for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){
2088                         i++;
2089                         kName.add(i, entry.getKey());
2090                         kVal.add(i, entry.getValue());
2091                 }
2092                 int topPropIndex = i;
2093                 Vertex tiV = null;
2094                 String propsAndValuesForMsg = "";
2095                 Iterator <Vertex> verts = null;
2096
2097                 try {
2098                         if( topPropIndex == 0 ){
2099                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
2100                                 verts= graph.V().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType);
2101                         }
2102                         else if( topPropIndex == 1 ){
2103                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2104                                                 + kName.get(1) + " = " + kVal.get(1) + ") ";
2105                                 verts =  graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType);
2106                         }
2107                         else if( topPropIndex == 2 ){
2108                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2109                                                 + kName.get(1) + " = " + kVal.get(1) + ", "
2110                                                 + kName.get(2) + " = " + kVal.get(2) +  ") ";
2111                                 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType);
2112                         }
2113                         else if( topPropIndex == 3 ){
2114                                 propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
2115                                                 + kName.get(1) + " = " + kVal.get(1) + ", "
2116                                                 + kName.get(2) + " = " + kVal.get(2) + ", "
2117                                                 + kName.get(3) + " = " + kVal.get(3) +  ") ";
2118                                 verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType);
2119                         }
2120                         else {
2121                                 throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
2122                         }
2123                 }
2124                 catch( Exception ex ){
2125                         LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]", ex);
2126                 }
2127
2128                 if( verts != null ){
2129                         while( verts.hasNext() ){
2130                                 tiV = verts.next();
2131                                 retVertList.add(tiV);
2132                         }
2133                 }
2134
2135                 if( retVertList.size() == 0 ){
2136                         LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
2137                                         "], propsAndVal = " + propsAndValuesForMsg );
2138                 }
2139
2140                 return retVertList;
2141
2142         }// End of getNodeJustUsingKeyParams()
2143
2144         /**
2145          * Show all edges for node.
2146          *
2147          * @param transId the trans id
2148          * @param fromAppId the from app id
2149          * @param tVert the t vert
2150          * @return the array list
2151          */
2152         private static ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, Vertex tVert ){
2153
2154                 ArrayList <String> retArr = new ArrayList <> ();
2155                 Iterator <Edge> eI = tVert.edges(Direction.IN);
2156                 if( ! eI.hasNext() ){
2157                         retArr.add("No IN edges were found for this vertex. ");
2158                 }
2159                 while( eI.hasNext() ){
2160                         Edge ed = eI.next();
2161                         String lab = ed.label();
2162                         Vertex vtx;
2163                         if (tVert.equals(ed.inVertex())) {
2164                                 vtx = ed.outVertex();
2165                         } else {
2166                                 vtx = ed.inVertex();
2167                         }
2168                         if( vtx == null ){
2169                                 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2170                         }
2171                         else {
2172                                 String nType = vtx.<String>property("aai-node-type").orElse(null);
2173                                 String vid = vtx.id().toString();
2174                                 retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
2175
2176                         }
2177                 }
2178
2179                 eI = tVert.edges(Direction.OUT);
2180                 if( ! eI.hasNext() ){
2181                         retArr.add("No OUT edges were found for this vertex. ");
2182                 }
2183                 while( eI.hasNext() ){
2184                         Edge ed =  eI.next();
2185                         String lab = ed.label();
2186                         Vertex vtx;
2187                         if (tVert.equals(ed.inVertex())) {
2188                                 vtx = ed.outVertex();
2189                         } else {
2190                                 vtx = ed.inVertex();
2191                         }
2192                         if( vtx == null ){
2193                                 retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
2194                         }
2195                         else {
2196                                 String nType = vtx.<String>property("aai-node-type").orElse(null);
2197                                 String vid = vtx.id().toString();
2198                                 retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
2199                         }
2200                 }
2201                 return retArr;
2202         }
2203
2204
2205         /**
2206          * Show properties for node.
2207          *
2208          * @param transId the trans id
2209          * @param fromAppId the from app id
2210          * @param tVert the t vert
2211          * @return the array list
2212          */
2213         private static ArrayList <String> showPropertiesForNode( String transId, String fromAppId, Vertex tVert ){
2214
2215                 ArrayList <String> retArr = new ArrayList <> ();
2216                 if( tVert == null ){
2217                         retArr.add("null Node object passed to showPropertiesForNode()\n");
2218                 }
2219                 else {
2220                         String nodeType = "";
2221                         Object ob = tVert.<Object>property("aai-node-type").orElse(null);
2222                         if( ob == null ){
2223                                 nodeType = "null";
2224                         }
2225                         else{
2226                                 nodeType = ob.toString();
2227                         }
2228
2229                         retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
2230                         retArr.add(" Property Detail: ");
2231                         Iterator<VertexProperty<Object>> pI = tVert.properties();
2232                         while( pI.hasNext() ){
2233                                 VertexProperty<Object> tp = pI.next();
2234                                 Object val = tp.value();
2235                                 retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
2236                         }
2237                 }
2238                 return retArr;
2239         }
2240
2241
2242         private static ArrayList <Vertex> getConnectedNodes(GraphTraversalSource g, Vertex startVtx )
2243                         throws AAIException {
2244
2245                 ArrayList <Vertex> retArr = new ArrayList <> ();
2246                 if( startVtx == null ){
2247                         return retArr;
2248                 }
2249                 else {
2250                          GraphTraversal<Vertex, Vertex> modPipe = null;
2251                          modPipe = g.V(startVtx).both();
2252                          if( modPipe != null && modPipe.hasNext() ){
2253                                 while( modPipe.hasNext() ){
2254                                         Vertex conVert = modPipe.next();
2255                                         retArr.add(conVert);
2256                                 }
2257                         }
2258                 }
2259                 return retArr;
2260
2261         }// End of getConnectedNodes()
2262
2263
2264         private static ArrayList <Vertex> getConnectedChildrenOfOneType( GraphTraversalSource g,
2265                         Vertex startVtx, String childNType ) throws AAIException{
2266
2267                 ArrayList <Vertex> childList = new ArrayList <> ();
2268                 Iterator <Vertex> vertI =  g.V(startVtx).union(__.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).inV(), __.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).outV());
2269
2270                 Vertex tmpVtx = null;
2271                 while( vertI != null && vertI.hasNext() ){
2272                         tmpVtx = vertI.next();
2273                         Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
2274                         if (ob != null) {
2275                                 String tmpNt = ob.toString();
2276                                 if( tmpNt.equals(childNType)){
2277                                         childList.add(tmpVtx);
2278                                 }
2279                         }
2280                 }
2281
2282                 return childList;
2283
2284         }// End of getConnectedChildrenOfOneType()
2285
2286
2287         private static Vertex getConnectedParent( GraphTraversalSource g,
2288                         Vertex startVtx ) throws AAIException{
2289
2290                 Vertex parentVtx = null;
2291                 Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
2292
2293                 while( vertI != null && vertI.hasNext() ){
2294                         // Note - there better only be one!
2295                         parentVtx = vertI.next();
2296                 }
2297
2298                 return parentVtx;
2299
2300         }// End of getConnectedParent()
2301
2302
2303         private static long figureWindowStartTime( int timeWindowMinutes ){
2304                 // Given a window size, calculate what the start-timestamp would be.
2305
2306                 if( timeWindowMinutes <= 0 ){
2307                         // This just means that there is no window...
2308                         return 0;
2309                 }
2310                 long unixTimeNow = System.currentTimeMillis();
2311                 long windowInMillis = timeWindowMinutes * 60L * 1000;
2312
2313                 long startTimeStamp = unixTimeNow - windowInMillis;
2314
2315                 return startTimeStamp;
2316         } // End of figureWindowStartTime()
2317
2318
2319 }