aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  *
  20  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  */
  22 package org.onap.aai.serialization.db;
  23
  24
  25 import com.att.eelf.configuration.EELFLogger;
  26 import com.att.eelf.configuration.EELFManager;
  27 import com.google.common.base.CaseFormat;
  28 import com.thinkaurelius.titan.core.SchemaViolationException;
  29 import org.apache.commons.collections.IteratorUtils;
  30 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
  31 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
  32 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
  33 import org.apache.tinkerpop.gremlin.structure.*;
  34 import org.javatuples.Pair;
  35 import org.javatuples.Triplet;
  36 import org.onap.aai.db.props.AAIProperties;
  37 import org.onap.aai.exceptions.AAIException;
  38 import org.onap.aai.introspection.*;
  39 import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
  40 import org.onap.aai.introspection.sideeffect.DataCopy;
  41 import org.onap.aai.introspection.sideeffect.DataLinkReader;
  42 import org.onap.aai.introspection.sideeffect.DataLinkWriter;
  43 import org.onap.aai.introspection.sideeffect.SideEffectRunner;
  44 import org.onap.aai.logging.ErrorLogHelper;
  45 import org.onap.aai.logging.LogFormatTools;
  46 import org.onap.aai.parsers.query.QueryParser;
  47 import org.onap.aai.parsers.uri.URIParser;
  48 import org.onap.aai.parsers.uri.URIToRelationshipObject;
  49 import org.onap.aai.query.builder.QueryBuilder;
  50 import org.onap.aai.schema.enums.ObjectMetadata;
  51 import org.onap.aai.schema.enums.PropertyMetadata;
  52 import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException;
  53 import org.onap.aai.serialization.db.util.VersionChecker;
  54 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
  55 import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
  56 import org.onap.aai.util.AAIApiServerURLBase;
  57 import org.onap.aai.util.AAIConfig;
  58 import org.onap.aai.util.AAIConstants;
  59 import org.onap.aai.workarounds.NamingExceptions;
  60 import org.onap.aai.logging.StopWatch;
  61
  62 import javax.ws.rs.core.UriBuilder;
  63 import java.io.UnsupportedEncodingException;
  64 import java.lang.reflect.Array;
  65 import java.lang.reflect.InvocationTargetException;
  66 import java.net.MalformedURLException;
  67 import java.net.URI;
  68 import java.net.URISyntaxException;
  69 import java.util.*;
  70 import java.util.concurrent.Callable;
  71 import java.util.concurrent.ExecutionException;
  72 import java.util.concurrent.ExecutorService;
  73 import java.util.concurrent.Future;
  74 import java.util.regex.Matcher;
  75 import java.util.regex.Pattern;
  76
  77 public class DBSerializer {
  78
  79         private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DBSerializer.class);
  80
  81         private final TransactionalGraphEngine engine;
  82         private final String sourceOfTruth;
  83         private final ModelType introspectionType;
  84         private final Version version;
  85         private final Loader latestLoader;
  86         private final EdgeRules edgeRules = EdgeRules.getInstance();
  87         private final Loader loader;
  88         private final String baseURL;
  89         private double dbTimeMsecs = 0;
  90         /**
  91          * Instantiates a new DB serializer.
  92          *
  93          * @param version the version
  94          * @param engine the engine
  95          * @param g the g
  96          * @param introspectionType the introspection type
  97          * @param sourceOfTruth the source of truth
  98          * @param llBuilder the ll builder
  99          * @throws AAIException
 100          */
 101         public DBSerializer(Version version, TransactionalGraphEngine engine, ModelType introspectionType, String sourceOfTruth) throws AAIException {
 102                 this.engine = engine;
 103                 this.sourceOfTruth = sourceOfTruth;
 104                 this.introspectionType = introspectionType;
 105                 this.latestLoader = LoaderFactory.createLoaderForVersion(introspectionType, AAIProperties.LATEST);
 106                 this.version = version;
 107                 this.loader = LoaderFactory.createLoaderForVersion(introspectionType, version);
 108                 this.baseURL = AAIApiServerURLBase.get(version);
 109         }
 110
 111         /**
 112          * Touch standard vertex properties.
 113          *
 114          * @param v the v
 115          * @param isNewVertex the is new vertex
 116          */
 117         public void touchStandardVertexProperties(Vertex v, boolean isNewVertex) {
 118
 119                 String timeNowInSec = Long.toString(System.currentTimeMillis());
 120                 if (isNewVertex) {
 121                         v.property(AAIProperties.SOURCE_OF_TRUTH, this.sourceOfTruth);
 122                         v.property(AAIProperties.CREATED_TS, timeNowInSec);
 123                         v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString());
 124                 }
 125                 v.property(AAIProperties.RESOURCE_VERSION, timeNowInSec );
 126                 v.property(AAIProperties.LAST_MOD_TS, timeNowInSec);
 127                 v.property(AAIProperties.LAST_MOD_SOURCE_OF_TRUTH, this.sourceOfTruth);
 128
 129         }
 130
 131         private void touchStandardVertexProperties(String nodeType, Vertex v, boolean isNewVertex) {
 132
 133                 v.property(AAIProperties.NODE_TYPE, nodeType);
 134                 touchStandardVertexProperties(v, isNewVertex);
 135
 136         }
 137
 138
 139
 140         /**
 141          * Creates the new vertex.
 142          *
 143          * @param wrappedObject the wrapped object
 144          * @return the vertex
 145          * @throws UnsupportedEncodingException the unsupported encoding exception
 146          * @throws AAIException the AAI exception
 147          */
 148         public Vertex createNewVertex(Introspector wrappedObject) {
 149                 Vertex v;
 150                 try {
 151                         StopWatch.conditionalStart();
 152                         v = this.engine.tx().addVertex();
 153                         touchStandardVertexProperties(wrappedObject.getDbName(), v, true);
 154                 }
 155                 finally {
 156                         dbTimeMsecs += StopWatch.stopIfStarted();
 157                 }
 158                 return v;
 159         }
 160
 161         /**
 162          * Trim class name.
 163          *
 164          * @param className the class name
 165          * @return the string
 166          */
 167         /*
 168          * Removes the classpath from a class name
 169          */
 170         public String trimClassName (String className) {
 171                 String returnValue = "";
 172
 173                 if (className.lastIndexOf('.') == -1) {
 174                         return className;
 175                 }
 176                 returnValue = className.substring(className.lastIndexOf('.') + 1, className.length());
 177
 178                 return returnValue;
 179         }
 180
 181         /**
 182          * Serialize to db.
 183          *
 184          * @param obj the obj
 185          * @param v the v
 186          * @param uriQuery the uri query
 187          * @param identifier the identifier
 188          * @throws SecurityException the security exception
 189          * @throws IllegalAccessException the illegal access exception
 190          * @throws IllegalArgumentException the illegal argument exception
 191          * @throws InvocationTargetException the invocation target exception
 192          * @throws InstantiationException the instantiation exception
 193          * @throws InterruptedException the interrupted exception
 194          * @throws NoSuchMethodException the no such method exception
 195          * @throws AAIException the AAI exception
 196          * @throws UnsupportedEncodingException the unsupported encoding exception
 197          * @throws AAIUnknownObjectException
 198          */
 199         public void serializeToDb(Introspector obj, Vertex v, QueryParser uriQuery, String identifier, String requestContext) throws AAIException, UnsupportedEncodingException {
 200                 StopWatch.conditionalStart();
 201                 try {
 202                         if (uriQuery.isDependent()) {
 203                                 //try to find the parent
 204                                 List<Vertex> vertices = uriQuery.getQueryBuilder().getParentQuery().toList();
 205                                 if (!vertices.isEmpty()) {
 206                                         Vertex parent = vertices.get(0);
 207                                         this.reflectDependentVertex(parent, v, obj, requestContext);
 208                                 } else {
 209                                         dbTimeMsecs += StopWatch.stopIfStarted();
 210                                         throw new AAIException("AAI_6114", "No parent Node of type " + uriQuery.getParentResultType() + " for " + identifier);
 211                                 }
 212                         } else {
 213                                 serializeSingleVertex(v, obj, requestContext);
 214                         }
 215
 216                 } catch (SchemaViolationException e) {
 217                         dbTimeMsecs += StopWatch.stopIfStarted();
 218                         throw new AAIException("AAI_6117", e);
 219                 }
 220                 dbTimeMsecs += StopWatch.stopIfStarted();
 221         }
 222
 223         public void serializeSingleVertex(Vertex v, Introspector obj, String requestContext) throws UnsupportedEncodingException, AAIException {
 224                 StopWatch.conditionalStart();
 225                 try {
 226                         boolean isTopLevel = obj.isTopLevel();
 227                         if (isTopLevel) {
 228                                 v.property(AAIProperties.AAI_URI, obj.getURI());
 229                         }
 230
 231                         processObject(obj, v, requestContext);
 232                         if (!isTopLevel) {
 233                                 URI uri = this.getURIForVertex(v);
 234                                 URIParser parser = new URIParser(this.loader, uri);
 235                                 if (parser.validate()) {
 236                                         VertexProperty<String> uriProp = v.property(AAIProperties.AAI_URI);
 237                                         if (!uriProp.isPresent() || uriProp.isPresent() && !uriProp.value().equals(uri.toString())) {
 238                                                 v.property(AAIProperties.AAI_URI, uri.toString());
 239                                         }
 240                                 }
 241                         }
 242                 } catch (SchemaViolationException e) {
 243                         throw new AAIException("AAI_6117", e);
 244                 }
 245                 finally {
 246                         dbTimeMsecs += StopWatch.stopIfStarted();
 247                 }
 248         }
 249
 250         /**
 251          * Process object.
 252          *
 253          * @param <T> the generic type
 254          * @param obj the obj
 255          * @param v the v
 256          * @return the list
 257          * @throws IllegalAccessException the illegal access exception
 258          * @throws IllegalArgumentException the illegal argument exception
 259          * @throws InvocationTargetException the invocation target exception
 260          * @throws InstantiationException the instantiation exception
 261          * @throws NoSuchMethodException the no such method exception
 262          * @throws SecurityException the security exception
 263          * @throws AAIException the AAI exception
 264          * @throws UnsupportedEncodingException the unsupported encoding exception
 265          * @throws AAIUnknownObjectException
 266          */
 267         /*
 268          * Helper method for reflectToDb
 269          * Handles all the property setting
 270          */
 271         private <T> List<Vertex> processObject (Introspector obj, Vertex v, String requestContext) throws UnsupportedEncodingException, AAIException {
 272                 Set<String> properties = new LinkedHashSet<>(obj.getProperties());
 273                 properties.remove(AAIProperties.RESOURCE_VERSION);
 274                 List<Vertex> dependentVertexes = new ArrayList<>();
 275                 List<Vertex> processedVertexes = new ArrayList<>();
 276                 boolean isComplexType = false;
 277                 boolean isListType = false;
 278                 if (!obj.isContainer()) {
 279                         this.touchStandardVertexProperties(obj.getDbName(), v, false);
 280                 }
 281                 this.executePreSideEffects(obj, v);
 282                 for (String property : properties) {
 283                         Object value = null;
 284                         final String propertyType;
 285                         propertyType = obj.getType(property);
 286                         isComplexType = obj.isComplexType(property);
 287                         isListType = obj.isListType(property);
 288                         value = obj.getValue(property);
 289
 290                         if (!(isComplexType || isListType)) {
 291                                 boolean canModify = this.canModify(obj, property, requestContext);
 292
 293                                 if (canModify) {
 294                                         final Map<PropertyMetadata, String> metadata = obj.getPropertyMetadata(property);
 295                                         String dbProperty = property;
 296                                         if (metadata.containsKey(PropertyMetadata.DB_ALIAS)) {
 297                                                 dbProperty = metadata.get(PropertyMetadata.DB_ALIAS);
 298                                         }
 299                                         if (metadata.containsKey(PropertyMetadata.DATA_LINK)) {
 300                                                 //data linked properties are ephemeral
 301                                                 //they are populated dynamically on GETs
 302                                                 continue;
 303                                         }
 304                                         if (value != null) {
 305                                                 if (!value.equals(v.property(dbProperty).orElse(null))) {
 306                                                         if (propertyType.toLowerCase().contains(".long")) {
 307                                                                 v.property(dbProperty, new Integer(((Long)value).toString()));
 308                                                         } else {
 309                                                                 v.property(dbProperty, value);
 310                                                         }
 311                                                 }
 312                                         } else {
 313                                                 v.property(dbProperty).remove();
 314                                         }
 315                                 }
 316                         } else if (isListType) {
 317                                 List<Object> list = (List<Object>)value;
 318                                 if (obj.isComplexGenericType(property)) {
 319                                         if (list != null) {
 320                                                 for (Object o : list) {
 321                                                         Introspector child = IntrospectorFactory.newInstance(this.introspectionType, o);
 322                                                         child.setURIChain(obj.getURI());
 323                                                         processedVertexes.add(reflectDependentVertex(v, child, requestContext));
 324                                                 }
 325                                         }
 326                                 } else {
 327                                         //simple list case
 328                                         engine.setListProperty(v, property, list);
 329                                 }
 330                         } else {
 331                                 //method.getReturnType() is not 'simple' then create a vertex and edge recursively returning an edge back to this method
 332                                 if (value != null) { //effectively ignore complex properties not included in the object we're processing
 333                                         if (value.getClass().isArray()) {
 334
 335                                                 int length = Array.getLength(value);
 336                                             for (int i = 0; i < length; i ++) {
 337                                                 Object arrayElement = Array.get(value, i);
 338                                                 Introspector child = IntrospectorFactory.newInstance(this.introspectionType, arrayElement);
 339                                                         child.setURIChain(obj.getURI());
 340                                                         processedVertexes.add(reflectDependentVertex(v, child, requestContext));
 341
 342                                             }
 343                                         } else if (!property.equals("relationship-list")) {
 344                                                 // container case
 345                                                 Introspector introspector = IntrospectorFactory.newInstance(this.introspectionType, value);
 346                                                 if (introspector.isContainer()) {
 347                                                         dependentVertexes.addAll(this.engine.getQueryEngine().findChildrenOfType(v, introspector.getChildDBName()));
 348                                                         introspector.setURIChain(obj.getURI());
 349
 350                                                         processedVertexes.addAll(processObject(introspector, v, requestContext));
 351
 352                                                 } else {
 353                                                         dependentVertexes.addAll(this.engine.getQueryEngine().findChildrenOfType(v, introspector.getDbName()));
 354                                                         processedVertexes.add(reflectDependentVertex(v, introspector, requestContext));
 355
 356                                                 }
 357                                         } else if (property.equals("relationship-list")) {
 358                                                 handleRelationships(obj, v);
 359                                         }
 360                                 }
 361                         }
 362                 }
 363                 this.writeThroughDefaults(v, obj);
 364                 /* handle those vertexes not touched */
 365                 for (Vertex toBeRemoved : processedVertexes) {
 366                         dependentVertexes.remove(toBeRemoved);
 367                 }
 368                 this.deleteItemsWithTraversal(dependentVertexes);
 369
 370                 this.executePostSideEffects(obj, v);
 371                 return processedVertexes;
 372         }
 373
 374         /**
 375          * Handle relationships.
 376          *
 377          * @param obj the obj
 378          * @param vertex the vertex
 379          * @throws SecurityException the security exception
 380          * @throws IllegalAccessException the illegal access exception
 381          * @throws IllegalArgumentException the illegal argument exception
 382          * @throws InvocationTargetException the invocation target exception
 383          * @throws UnsupportedEncodingException the unsupported encoding exception
 384          * @throws AAIException the AAI exception
 385          */
 386         /*
 387          * Handles the explicit relationships defined for an obj
 388          */
 389         private void handleRelationships(Introspector obj, Vertex vertex) throws UnsupportedEncodingException, AAIException {
 390
 391
 392
 393                 Introspector wrappedRl = obj.getWrappedValue("relationship-list");
 394                 processRelationshipList(wrappedRl, vertex);
 395
 396
 397         }
 398
 399
 400         /**
 401          * Process relationship list.
 402          *
 403          * @param wrapped the wrapped
 404          * @param v the v
 405          * @throws UnsupportedEncodingException the unsupported encoding exception
 406          * @throws AAIException the AAI exception
 407          */
 408         private void processRelationshipList(Introspector wrapped, Vertex v) throws UnsupportedEncodingException, AAIException {
 409
 410                 List<Object> relationships = (List<Object>)wrapped.getValue("relationship");
 411
 412                 List<Triplet<Vertex, Vertex, String>> addEdges = new ArrayList<>();
 413                 List<Edge> existingEdges = this.engine.getQueryEngine().findEdgesForVersion(v, wrapped.getLoader());
 414
 415                 for (Object relationship : relationships) {
 416                         Edge e = null;
 417                         Vertex cousinVertex = null;
 418                         String label = null;
 419                         Introspector wrappedRel = IntrospectorFactory.newInstance(this.introspectionType, relationship);
 420                         QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(wrappedRel);
 421
 422                         if (wrappedRel.hasProperty("relationship-label")) {
 423                                 label = wrappedRel.getValue("relationship-label");
 424                         }
 425
 426                         List<Vertex> results = parser.getQueryBuilder().toList();
 427                         if (results.isEmpty()) {
 428                                 final AAIException ex = new AAIException("AAI_6129", "Node of type " + parser.getResultType() + ". Could not find object at: " + parser.getUri());
 429                                 ex.getTemplateVars().add(parser.getResultType());
 430                                 ex.getTemplateVars().add(parser.getUri().toString());
 431                                 throw ex;
 432                         } else {
 433                                 //still an issue if there's more than one
 434                                 cousinVertex = results.get(0);
 435                         }
 436
 437                         if (cousinVertex != null) {
 438
 439                                 if (!edgeRules.hasEdgeRule(v, cousinVertex, label)) {
 440                                         throw new AAIException("AAI_6120", "No EdgeRule found for passed nodeTypes: " + v.property(AAIProperties.NODE_TYPE).value().toString() + ", "
 441                                                         + cousinVertex.property(AAIProperties.NODE_TYPE).value().toString() + (label != null ? (" with label " + label):"")  +".");
 442                                 } else if (edgeRules.hasTreeEdgeRule(v, cousinVertex) && !edgeRules.hasCousinEdgeRule(v, cousinVertex, label)) {
 443                                         throw new AAIException("AAI_6145");
 444                                 }
 445
 446                                 e = this.getEdgeBetween(EdgeType.COUSIN, v, cousinVertex, label);
 447
 448                                 if (e == null) {
 449                                         addEdges.add(new Triplet<>(v, cousinVertex, label));
 450                                 } else {
 451                                         existingEdges.remove(e);
 452                                 }
 453                         }
 454                 }
 455
 456                 for (Edge edge : existingEdges) {
 457                         edge.remove();
 458                 }
 459                 for (Triplet<Vertex, Vertex, String> triplet : addEdges) {
 460                          try {
 461                                 edgeRules.addEdge(this.engine.asAdmin().getTraversalSource(), triplet.getValue0(), triplet.getValue1(), triplet.getValue2());
 462                         } catch (NoEdgeRuleFoundException e) {
 463                                 throw new AAIException("AAI_6129", e);
 464                         }
 465                 }
 466
 467         }
 468
 469         /**
 470          * Write through defaults.
 471          *
 472          * @param v the v
 473          * @param obj the obj
 474          * @throws AAIUnknownObjectException
 475          */
 476         private void writeThroughDefaults(Vertex v, Introspector obj) throws AAIUnknownObjectException {
 477                 Introspector latest = this.latestLoader.introspectorFromName(obj.getName());
 478                 if (latest != null) {
 479                         Set<String> required  = latest.getRequiredProperties();
 480
 481                         for (String field : required) {
 482                                 String defaultValue = null;
 483                                 Object vertexProp = null;
 484                                 defaultValue = latest.getPropertyMetadata(field).get(PropertyMetadata.DEFAULT_VALUE);
 485                                 if (defaultValue != null) {
 486                                         vertexProp = v.<Object>property(field).orElse(null);
 487                                         if (vertexProp == null) {
 488                                                 v.property(field, defaultValue);
 489                                         }
 490                                 }
 491                         }
 492                 }
 493
 494         }
 495
 496
 497         /**
 498           * Reflect dependent vertex.
 499           *
 500           * @param v the v
 501           * @param dependentObj the dependent obj
 502           * @return the vertex
 503           * @throws IllegalAccessException the illegal access exception
 504           * @throws IllegalArgumentException the illegal argument exception
 505           * @throws InvocationTargetException the invocation target exception
 506           * @throws InstantiationException the instantiation exception
 507           * @throws NoSuchMethodException the no such method exception
 508           * @throws SecurityException the security exception
 509           * @throws AAIException the AAI exception
 510           * @throws UnsupportedEncodingException the unsupported encoding exception
 511          * @throws AAIUnknownObjectException
 512           */
 513          private Vertex reflectDependentVertex(Vertex v, Introspector dependentObj, String requestContext) throws AAIException, UnsupportedEncodingException {
 514
 515                 //QueryParser p = this.engine.getQueryBuilder().createQueryFromURI(obj.getURI());
 516                 //List<Vertex> items = p.getQuery().toList();
 517                 QueryBuilder<Vertex> query = this.engine.getQueryBuilder(v);
 518                 query.createEdgeTraversal(EdgeType.TREE, v, dependentObj);
 519                 query.createKeyQuery(dependentObj);
 520
 521                 List<Vertex> items = query.toList();
 522
 523                 Vertex dependentVertex = null;
 524                 if (items.size() == 1) {
 525                         dependentVertex = items.get(0);
 526                         this.verifyResourceVersion("update", dependentObj.getDbName(), dependentVertex.<String>property(AAIProperties.RESOURCE_VERSION).orElse(null), (String)dependentObj.getValue(AAIProperties.RESOURCE_VERSION), (String)dependentObj.getURI());
 527                 } else {
 528                         this.verifyResourceVersion("create", dependentObj.getDbName(), "", (String)dependentObj.getValue(AAIProperties.RESOURCE_VERSION), (String)dependentObj.getURI());
 529                         dependentVertex = createNewVertex(dependentObj);
 530                 }
 531
 532                 return reflectDependentVertex(v, dependentVertex, dependentObj, requestContext);
 533
 534         }
 535
 536         /**
 537           * Reflect dependent vertex.
 538           *
 539           * @param parent the parent
 540           * @param child the child
 541           * @param obj the obj
 542           * @return the vertex
 543           * @throws IllegalAccessException the illegal access exception
 544           * @throws IllegalArgumentException the illegal argument exception
 545           * @throws InvocationTargetException the invocation target exception
 546           * @throws InstantiationException the instantiation exception
 547           * @throws NoSuchMethodException the no such method exception
 548           * @throws SecurityException the security exception
 549           * @throws AAIException the AAI exception
 550           * @throws UnsupportedEncodingException the unsupported encoding exception
 551          * @throws AAIUnknownObjectException
 552           */
 553          private Vertex reflectDependentVertex(Vertex parent, Vertex child, Introspector obj, String requestContext) throws AAIException, UnsupportedEncodingException {
 554
 555                 String parentUri = parent.<String>property(AAIProperties.AAI_URI).orElse(null);
 556                 if (parentUri != null) {
 557                         String uri;
 558                         uri = obj.getURI();
 559                         child.property(AAIProperties.AAI_URI, parentUri + uri);
 560                 }
 561                 processObject(obj, child, requestContext);
 562
 563                 Edge e;
 564                 e = this.getEdgeBetween(EdgeType.TREE, parent, child, null);
 565                 if (e == null) {
 566                         String canBeLinked = obj.getMetadata(ObjectMetadata.CAN_BE_LINKED);
 567                         if (canBeLinked != null && canBeLinked.equals("true")) {
 568                                 Loader ldrForCntxt = LoaderFactory.createLoaderForVersion(introspectionType, getVerForContext(requestContext));
 569                                 boolean isFirst = !this.engine.getQueryBuilder(ldrForCntxt, parent).createEdgeTraversal(EdgeType.TREE, parent, obj).hasNext();
 570                                 if (isFirst) {
 571                                         child.property(AAIProperties.LINKED, true);
 572                                 }
 573                         }
 574                         edgeRules.addTreeEdge(this.engine.asAdmin().getTraversalSource(), parent, child);
 575                 }
 576                 return child;
 577
 578         }
 579
 580         private Version getVerForContext(String requestContext) {
 581                 Pattern pattern = Pattern.compile("v[0-9]+");
 582                 Matcher m = pattern.matcher(requestContext);
 583                 if (!m.find()) {
 584                         return this.version;
 585                 } else {
 586                         return Version.valueOf(requestContext);
 587                 }
 588         }
 589
 590         /**
 591           * Db to object.
 592           *
 593           * @param vertices the vertices
 594           * @param obj the obj
 595           * @param depth the depth
 596           * @param cleanUp the clean up
 597           * @return the introspector
 598           * @throws AAIException the AAI exception
 599           * @throws IllegalAccessException the illegal access exception
 600           * @throws IllegalArgumentException the illegal argument exception
 601           * @throws InvocationTargetException the invocation target exception
 602           * @throws SecurityException the security exception
 603           * @throws InstantiationException the instantiation exception
 604           * @throws NoSuchMethodException the no such method exception
 605           * @throws UnsupportedEncodingException the unsupported encoding exception
 606           * @throws MalformedURLException the malformed URL exception
 607          * @throws AAIUnknownObjectException
 608          * @throws URISyntaxException
 609           */
 610          public Introspector dbToObject(List<Vertex> vertices, final Introspector obj, int depth, boolean nodeOnly, String cleanUp) throws UnsupportedEncodingException, AAIException {
 611                 final int internalDepth;
 612                 if (depth == Integer.MAX_VALUE) {
 613                         internalDepth = depth--;
 614                 } else {
 615                         internalDepth = depth;
 616                 }
 617                 StopWatch.conditionalStart();
 618                 if (vertices.size() > 1 && !obj.isContainer()) {
 619                         dbTimeMsecs += StopWatch.stopIfStarted();
 620                         throw new AAIException("AAI_6136", "query object mismatch: this object cannot hold multiple items." + obj.getDbName());
 621                 } else if (obj.isContainer()) {
 622                         final List getList;
 623                         String listProperty = null;
 624                         for (String property : obj.getProperties()) {
 625                                 if (obj.isListType(property) && obj.isComplexGenericType(property)) {
 626                                         listProperty = property;
 627                                         break;
 628                                 }
 629                         }
 630                         final String propertyName = listProperty;
 631                         getList = (List)obj.getValue(listProperty);
 632
 633                         /* This is an experimental multithreading experiment
 634                          * on get alls.
 635                          */
 636                         ExecutorService pool = GetAllPool.getInstance().getPool();
 637
 638                         List<Future<Object>> futures = new ArrayList<>();
 639
 640
 641                         for (Vertex v : vertices) {
 642                                 Callable<Object> task = () -> {
 643                                         Set<Vertex> seen = new HashSet<>();
 644                                         Introspector childObject = obj.newIntrospectorInstanceOfNestedProperty(propertyName);
 645                                         Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(v, internalDepth, nodeOnly);
 646                                         TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree);
 647                                         dbToObject(childObject, treeVertex, seen, internalDepth, nodeOnly, cleanUp);
 648                                         return childObject.getUnderlyingObject();
 649                                         //getList.add(childObject.getUnderlyingObject());
 650                                 };
 651                                 futures.add(pool.submit(task));
 652                         }
 653
 654                         for (Future<Object> future : futures) {
 655                                 try {
 656                                         getList.add(future.get());
 657                                 } catch (ExecutionException e) {
 658                                         dbTimeMsecs += StopWatch.stopIfStarted();
 659                                         throw new AAIException("AAI_4000", e);
 660                                 } catch (InterruptedException e) {
 661                                         dbTimeMsecs += StopWatch.stopIfStarted();
 662                                         throw new AAIException("AAI_4000", e);
 663                                 }
 664                         }
 665                 } else if (vertices.size() == 1) {
 666                         Set<Vertex> seen = new HashSet<>();
 667                         Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(vertices.get(0), depth, nodeOnly);
 668                         TreeBackedVertex treeVertex = new TreeBackedVertex(vertices.get(0), tree);
 669                         dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp);
 670                 } else {
 671                         //obj = null;
 672                 }
 673
 674                 dbTimeMsecs += StopWatch.stopIfStarted();
 675                 return obj;
 676         }
 677
 678         /**
 679          * Db to object.
 680          *
 681          * @param obj the obj
 682          * @param v the v
 683          * @param seen the seen
 684          * @param depth the depth
 685          * @param cleanUp the clean up
 686          * @return the introspector
 687          * @throws IllegalAccessException the illegal access exception
 688          * @throws IllegalArgumentException the illegal argument exception
 689          * @throws InvocationTargetException the invocation target exception
 690          * @throws SecurityException the security exception
 691          * @throws InstantiationException the instantiation exception
 692          * @throws NoSuchMethodException the no such method exception
 693          * @throws UnsupportedEncodingException the unsupported encoding exception
 694          * @throws AAIException the AAI exception
 695          * @throws MalformedURLException the malformed URL exception
 696          * @throws AAIUnknownObjectException
 697          * @throws URISyntaxException
 698          */
 699         private Introspector dbToObject(Introspector obj, Vertex v, Set<Vertex> seen, int depth, boolean nodeOnly, String cleanUp) throws AAIException, UnsupportedEncodingException {
 700
 701                 if (depth < 0) {
 702                         return null;
 703                 }
 704                 depth--;
 705                 seen.add(v);
 706
 707                 boolean modified = false;
 708                 for (String property : obj.getProperties(PropertyPredicates.isVisible())) {
 709                         List<Object> getList = null;
 710                         Vertex[] vertices = null;
 711
 712                         if (!(obj.isComplexType(property) || obj.isListType(property))) {
 713                                 this.copySimpleProperty(property, obj, v);
 714                                 modified = true;
 715                         } else {
 716                                 if (obj.isComplexType(property)) {
 717                                 /* container case */
 718
 719                                         if (!property.equals("relationship-list") && depth >= 0) {
 720                                                 Introspector argumentObject = obj.newIntrospectorInstanceOfProperty(property);
 721                                                 Object result  = dbToObject(argumentObject, v, seen, depth+1, nodeOnly, cleanUp);
 722                                                 if (result != null) {
 723                                                         obj.setValue(property, argumentObject.getUnderlyingObject());
 724                                                         modified = true;
 725                                                 }
 726                                         } else if (property.equals("relationship-list") && !nodeOnly){
 727                                                 /* relationships need to be handled correctly */
 728                                                 Introspector relationshipList = obj.newIntrospectorInstanceOfProperty(property);
 729                                                 relationshipList = createRelationshipList(v, relationshipList, cleanUp);
 730                                                 if (relationshipList != null) {
 731                                                         modified = true;
 732                                                         obj.setValue(property, relationshipList.getUnderlyingObject());
 733                                                         modified = true;
 734                                                 }
 735
 736                                         }
 737                                 } else if (obj.isListType(property)) {
 738
 739                                         if (property.equals("any")) {
 740                                                 continue;
 741                                         }
 742                                         String genericType = obj.getGenericTypeClass(property).getSimpleName();
 743                                         if (obj.isComplexGenericType(property) && depth >= 0) {
 744                                                 final String childDbName = convertFromCamelCase(genericType);
 745                                                 String vType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null);
 746                                                 EdgeRule rule;
 747
 748                                                 rule = edgeRules.getEdgeRule(EdgeType.TREE, vType, childDbName);
 749                                                 if (!rule.getContains().equals(AAIDirection.NONE.toString())) {
 750                                                         //vertices = this.queryEngine.findRelatedVertices(v, Direction.OUT, rule.getLabel(), childDbName);
 751                                                         Direction ruleDirection = rule.getDirection();
 752                                                         Iterator<Vertex> itr = v.vertices(ruleDirection, rule.getLabel());
 753                                                         List<Vertex> verticesList = (List<Vertex>)IteratorUtils.toList(itr);
 754                                                         itr = verticesList.stream().filter(item -> {
 755                                                                 return item.property(AAIProperties.NODE_TYPE).orElse("").equals(childDbName);
 756                                                         }).iterator();
 757                                                         if (itr.hasNext()) {
 758                                                                 getList = (List<Object>)obj.getValue(property);
 759                                                         }
 760                                                         int processed = 0;
 761                                                         int removed = 0;
 762                                                         while (itr.hasNext()) {
 763                                                                 Vertex childVertex = itr.next();
 764                                                                 if (!seen.contains(childVertex)) {
 765                                                                         Introspector argumentObject = obj.newIntrospectorInstanceOfNestedProperty(property);
 766
 767                                                                         Object result = dbToObject(argumentObject, childVertex, seen, depth, nodeOnly, cleanUp);
 768                                                                         if (result != null) {
 769                                                                                 getList.add(argumentObject.getUnderlyingObject());
 770                                                                         }
 771
 772                                                                         processed++;
 773                                                                 } else {
 774                                                                         removed++;
 775                                                                         LOGGER.warn("Cycle found while serializing vertex id={}", childVertex.id().toString());
 776                                                                 }
 777                                                         }
 778                                                         if (processed == 0) {
 779                                                                 //vertices were all seen, reset the list
 780                                                                 getList = null;
 781                                                         }
 782                                                         if (processed > 0) {
 783                                                                 modified = true;
 784                                                         }
 785                                                 }
 786                                         } else if (obj.isSimpleGenericType(property)) {
 787                                                 List<Object> temp = this.engine.getListProperty(v, property);
 788                                                 if (temp != null) {
 789                                                         getList = (List<Object>)obj.getValue(property);
 790                                                         getList.addAll(temp);
 791                                                         modified = true;
 792                                                 }
 793
 794                                         }
 795
 796                                 }
 797
 798                         }
 799                 }
 800
 801                 //no changes were made to this obj, discard the instance
 802                 if (!modified) {
 803                         return null;
 804                 }
 805                 this.enrichData(obj, v);
 806                 return obj;
 807
 808         }
 809
 810
 811         public Introspector getVertexProperties(Vertex v) throws AAIException, UnsupportedEncodingException {
 812                 String nodeType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null);
 813                 if (nodeType == null) {
 814                         throw new AAIException("AAI_6143");
 815                 }
 816
 817                 Introspector obj = this.latestLoader.introspectorFromName(nodeType);
 818                 Set<Vertex> seen = new HashSet<>();
 819                 int depth = 0;
 820                 String cleanUp = "false";
 821                 boolean nodeOnly = true;
 822                 StopWatch.conditionalStart();
 823                 this.dbToObject(obj, v, seen, depth, nodeOnly, cleanUp);
 824                 dbTimeMsecs += StopWatch.stopIfStarted();
 825                 return obj;
 826
 827         }
 828         public Introspector getLatestVersionView(Vertex v) throws AAIException, UnsupportedEncodingException {
 829                 String nodeType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null);
 830                 if (nodeType == null) {
 831                         throw new AAIException("AAI_6143");
 832                 }
 833                 Introspector obj = this.latestLoader.introspectorFromName(nodeType);
 834                 Set<Vertex> seen = new HashSet<>();
 835                 int depth = AAIProperties.MAXIMUM_DEPTH;
 836                 String cleanUp = "false";
 837                 boolean nodeOnly = false;
 838                 StopWatch.conditionalStart();
 839                 Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(v, depth, nodeOnly);
 840                 TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree);
 841                 this.dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp);
 842                 dbTimeMsecs += StopWatch.stopIfStarted();
 843                 return obj;
 844         }
 845         /**
 846          * Copy simple property.
 847          *
 848          * @param property the property
 849          * @param obj the obj
 850          * @param v the v
 851          * @throws InstantiationException the instantiation exception
 852          * @throws IllegalAccessException the illegal access exception
 853          * @throws IllegalArgumentException the illegal argument exception
 854          * @throws InvocationTargetException the invocation target exception
 855          * @throws NoSuchMethodException the no such method exception
 856          * @throws SecurityException the security exception
 857          */
 858         private void copySimpleProperty(String property, Introspector obj, Vertex v) {
 859                 final Map<PropertyMetadata, String> metadata = obj.getPropertyMetadata(property);
 860                 String dbPropertyName = property;
 861                 if (metadata.containsKey(PropertyMetadata.DB_ALIAS)) {
 862                         dbPropertyName = metadata.get(PropertyMetadata.DB_ALIAS);
 863                 }
 864                 final Object temp = v.<Object>property(dbPropertyName).orElse(null);
 865                 if (temp != null) {
 866
 867                         obj.setValue(property, temp);
 868                 }
 869         }
 870
 871         /**
 872          * Simple db to object.
 873          *
 874          * @param obj the obj
 875          * @param v the v
 876          * @throws InstantiationException the instantiation exception
 877          * @throws IllegalAccessException the illegal access exception
 878          * @throws IllegalArgumentException the illegal argument exception
 879          * @throws InvocationTargetException the invocation target exception
 880          * @throws NoSuchMethodException the no such method exception
 881          * @throws SecurityException the security exception
 882          */
 883         private void simpleDbToObject (Introspector obj, Vertex v) {
 884                 for (String property : obj.getProperties()) {
 885
 886
 887                         if (!(obj.isComplexType(property) || obj.isListType(property))) {
 888                                 this.copySimpleProperty(property, obj, v);
 889                         }
 890                 }
 891         }
 892
 893         /**
 894          * Creates the relationship list.
 895          *
 896          * @param v the v
 897          * @param obj the obj
 898          * @param cleanUp the clean up
 899          * @return the object
 900          * @throws InstantiationException the instantiation exception
 901          * @throws IllegalAccessException the illegal access exception
 902          * @throws IllegalArgumentException the illegal argument exception
 903          * @throws InvocationTargetException the invocation target exception
 904          * @throws NoSuchMethodException the no such method exception
 905          * @throws SecurityException the security exception
 906          * @throws UnsupportedEncodingException the unsupported encoding exception
 907          * @throws AAIException the AAI exception
 908          * @throws MalformedURLException the malformed URL exception
 909          * @throws URISyntaxException
 910          */
 911         private Introspector createRelationshipList(Vertex v, Introspector obj, String cleanUp) throws UnsupportedEncodingException, AAIException {
 912
 913                 List<Vertex> cousins = this.engine.getQueryEngine().findCousinVertices(v);
 914
 915                 List<Object> relationshipObjList = obj.getValue("relationship");
 916
 917                 for (Vertex cousin : cousins) {
 918                         if (VersionChecker.apiVersionNeedsEdgeLabel(obj.getVersion())) {
 919                                 List<Edge> edges = this.getEdgesBetween(EdgeType.COUSIN, v, cousin);
 920                                 for (Edge e : edges) {
 921                                         Introspector relationshipObj = obj.newIntrospectorInstanceOfNestedProperty("relationship");
 922                                         Object result = processEdgeRelationship(relationshipObj, cousin, cleanUp, e);
 923                                         if (result != null) {
 924                                                 relationshipObjList.add(result);
 925                                         }
 926                                 }
 927                         } else {
 928                                 Introspector relationshipObj = obj.newIntrospectorInstanceOfNestedProperty("relationship");
 929                                 Object result = processEdgeRelationship(relationshipObj, cousin, cleanUp, null);
 930                                 if (result != null) {
 931                                         relationshipObjList.add(result);
 932                                 }
 933                         }
 934
 935                 }
 936
 937                 if (relationshipObjList.isEmpty()) {
 938                         return null;
 939                 } else {
 940                         return obj;
 941                 }
 942         }
 943
 944         /**
 945          * Process edge relationship.
 946          *
 947          * @param relationshipObj the relationship obj
 948          * @param edge the edge
 949          * @param direction the direction
 950          * @param cleanUp the clean up
 951          * @return the object
 952          * @throws InstantiationException the instantiation exception
 953          * @throws IllegalAccessException the illegal access exception
 954          * @throws IllegalArgumentException the illegal argument exception
 955          * @throws InvocationTargetException the invocation target exception
 956          * @throws NoSuchMethodException the no such method exception
 957          * @throws SecurityException the security exception
 958          * @throws UnsupportedEncodingException the unsupported encoding exception
 959          * @throws AAIException the AAI exception
 960          * @throws MalformedURLException the malformed URL exception
 961          * @throws AAIUnknownObjectException
 962          * @throws URISyntaxException
 963          */
 964         private Object processEdgeRelationship(Introspector relationshipObj, Vertex cousin, String cleanUp, Edge edge) throws UnsupportedEncodingException, AAIUnknownObjectException {
 965
 966
 967                 //we must look up all parents in this case because we need to compute name-properties
 968                 //we cannot used the cached aaiUri to perform this action currently
 969                 Optional<Pair<Vertex, List<Introspector>>> tuple = this.getParents(relationshipObj.getLoader(), cousin, "true".equals(cleanUp));
 970                 //damaged vertex found, ignore
 971                 if (!tuple.isPresent()) {
 972                         return null;
 973                 }
 974                 List<Introspector> list = tuple.get().getValue1();
 975                 URI uri = this.getURIFromList(list);
 976
 977                 URIToRelationshipObject uriParser = null;
 978                 Introspector result = null;
 979                 try {
 980                         uriParser = new URIToRelationshipObject(relationshipObj.getLoader(), uri, this.baseURL);
 981                         result = uriParser.getResult();
 982                 } catch (AAIException | URISyntaxException e) {
 983                         LOGGER.error("Error while processing edge relationship in version " + relationshipObj.getVersion() + " (bad vertex ID=" + tuple.get().getValue0().id().toString() + ": "
 984                                         + e.getMessage() + " " + LogFormatTools.getStackTop(e));
 985                         if ("true".equals(cleanUp)) {
 986                                 this.deleteWithTraversal(tuple.get().getValue0());
 987                         }
 988                         return null;
 989                 }
 990                 if (!list.isEmpty()) {
 991                         this.addRelatedToProperty(result, list.get(0));
 992                 }
 993
 994                 if (edge != null && result.hasProperty("relationship-label")) {
 995                         result.setValue("relationship-label", edge.label());
 996                 }
 997
 998                 return result.getUnderlyingObject();
 999         }
1000
1001         /**
1002          * Gets the URI for vertex.
1003          *
1004          * @param v the v
1005          * @return the URI for vertex
1006          * @throws InstantiationException the instantiation exception
1007          * @throws IllegalAccessException the illegal access exception
1008          * @throws IllegalArgumentException the illegal argument exception
1009          * @throws InvocationTargetException the invocation target exception
1010          * @throws NoSuchMethodException the no such method exception
1011          * @throws SecurityException the security exception
1012          * @throws UnsupportedEncodingException the unsupported encoding exception
1013          * @throws AAIUnknownObjectException
1014          */
1015         public URI getURIForVertex(Vertex v) throws UnsupportedEncodingException {
1016
1017                 return getURIForVertex(v, false);
1018         }
1019
1020         public URI getURIForVertex(Vertex v, boolean overwrite) throws UnsupportedEncodingException {
1021                 URI uri = UriBuilder.fromPath("/unknown-uri").build();
1022
1023                 String aaiUri = v.<String>property(AAIProperties.AAI_URI).orElse(null);
1024
1025                 if (aaiUri != null && !overwrite) {
1026                         uri = UriBuilder.fromPath(aaiUri).build();
1027                 } else {
1028                         StopWatch.conditionalStart();
1029                         Optional<Pair<Vertex, List<Introspector>>> tuple = this.getParents(this.loader, v, false);
1030                         dbTimeMsecs += StopWatch.stopIfStarted();
1031                         if (tuple.isPresent()) {
1032                                 List<Introspector> list = tuple.get().getValue1();
1033                                 uri = this.getURIFromList(list);
1034                         }
1035
1036
1037                 }
1038                 return uri;
1039         }
1040         /**
1041          * Gets the URI from list.
1042          *
1043          * @param list the list
1044          * @return the URI from list
1045          * @throws UnsupportedEncodingException the unsupported encoding exception
1046          */
1047         private URI getURIFromList(List<Introspector> list) throws UnsupportedEncodingException {
1048                 String uri = "";
1049                 StringBuilder sb = new StringBuilder();
1050                 for (Introspector i : list) {
1051                         sb.insert(0, i.getURI());
1052                 }
1053
1054                 uri = sb.toString();
1055                 return UriBuilder.fromPath(uri).build();
1056         }
1057
1058         /**
1059          * Gets the parents.
1060          *
1061          * @param start the start
1062          * @param removeDamaged the remove damaged
1063          * @return the parents
1064          * @throws InstantiationException the instantiation exception
1065          * @throws IllegalAccessException the illegal access exception
1066          * @throws IllegalArgumentException the illegal argument exception
1067          * @throws InvocationTargetException the invocation target exception
1068          * @throws NoSuchMethodException the no such method exception
1069          * @throws SecurityException the security exception
1070          * @throws AAIUnknownObjectException
1071          */
1072         private Optional<Pair<Vertex, List<Introspector>>> getParents(Loader loader, Vertex start, boolean removeDamaged) {
1073
1074                 List<Vertex> results = this.engine.getQueryEngine().findParents(start);
1075                 List<Introspector> objs = new ArrayList<>();
1076                 boolean shortCircuit = false;
1077                 for (Vertex v : results) {
1078                         String nodeType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null);
1079                         Introspector obj = null;
1080                         //vertex on the other end of this edge is bad
1081                         if (nodeType == null) {
1082                                 //log something here about what was found and that it was removed
1083                                 ErrorLogHelper.logError("AAI-6143", "Found a damaged parent vertex " + v.id().toString());
1084                                 if (removeDamaged) {
1085                                         this.deleteWithTraversal(v);
1086                                 }
1087                                 shortCircuit = true;
1088                         } else {
1089                                 try {
1090                                         obj = loader.introspectorFromName(nodeType);
1091                                 } catch (AAIUnknownObjectException e) {
1092                                         LOGGER.info("attempted to create node type " + nodeType + " but we do not understand it for version: " + loader.getVersion());
1093                                         obj = null;
1094                                 }
1095                         }
1096
1097                         if (obj == null) {
1098                                 //can't make a valid path because we don't understand this object
1099                                 // don't include it
1100                         } else {
1101                                 this.simpleDbToObject(obj, v);
1102                                 objs.add(obj);
1103                         }
1104                 }
1105
1106                 //stop processing and don't return anything for this bad vertex
1107                 if (shortCircuit) {
1108                         return Optional.empty();
1109                 }
1110
1111                 return Optional.of(new Pair<>(results.get(results.size()-1), objs));
1112         }
1113         /**
1114          * Takes a list of vertices and a list of objs and assumes they are in
1115          * the order you want the URIs to be nested.
1116          * [A,B,C] creates uris [A, AB, ABC]
1117          * @param vertices
1118          * @param objs
1119          * @throws UnsupportedEncodingException
1120          * @throws URISyntaxException
1121          */
1122         public void setCachedURIs(List<Vertex> vertices, List<Introspector> objs) throws UnsupportedEncodingException, URISyntaxException {
1123
1124                 StringBuilder uriChain = new StringBuilder();
1125                 for (int i = 0; i < vertices.size(); i++) {
1126                         String aaiUri = "";
1127                         Vertex v = null;
1128                         v = vertices.get(i);
1129                         aaiUri = v.<String>property(AAIProperties.AAI_URI).orElse(null);
1130                         if (aaiUri != null) {
1131                                 uriChain.append(aaiUri);
1132                         } else {
1133                                 URI uri = UriBuilder.fromPath(objs.get(i).getURI()).build();
1134                                 aaiUri = uri.toString();
1135                                 uriChain.append(aaiUri);
1136                                 v.property(AAIProperties.AAI_URI, uriChain.toString());
1137                         }
1138                 }
1139
1140
1141
1142         }
1143
1144
1145         /**
1146          * Adds the r
1147          * @throws AAIUnknownObjectException
1148          * @throws IllegalArgumentException elated to property.
1149          *
1150          * @param relationship the relationship
1151          * @param child the throws IllegalArgumentException, AAIUnknownObjectException child
1152          */
1153         public void addRelatedToProperty(Introspector relationship, Introspector child) throws AAIUnknownObjectException {
1154                 String nameProps = child.getMetadata(ObjectMetadata.NAME_PROPS);
1155                 List<Introspector> relatedToProperties = new ArrayList<>();
1156
1157                 if (nameProps != null) {
1158                         String[] props = nameProps.split(",");
1159                         for (String prop : props) {
1160                                 Introspector relatedTo = relationship.newIntrospectorInstanceOfNestedProperty("related-to-property");
1161                                 relatedTo.setValue("property-key", child.getDbName() + "." + prop);
1162                                 relatedTo.setValue("property-value", child.getValue(prop));
1163                                 relatedToProperties.add(relatedTo);
1164                         }
1165                 }
1166
1167                 if (!relatedToProperties.isEmpty()) {
1168                         List relatedToList = (List)relationship.getValue("related-to-property");
1169                         for (Introspector obj : relatedToProperties) {
1170                                 relatedToList.add(obj.getUnderlyingObject());
1171                         }
1172                 }
1173
1174         }
1175
1176         /**
1177          * Creates the edge.
1178          *
1179          * @param relationship the relationship
1180          * @param inputVertex the input vertex
1181          * @return true, if successful
1182          * @throws UnsupportedEncodingException the unsupported encoding exception
1183          * @throws AAIException the AAI exception
1184          */
1185         public boolean createEdge(Introspector relationship, Vertex inputVertex) throws UnsupportedEncodingException, AAIException {
1186
1187                 Vertex relatedVertex = null;
1188                 StopWatch.conditionalStart();
1189                 QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(relationship);
1190
1191                 String label = null;
1192                 if (relationship.hasProperty("relationship-label")) {
1193                         label = relationship.getValue("relationship-label");
1194                 }
1195
1196                 List<Vertex> results = parser.getQueryBuilder().toList();
1197                 if (results.isEmpty()) {
1198                         dbTimeMsecs += StopWatch.stopIfStarted();
1199                         AAIException e = new AAIException("AAI_6129", "Node of type " + parser.getResultType() + ". Could not find object at: " + parser.getUri());
1200                         e.getTemplateVars().add(parser.getResultType());
1201                         e.getTemplateVars().add(parser.getUri().toString());
1202                         throw e;
1203                 } else {
1204                         //still an issue if there's more than one
1205                         relatedVertex = results.get(0);
1206                 }
1207
1208                 if (relatedVertex != null) {
1209
1210                         Edge e;
1211                         try {
1212                                 e = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label);
1213                                 if (e == null) {
1214                                         edgeRules.addEdge(this.engine.asAdmin().getTraversalSource(), inputVertex, relatedVertex, label);
1215                                 } else {
1216                                         //attempted to link two vertexes already linked
1217                                 }
1218                         } finally {
1219                                 dbTimeMsecs += StopWatch.stopIfStarted();
1220                         }
1221                 }
1222
1223                 dbTimeMsecs += StopWatch.stopIfStarted();
1224                 return true;
1225         }
1226
1227         /**
1228          * Gets all the edges between of the type.
1229          *
1230          * @param aVertex the out vertex
1231          * @param bVertex the in vertex
1232          * @return the edges between
1233          * @throws AAIException the AAI exception
1234          * @throws NoEdgeRuleFoundException
1235          */
1236         private List<Edge> getEdgesBetween(EdgeType type, Vertex aVertex, Vertex bVertex) {
1237
1238                 List<Edge> result = new ArrayList<>();
1239
1240                 if (bVertex != null) {
1241                         GraphTraversal<Vertex, Edge> findEdgesBetween = null;
1242                         findEdgesBetween = this.engine.asAdmin().getTraversalSource().V(aVertex).bothE();
1243                         if (EdgeType.TREE.equals(type)) {
1244                                 findEdgesBetween = findEdgesBetween.not(__.has(EdgeProperty.CONTAINS.toString(), "NONE"));
1245                         } else {
1246                                 findEdgesBetween = findEdgesBetween.has(EdgeProperty.CONTAINS.toString(), "NONE");
1247                         }
1248                         findEdgesBetween = findEdgesBetween.filter(__.otherV().hasId(bVertex.id()));
1249                         result = findEdgesBetween.toList();
1250                 }
1251
1252                 return result;
1253         }
1254         /**
1255          * Gets all the edges between the vertexes with the label and type.
1256          *
1257          * @param aVertex the out vertex
1258          * @param bVertex the in vertex
1259          * @param label
1260          * @return the edges between
1261          * @throws AAIException the AAI exception
1262          */
1263         private List<Edge> getEdgesBetween(EdgeType type, Vertex aVertex, Vertex bVertex, String label) throws AAIException {
1264
1265                 List<Edge> result = new ArrayList<>();
1266
1267                 if (bVertex != null) {
1268                         EdgeRule rule = edgeRules.getEdgeRule(type, aVertex, bVertex, label);
1269                         List<Edge> edges = this.getEdgesBetween(type, aVertex, bVertex);
1270                         for (Edge edge : edges) {
1271                                 if (edge.label().equals(rule.getLabel())) {
1272                                         result.add(edge);
1273                                 }
1274                         }
1275                 }
1276
1277                 return result;
1278         }
1279
1280         /**
1281          * Gets the edge between with the label and edge type.
1282          *
1283          * @param aVertex the out vertex
1284          * @param bVertex the in vertex
1285          * @param label
1286          * @return the edge between
1287          * @throws AAIException the AAI exception
1288          * @throws NoEdgeRuleFoundException
1289          */
1290         public Edge getEdgeBetween(EdgeType type, Vertex aVertex, Vertex bVertex, String label) throws AAIException {
1291
1292                 StopWatch.conditionalStart();
1293                 if (bVertex != null) {
1294
1295                                 List<Edge> edges = this.getEdgesBetween(type, aVertex, bVertex, label);
1296
1297                                 if (!edges.isEmpty()) {
1298                                         dbTimeMsecs += StopWatch.stopIfStarted();
1299                                         return edges.get(0);
1300                                 }
1301
1302                 }
1303                 dbTimeMsecs += StopWatch.stopIfStarted();
1304                 return null;
1305         }
1306         public Edge getEdgeBetween(EdgeType type, Vertex aVertex, Vertex bVertex) throws AAIException {
1307                 return this.getEdgeBetween(type, aVertex, bVertex, null);
1308         }
1309
1310
1311         /**
1312          * Delete edge.
1313          *
1314          * @param relationship the relationship
1315          * @param inputVertex the input vertex
1316          * @return true, if successful
1317          * @throws UnsupportedEncodingException the unsupported encoding exception
1318          * @throws AAIException the AAI exception
1319          */
1320         public boolean deleteEdge(Introspector relationship, Vertex inputVertex) throws UnsupportedEncodingException, AAIException {
1321
1322                 Vertex relatedVertex = null;
1323                 StopWatch.conditionalStart();
1324                 QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(relationship);
1325
1326                 List<Vertex> results = parser.getQueryBuilder().toList();
1327
1328                 String label = null;
1329                 if (relationship.hasProperty("relationship-label")) {
1330                         label = relationship.getValue("relationship-label");
1331                 }
1332
1333                 if (results.isEmpty()) {
1334                         dbTimeMsecs += StopWatch.stopIfStarted();
1335                         return false;
1336                 }
1337
1338                 relatedVertex = results.get(0);
1339                 Edge edge;
1340                 try {
1341                         edge = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label);
1342                 } catch (NoEdgeRuleFoundException e) {
1343                         dbTimeMsecs += StopWatch.stopIfStarted();
1344                         throw new AAIException("AAI_6129", e);
1345                 }
1346                 if (edge != null) {
1347                         edge.remove();
1348                         dbTimeMsecs += StopWatch.stopIfStarted();
1349                         return true;
1350                 } else {
1351                         dbTimeMsecs += StopWatch.stopIfStarted();
1352                         return false;
1353                 }
1354
1355         }
1356
1357         /**
1358          * Delete items with traversal.
1359          *
1360          * @param vertexes the vertexes
1361          * @throws IllegalStateException the illegal state exception
1362          */
1363         public void deleteItemsWithTraversal(List<Vertex> vertexes) throws IllegalStateException {
1364
1365                 for (Vertex v : vertexes) {
1366             LOGGER.debug("About to delete the vertex with id: " + v.id());
1367                         deleteWithTraversal(v);
1368                 }
1369
1370         }
1371
1372         /**
1373          * Delete with traversal.
1374          *
1375          * @param startVertex the start vertex
1376          */
1377         public void deleteWithTraversal(Vertex startVertex) {
1378                 StopWatch.conditionalStart();
1379                 List<Vertex> results = this.engine.getQueryEngine().findDeletable(startVertex);
1380
1381                 for (Vertex v : results) {
1382                         LOGGER.warn("Removing vertex " + v.id().toString());
1383
1384                         v.remove();
1385                 }
1386                 dbTimeMsecs += StopWatch.stopIfStarted();
1387         }
1388
1389         /**
1390          * Delete.
1391          *
1392          * @param v the v
1393          * @param resourceVersion the resource version
1394          * @throws IllegalArgumentException the illegal argument exception
1395          * @throws AAIException the AAI exception
1396          * @throws InterruptedException the interrupted exception
1397          */
1398         public void delete(Vertex v, String resourceVersion, boolean enableResourceVersion) throws IllegalArgumentException, AAIException {
1399
1400                 boolean result = verifyDeleteSemantics(v, resourceVersion, enableResourceVersion);
1401                 if (result) {
1402                         try {
1403                                 deleteWithTraversal(v);
1404                         } catch (IllegalStateException e) {
1405                                 throw new AAIException("AAI_6110", e);
1406                         }
1407
1408                 }
1409
1410         }
1411
1412
1413         /**
1414          * Verify delete semantics.
1415          *
1416          * @param vertex the vertex
1417          * @param resourceVersion the resource version
1418          * @return true, if successful
1419          * @throws AAIException the AAI exception
1420          */
1421         private boolean verifyDeleteSemantics(Vertex vertex, String resourceVersion, boolean enableResourceVersion) throws AAIException {
1422                 boolean result = true;
1423                 String nodeType = "";
1424                 String errorDetail = " unknown delete semantic found";
1425                 String aaiExceptionCode = "";
1426                 nodeType = vertex.<String>property(AAIProperties.NODE_TYPE).orElse(null);
1427                 if (enableResourceVersion && !this.verifyResourceVersion("delete", nodeType, vertex.<String>property(AAIProperties.RESOURCE_VERSION).orElse(null), resourceVersion, nodeType)) {
1428                 }
1429                 StopWatch.conditionalStart();
1430                 List<Object> preventDeleteVertices = this.engine.asAdmin().getReadOnlyTraversalSource().V(vertex).union(__.inE().has(EdgeProperty.PREVENT_DELETE.toString(), AAIDirection.IN.toString()).outV().values(AAIProperties.NODE_TYPE), __.outE().has(EdgeProperty.PREVENT_DELETE.toString(), AAIDirection.OUT.toString()).inV().values(AAIProperties.NODE_TYPE)).dedup().toList();
1431
1432                 dbTimeMsecs += StopWatch.stopIfStarted();
1433                 if (!preventDeleteVertices.isEmpty()) {
1434                         aaiExceptionCode = "AAI_6110";
1435                         errorDetail = String.format("Object is being reference by additional objects preventing it from being deleted. Please clean up references from the following types %s", preventDeleteVertices);
1436                         result = false;
1437                 }
1438                 if (!result) {
1439                         throw new AAIException(aaiExceptionCode, errorDetail);
1440                 }
1441                 return result;
1442         }
1443
1444         /**
1445          * Verify resource version.
1446          *
1447          * @param action the action
1448          * @param nodeType the node type
1449          * @param currentResourceVersion the current resource version
1450          * @param resourceVersion the resource version
1451          * @param uri the uri
1452          * @return true, if successful
1453          * @throws AAIException the AAI exception
1454          */
1455         public boolean verifyResourceVersion(String action, String nodeType, String currentResourceVersion, String resourceVersion, String uri) throws AAIException {
1456                 String enabled = "";
1457                 String errorDetail = "";
1458                 String aaiExceptionCode = "";
1459                 if (currentResourceVersion == null) {
1460                         currentResourceVersion = "";
1461                 }
1462
1463                 if (resourceVersion == null) {
1464                         resourceVersion = "";
1465                 }
1466                 try {
1467                         enabled = AAIConfig.get(AAIConstants.AAI_RESVERSION_ENABLEFLAG);
1468                 } catch (AAIException e) {
1469                         ErrorLogHelper.logException(e);
1470                 }
1471                 // We're only doing the resource version checks for v5 and later
1472                 if (enabled.equals("true")) {
1473                         if (!currentResourceVersion.equals(resourceVersion)) {
1474                                 if (action.equals("create") && !resourceVersion.equals("")) {
1475                                         errorDetail = "resource-version passed for " + action + " of " + uri;
1476                                         aaiExceptionCode = "AAI_6135";
1477                                 } else if (resourceVersion.equals("")) {
1478                                         errorDetail = "resource-version not passed for " + action + " of " + uri;
1479                                         aaiExceptionCode = "AAI_6130";
1480                                 } else {
1481                                         errorDetail = "resource-version MISMATCH for " + action + " of " + uri;
1482                                         aaiExceptionCode = "AAI_6131";
1483                                 }
1484
1485                                 throw new AAIException(aaiExceptionCode, errorDetail);
1486
1487                         }
1488                 }
1489                 return true;
1490         }
1491
1492         /**
1493          * Convert from camel case.
1494          *
1495          * @param name the name
1496          * @return the string
1497          */
1498         private String convertFromCamelCase (String name) {
1499                 String result = "";
1500                 result = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, name);
1501
1502                 NamingExceptions exceptions = NamingExceptions.getInstance();
1503                 result = exceptions.getDBName(result);
1504
1505                 return result;
1506         }
1507
1508         private boolean canModify(Introspector obj, String propName, String requestContext) {
1509                 final String readOnly = obj.getPropertyMetadata(propName).get(PropertyMetadata.READ_ONLY);
1510                 if (readOnly != null) {
1511                         final String[] items = readOnly.split(",");
1512                         for (String item : items) {
1513                                 if (requestContext.equals(item)) {
1514                                         return false;
1515                                 }
1516                         }
1517                 }
1518                 return true;
1519         }
1520
1521         private void executePreSideEffects(Introspector obj, Vertex self) throws AAIException {
1522
1523                 SideEffectRunner runner = new SideEffectRunner
1524                                 .Builder(this.engine, this).addSideEffect(DataCopy.class).build();
1525
1526                 runner.execute(obj, self);
1527         }
1528
1529         private void executePostSideEffects(Introspector obj, Vertex self) throws AAIException {
1530
1531                 SideEffectRunner runner = new SideEffectRunner
1532                                 .Builder(this.engine, this).addSideEffect(DataLinkWriter.class).build();
1533
1534                 runner.execute(obj, self);
1535         }
1536
1537         private void enrichData(Introspector obj, Vertex self) throws AAIException  {
1538
1539                 SideEffectRunner runner = new SideEffectRunner
1540                                 .Builder(this.engine, this).addSideEffect(DataLinkReader.class).build();
1541
1542                 runner.execute(obj, self);
1543         }
1544
1545         public double getDBTimeMsecs() {
1546                 return (dbTimeMsecs);
1547         }
1548
1549         /**
1550           * Db to object With Filters
1551           * This is for a one-time run with Tenant Isloation to only filter relationships
1552           * TODO: Chnage the original dbToObject to take filter parent/cousins
1553           *
1554           * @param vertices the vertices
1555           * @param obj the obj
1556           * @param depth the depth
1557           * @param cleanUp the clean up
1558           * @return the introspector
1559           * @throws AAIException the AAI exception
1560           * @throws IllegalAccessException the illegal access exception
1561           * @throws IllegalArgumentException the illegal argument exception
1562           * @throws InvocationTargetException the invocation target exception
1563           * @throws SecurityException the security exception
1564           * @throws InstantiationException the instantiation exception
1565           * @throws NoSuchMethodException the no such method exception
1566           * @throws UnsupportedEncodingException the unsupported encoding exception
1567           * @throws MalformedURLException the malformed URL exception
1568          * @throws AAIUnknownObjectException
1569          * @throws URISyntaxException
1570           */
1571          //TODO - See if you can merge the 2 dbToObjectWithFilters
1572          public Introspector dbToObjectWithFilters(Introspector obj, Vertex v, Set<Vertex> seen, int depth, boolean nodeOnly,  List<String> filterCousinNodes, List<String> filterParentNodes) throws AAIException, UnsupportedEncodingException {
1573                         String cleanUp = "false";
1574                         if (depth < 0) {
1575                                 return null;
1576                         }
1577                         depth--;
1578                         seen.add(v);
1579                         boolean modified = false;
1580                         for (String property : obj.getProperties(PropertyPredicates.isVisible())) {
1581                                 List<Object> getList = null;
1582                                 Vertex[] vertices = null;
1583
1584                                 if (!(obj.isComplexType(property) || obj.isListType(property))) {
1585                                         this.copySimpleProperty(property, obj, v);
1586                                         modified = true;
1587                                 } else {
1588                                         if (obj.isComplexType(property)) {
1589                                         /* container case */
1590
1591                                                 if (!property.equals("relationship-list") && depth >= 0) {
1592                                                         Introspector argumentObject = obj.newIntrospectorInstanceOfProperty(property);
1593                                                         Object result  = dbToObjectWithFilters(argumentObject, v, seen, depth+1, nodeOnly,  filterCousinNodes, filterParentNodes);
1594                                                         if (result != null) {
1595                                                                 obj.setValue(property, argumentObject.getUnderlyingObject());
1596                                                                 modified = true;
1597                                                         }
1598                                                 } else if (property.equals("relationship-list") && !nodeOnly){
1599                                                         /* relationships need to be handled correctly */
1600                                                         Introspector relationshipList = obj.newIntrospectorInstanceOfProperty(property);
1601                                                         relationshipList = createFilteredRelationshipList(v, relationshipList, cleanUp, filterCousinNodes);
1602                                                         if (relationshipList != null) {
1603                                                                 modified = true;
1604                                                                 obj.setValue(property, relationshipList.getUnderlyingObject());
1605                                                                 modified = true;
1606                                                         }
1607
1608                                                 }
1609                                         } else if (obj.isListType(property)) {
1610
1611                                                 if (property.equals("any")) {
1612                                                         continue;
1613                                                 }
1614                                                 String genericType = obj.getGenericTypeClass(property).getSimpleName();
1615                                                 if (obj.isComplexGenericType(property) && depth >= 0) {
1616                                                         final String childDbName = convertFromCamelCase(genericType);
1617                                                         String vType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null);
1618                                                         EdgeRule rule;
1619
1620                                                         boolean isthisParentRequired = filterParentNodes.parallelStream().anyMatch(childDbName::contains);
1621
1622
1623
1624                                                         rule = edgeRules.getEdgeRule(EdgeType.TREE, vType, childDbName);
1625                                                         if (!rule.getContains().equals(AAIDirection.NONE.toString()) && isthisParentRequired) {
1626                                                                 //vertices = this.queryEngine.findRelatedVertices(v, Direction.OUT, rule.getLabel(), childDbName);
1627                                                                 Direction ruleDirection = rule.getDirection();
1628                                                                 Iterator<Vertex> itr = v.vertices(ruleDirection, rule.getLabel());
1629                                                                 List<Vertex> verticesList = (List<Vertex>)IteratorUtils.toList(itr);
1630                                                                 itr = verticesList.stream().filter(item -> {
1631                                                                         return item.property(AAIProperties.NODE_TYPE).orElse("").equals(childDbName);
1632                                                                 }).iterator();
1633                                                                 if (itr.hasNext()) {
1634                                                                         getList = (List<Object>)obj.getValue(property);
1635                                                                 }
1636                                                                 int processed = 0;
1637                                                                 int removed = 0;
1638                                                                 while (itr.hasNext()) {
1639                                                                         Vertex childVertex = itr.next();
1640                                                                         if (!seen.contains(childVertex)) {
1641                                                                                 Introspector argumentObject = obj.newIntrospectorInstanceOfNestedProperty(property);
1642
1643                                                                                 Object result = dbToObjectWithFilters(argumentObject, childVertex, seen, depth, nodeOnly, filterCousinNodes, filterParentNodes);
1644                                                                                 if (result != null) {
1645                                                                                         getList.add(argumentObject.getUnderlyingObject());
1646                                                                                 }
1647
1648                                                                                 processed++;
1649                                                                         } else {
1650                                                                                 removed++;
1651                                                                                 LOGGER.warn("Cycle found while serializing vertex id={}", childVertex.id().toString());
1652                                                                         }
1653                                                                 }
1654                                                                 if (processed == 0) {
1655                                                                         //vertices were all seen, reset the list
1656                                                                         getList = null;
1657                                                                 }
1658                                                                 if (processed > 0) {
1659                                                                         modified = true;
1660                                                                 }
1661                                                         }
1662                                                 } else if (obj.isSimpleGenericType(property)) {
1663                                                         List<Object> temp = this.engine.getListProperty(v, property);
1664                                                         if (temp != null) {
1665                                                                 getList = (List<Object>)obj.getValue(property);
1666                                                                 getList.addAll(temp);
1667                                                                 modified = true;
1668                                                         }
1669
1670                                                 }
1671
1672                                         }
1673
1674                                 }
1675                         }
1676
1677                         //no changes were made to this obj, discard the instance
1678                         if (!modified) {
1679                                 return null;
1680                         }
1681                         this.enrichData(obj, v);
1682                         return obj;
1683
1684                 }
1685
1686          /**
1687                  * Creates the relationship list with the filtered node types.
1688                  *
1689                  * @param v the v
1690                  * @param obj the obj
1691                  * @param cleanUp the clean up
1692                  * @return the object
1693                  * @throws InstantiationException the instantiation exception
1694                  * @throws IllegalAccessException the illegal access exception
1695                  * @throws IllegalArgumentException the illegal argument exception
1696                  * @throws InvocationTargetException the invocation target exception
1697                  * @throws NoSuchMethodException the no such method exception
1698                  * @throws SecurityException the security exception
1699                  * @throws UnsupportedEncodingException the unsupported encoding exception
1700                  * @throws AAIException the AAI exception
1701                  * @throws MalformedURLException the malformed URL exception
1702                  * @throws URISyntaxException
1703                  */
1704                 private Introspector createFilteredRelationshipList(Vertex v, Introspector obj, String cleanUp, List<String> filterNodes) throws UnsupportedEncodingException, AAIException {
1705                         List<Vertex> allCousins = this.engine.getQueryEngine().findCousinVertices(v);
1706
1707                         Iterator<Vertex> cousinVertices = allCousins.stream().filter(item -> {
1708                                 String node = (String)item.property(AAIProperties.NODE_TYPE).orElse("");
1709                                 return filterNodes.parallelStream().anyMatch(node::contains);
1710                         }).iterator();
1711
1712
1713                         List<Vertex> cousins = (List<Vertex>)IteratorUtils.toList(cousinVertices);
1714
1715                         //items.parallelStream().anyMatch(inputStr::contains)
1716                         List<Object> relationshipObjList = obj.getValue("relationship");
1717                         for (Vertex cousin : cousins) {
1718
1719                                         Introspector relationshipObj = obj.newIntrospectorInstanceOfNestedProperty("relationship");
1720                                         Object result = processEdgeRelationship(relationshipObj, cousin, cleanUp, null);
1721                                         if (result != null) {
1722                                                 relationshipObjList.add(result);
1723                                         }
1724
1725
1726                         }
1727
1728                         if (relationshipObjList.isEmpty()) {
1729                                 return null;
1730                         } else {
1731                                 return obj;
1732                         }
1733                 }
1734
1735 }