-/*-
+/**
* ============LICENSE_START=======================================================
* org.onap.aai
* ================================================================================
- * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
* ================================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* limitations under the License.
* ============LICENSE_END=========================================================
*/
-
package org.onap.aai.serialization.db;
-import java.io.UnsupportedEncodingException;
-import java.lang.reflect.Array;
-import java.lang.reflect.InvocationTargetException;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import javax.ws.rs.core.UriBuilder;
-
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.google.common.base.CaseFormat;
+import com.thinkaurelius.titan.core.SchemaViolationException;
import org.apache.commons.collections.IteratorUtils;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.*;
import org.javatuples.Pair;
import org.javatuples.Triplet;
import org.onap.aai.db.props.AAIProperties;
import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.introspection.Introspector;
-import org.onap.aai.introspection.IntrospectorFactory;
-import org.onap.aai.introspection.Loader;
-import org.onap.aai.introspection.LoaderFactory;
-import org.onap.aai.introspection.ModelType;
-import org.onap.aai.introspection.PropertyPredicates;
-import org.onap.aai.introspection.Version;
+import org.onap.aai.introspection.*;
import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
import org.onap.aai.introspection.sideeffect.DataCopy;
import org.onap.aai.introspection.sideeffect.DataLinkReader;
import org.onap.aai.introspection.sideeffect.DataLinkWriter;
import org.onap.aai.introspection.sideeffect.SideEffectRunner;
import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
import org.onap.aai.parsers.query.QueryParser;
import org.onap.aai.parsers.uri.URIParser;
import org.onap.aai.parsers.uri.URIToRelationshipObject;
import org.onap.aai.util.AAIConfig;
import org.onap.aai.util.AAIConstants;
import org.onap.aai.workarounds.NamingExceptions;
+import org.onap.aai.logging.StopWatch;
-import com.att.eelf.configuration.EELFLogger;
-import com.att.eelf.configuration.EELFManager;
-import com.google.common.base.CaseFormat;
-import com.thinkaurelius.titan.core.SchemaViolationException;
+import javax.ws.rs.core.UriBuilder;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationTargetException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class DBSerializer {
private final EdgeRules edgeRules = EdgeRules.getInstance();
private final Loader loader;
private final String baseURL;
+ private double dbTimeMsecs = 0;
/**
* Instantiates a new DB serializer.
*
if (isNewVertex) {
v.property(AAIProperties.SOURCE_OF_TRUTH, this.sourceOfTruth);
v.property(AAIProperties.CREATED_TS, timeNowInSec);
+ v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString());
}
v.property(AAIProperties.RESOURCE_VERSION, timeNowInSec );
v.property(AAIProperties.LAST_MOD_TS, timeNowInSec);
* @throws AAIException the AAI exception
*/
public Vertex createNewVertex(Introspector wrappedObject) {
-
-
- Vertex v = this.engine.tx().addVertex();
- touchStandardVertexProperties(wrappedObject.getDbName(), v, true);
-
+ Vertex v;
+ try {
+ StopWatch.conditionalStart();
+ v = this.engine.tx().addVertex();
+ touchStandardVertexProperties(wrappedObject.getDbName(), v, true);
+ }
+ finally {
+ dbTimeMsecs += StopWatch.stopIfStarted();
+ }
return v;
}
* @throws AAIUnknownObjectException
*/
public void serializeToDb(Introspector obj, Vertex v, QueryParser uriQuery, String identifier, String requestContext) throws AAIException, UnsupportedEncodingException {
-
+ StopWatch.conditionalStart();
try {
if (uriQuery.isDependent()) {
//try to find the parent
Vertex parent = vertices.get(0);
this.reflectDependentVertex(parent, v, obj, requestContext);
} else {
+ dbTimeMsecs += StopWatch.stopIfStarted();
throw new AAIException("AAI_6114", "No parent Node of type " + uriQuery.getParentResultType() + " for " + identifier);
}
} else {
}
} catch (SchemaViolationException e) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
throw new AAIException("AAI_6117", e);
}
-
+ dbTimeMsecs += StopWatch.stopIfStarted();
}
public void serializeSingleVertex(Vertex v, Introspector obj, String requestContext) throws UnsupportedEncodingException, AAIException {
+ StopWatch.conditionalStart();
try {
boolean isTopLevel = obj.isTopLevel();
if (isTopLevel) {
v.property(AAIProperties.AAI_URI, obj.getURI());
}
+
processObject(obj, v, requestContext);
if (!isTopLevel) {
URI uri = this.getURIForVertex(v);
} catch (SchemaViolationException e) {
throw new AAIException("AAI_6117", e);
}
+ finally {
+ dbTimeMsecs += StopWatch.stopIfStarted();
+ }
}
/**
String parentUri = parent.<String>property(AAIProperties.AAI_URI).orElse(null);
if (parentUri != null) {
- String uri = obj.getURI();
+ String uri;
+ uri = obj.getURI();
child.property(AAIProperties.AAI_URI, parentUri + uri);
}
processObject(obj, child, requestContext);
if (e == null) {
String canBeLinked = obj.getMetadata(ObjectMetadata.CAN_BE_LINKED);
if (canBeLinked != null && canBeLinked.equals("true")) {
- boolean isFirst = !this.engine.getQueryBuilder(parent).createEdgeTraversal(EdgeType.TREE, parent, obj).hasNext();
+ Loader ldrForCntxt = LoaderFactory.createLoaderForVersion(introspectionType, getVerForContext(requestContext));
+ boolean isFirst = !this.engine.getQueryBuilder(ldrForCntxt, parent).createEdgeTraversal(EdgeType.TREE, parent, obj).hasNext();
if (isFirst) {
child.property(AAIProperties.LINKED, true);
}
return child;
}
+
+ private Version getVerForContext(String requestContext) {
+ Pattern pattern = Pattern.compile("v[0-9]+");
+ Matcher m = pattern.matcher(requestContext);
+ if (!m.find()) {
+ return this.version;
+ } else {
+ return Version.valueOf(requestContext);
+ }
+ }
/**
* Db to object.
* @throws URISyntaxException
*/
public Introspector dbToObject(List<Vertex> vertices, final Introspector obj, int depth, boolean nodeOnly, String cleanUp) throws UnsupportedEncodingException, AAIException {
-
final int internalDepth;
if (depth == Integer.MAX_VALUE) {
internalDepth = depth--;
} else {
internalDepth = depth;
}
+ StopWatch.conditionalStart();
if (vertices.size() > 1 && !obj.isContainer()) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
throw new AAIException("AAI_6136", "query object mismatch: this object cannot hold multiple items." + obj.getDbName());
} else if (obj.isContainer()) {
final List getList;
ExecutorService pool = GetAllPool.getInstance().getPool();
List<Future<Object>> futures = new ArrayList<>();
+
+
for (Vertex v : vertices) {
Callable<Object> task = () -> {
Set<Vertex> seen = new HashSet<>();
for (Future<Object> future : futures) {
try {
getList.add(future.get());
- } catch (ExecutionException | InterruptedException e) {
+ } catch (ExecutionException e) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
+ throw new AAIException("AAI_4000", e);
+ } catch (InterruptedException e) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
throw new AAIException("AAI_4000", e);
}
}
//obj = null;
}
-
+ dbTimeMsecs += StopWatch.stopIfStarted();
return obj;
}
Introspector argumentObject = obj.newIntrospectorInstanceOfNestedProperty(property);
Object result = dbToObject(argumentObject, childVertex, seen, depth, nodeOnly, cleanUp);
- if (result != null) {
+ if (result != null) {
getList.add(argumentObject.getUnderlyingObject());
}
if (nodeType == null) {
throw new AAIException("AAI_6143");
}
+
Introspector obj = this.latestLoader.introspectorFromName(nodeType);
Set<Vertex> seen = new HashSet<>();
int depth = 0;
String cleanUp = "false";
boolean nodeOnly = true;
+ StopWatch.conditionalStart();
this.dbToObject(obj, v, seen, depth, nodeOnly, cleanUp);
-
+ dbTimeMsecs += StopWatch.stopIfStarted();
return obj;
}
int depth = AAIProperties.MAXIMUM_DEPTH;
String cleanUp = "false";
boolean nodeOnly = false;
+ StopWatch.conditionalStart();
Tree<Element> tree = this.engine.getQueryEngine().findSubGraph(v, depth, nodeOnly);
TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree);
this.dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp);
-
+ dbTimeMsecs += StopWatch.stopIfStarted();
return obj;
}
/**
uriParser = new URIToRelationshipObject(relationshipObj.getLoader(), uri, this.baseURL);
result = uriParser.getResult();
} catch (AAIException | URISyntaxException e) {
- LOGGER.error("Error while processing edge relationship in version " + relationshipObj.getVersion() + " (bad vertex ID=" + tuple.get().getValue0().id().toString() + ": " + e.getMessage(), e);
+ LOGGER.error("Error while processing edge relationship in version " + relationshipObj.getVersion() + " (bad vertex ID=" + tuple.get().getValue0().id().toString() + ": "
+ + e.getMessage() + " " + LogFormatTools.getStackTop(e));
if ("true".equals(cleanUp)) {
this.deleteWithTraversal(tuple.get().getValue0());
}
if (aaiUri != null && !overwrite) {
uri = UriBuilder.fromPath(aaiUri).build();
} else {
+ StopWatch.conditionalStart();
Optional<Pair<Vertex, List<Introspector>>> tuple = this.getParents(this.loader, v, false);
+ dbTimeMsecs += StopWatch.stopIfStarted();
if (tuple.isPresent()) {
List<Introspector> list = tuple.get().getValue1();
uri = this.getURIFromList(list);
}
+
+
}
return uri;
}
public boolean createEdge(Introspector relationship, Vertex inputVertex) throws UnsupportedEncodingException, AAIException {
Vertex relatedVertex = null;
-
+ StopWatch.conditionalStart();
QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(relationship);
String label = null;
List<Vertex> results = parser.getQueryBuilder().toList();
if (results.isEmpty()) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
AAIException e = new AAIException("AAI_6129", "Node of type " + parser.getResultType() + ". Could not find object at: " + parser.getUri());
e.getTemplateVars().add(parser.getResultType());
e.getTemplateVars().add(parser.getUri().toString());
if (relatedVertex != null) {
- Edge e = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label);
- if (e == null) {
- edgeRules.addEdge(this.engine.asAdmin().getTraversalSource(), inputVertex, relatedVertex, label);
- } else {
- //attempted to link two vertexes already linked
+ Edge e;
+ try {
+ e = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label);
+ if (e == null) {
+ edgeRules.addEdge(this.engine.asAdmin().getTraversalSource(), inputVertex, relatedVertex, label);
+ } else {
+ //attempted to link two vertexes already linked
+ }
+ } finally {
+ dbTimeMsecs += StopWatch.stopIfStarted();
}
}
+ dbTimeMsecs += StopWatch.stopIfStarted();
return true;
}
*/
public Edge getEdgeBetween(EdgeType type, Vertex aVertex, Vertex bVertex, String label) throws AAIException {
+ StopWatch.conditionalStart();
if (bVertex != null) {
List<Edge> edges = this.getEdgesBetween(type, aVertex, bVertex, label);
if (!edges.isEmpty()) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
return edges.get(0);
}
}
-
+ dbTimeMsecs += StopWatch.stopIfStarted();
return null;
}
public Edge getEdgeBetween(EdgeType type, Vertex aVertex, Vertex bVertex) throws AAIException {
public boolean deleteEdge(Introspector relationship, Vertex inputVertex) throws UnsupportedEncodingException, AAIException {
Vertex relatedVertex = null;
-
+ StopWatch.conditionalStart();
QueryParser parser = engine.getQueryBuilder().createQueryFromRelationship(relationship);
List<Vertex> results = parser.getQueryBuilder().toList();
}
if (results.isEmpty()) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
return false;
}
try {
edge = this.getEdgeBetween(EdgeType.COUSIN, inputVertex, relatedVertex, label);
} catch (NoEdgeRuleFoundException e) {
+ dbTimeMsecs += StopWatch.stopIfStarted();
throw new AAIException("AAI_6129", e);
}
if (edge != null) {
edge.remove();
+ dbTimeMsecs += StopWatch.stopIfStarted();
return true;
} else {
+ dbTimeMsecs += StopWatch.stopIfStarted();
return false;
}
* @throws IllegalStateException the illegal state exception
*/
public void deleteItemsWithTraversal(List<Vertex> vertexes) throws IllegalStateException {
+
for (Vertex v : vertexes) {
- LOGGER.debug("About to delete the vertex with id: " + v.id());
+ LOGGER.debug("About to delete the vertex with id: " + v.id());
deleteWithTraversal(v);
}
+
}
/**
* @param startVertex the start vertex
*/
public void deleteWithTraversal(Vertex startVertex) {
-
+ StopWatch.conditionalStart();
List<Vertex> results = this.engine.getQueryEngine().findDeletable(startVertex);
for (Vertex v : results) {
v.remove();
}
-
+ dbTimeMsecs += StopWatch.stopIfStarted();
}
/**
nodeType = vertex.<String>property(AAIProperties.NODE_TYPE).orElse(null);
if (enableResourceVersion && !this.verifyResourceVersion("delete", nodeType, vertex.<String>property(AAIProperties.RESOURCE_VERSION).orElse(null), resourceVersion, nodeType)) {
}
+ StopWatch.conditionalStart();
List<Object> preventDeleteVertices = this.engine.asAdmin().getReadOnlyTraversalSource().V(vertex).union(__.inE().has(EdgeProperty.PREVENT_DELETE.toString(), AAIDirection.IN.toString()).outV().values(AAIProperties.NODE_TYPE), __.outE().has(EdgeProperty.PREVENT_DELETE.toString(), AAIDirection.OUT.toString()).inV().values(AAIProperties.NODE_TYPE)).dedup().toList();
+ dbTimeMsecs += StopWatch.stopIfStarted();
if (!preventDeleteVertices.isEmpty()) {
aaiExceptionCode = "AAI_6110";
errorDetail = String.format("Object is being reference by additional objects preventing it from being deleted. Please clean up references from the following types %s", preventDeleteVertices);
runner.execute(obj, self);
}
+ public double getDBTimeMsecs() {
+ return (dbTimeMsecs);
+ }
+
+ /**
+ * Db to object With Filters
+ * This is for a one-time run with Tenant Isloation to only filter relationships
+ * TODO: Chnage the original dbToObject to take filter parent/cousins
+ *
+ * @param vertices the vertices
+ * @param obj the obj
+ * @param depth the depth
+ * @param cleanUp the clean up
+ * @return the introspector
+ * @throws AAIException the AAI exception
+ * @throws IllegalAccessException the illegal access exception
+ * @throws IllegalArgumentException the illegal argument exception
+ * @throws InvocationTargetException the invocation target exception
+ * @throws SecurityException the security exception
+ * @throws InstantiationException the instantiation exception
+ * @throws NoSuchMethodException the no such method exception
+ * @throws UnsupportedEncodingException the unsupported encoding exception
+ * @throws MalformedURLException the malformed URL exception
+ * @throws AAIUnknownObjectException
+ * @throws URISyntaxException
+ */
+ //TODO - See if you can merge the 2 dbToObjectWithFilters
+ public Introspector dbToObjectWithFilters(Introspector obj, Vertex v, Set<Vertex> seen, int depth, boolean nodeOnly, List<String> filterCousinNodes, List<String> filterParentNodes) throws AAIException, UnsupportedEncodingException {
+ String cleanUp = "false";
+ if (depth < 0) {
+ return null;
+ }
+ depth--;
+ seen.add(v);
+ boolean modified = false;
+ for (String property : obj.getProperties(PropertyPredicates.isVisible())) {
+ List<Object> getList = null;
+ Vertex[] vertices = null;
+
+ if (!(obj.isComplexType(property) || obj.isListType(property))) {
+ this.copySimpleProperty(property, obj, v);
+ modified = true;
+ } else {
+ if (obj.isComplexType(property)) {
+ /* container case */
+
+ if (!property.equals("relationship-list") && depth >= 0) {
+ Introspector argumentObject = obj.newIntrospectorInstanceOfProperty(property);
+ Object result = dbToObjectWithFilters(argumentObject, v, seen, depth+1, nodeOnly, filterCousinNodes, filterParentNodes);
+ if (result != null) {
+ obj.setValue(property, argumentObject.getUnderlyingObject());
+ modified = true;
+ }
+ } else if (property.equals("relationship-list") && !nodeOnly){
+ /* relationships need to be handled correctly */
+ Introspector relationshipList = obj.newIntrospectorInstanceOfProperty(property);
+ relationshipList = createFilteredRelationshipList(v, relationshipList, cleanUp, filterCousinNodes);
+ if (relationshipList != null) {
+ modified = true;
+ obj.setValue(property, relationshipList.getUnderlyingObject());
+ modified = true;
+ }
+
+ }
+ } else if (obj.isListType(property)) {
+
+ if (property.equals("any")) {
+ continue;
+ }
+ String genericType = obj.getGenericTypeClass(property).getSimpleName();
+ if (obj.isComplexGenericType(property) && depth >= 0) {
+ final String childDbName = convertFromCamelCase(genericType);
+ String vType = v.<String>property(AAIProperties.NODE_TYPE).orElse(null);
+ EdgeRule rule;
+
+ boolean isthisParentRequired = filterParentNodes.parallelStream().anyMatch(childDbName::contains);
+
+
+
+ rule = edgeRules.getEdgeRule(EdgeType.TREE, vType, childDbName);
+ if (!rule.getContains().equals(AAIDirection.NONE.toString()) && isthisParentRequired) {
+ //vertices = this.queryEngine.findRelatedVertices(v, Direction.OUT, rule.getLabel(), childDbName);
+ Direction ruleDirection = rule.getDirection();
+ Iterator<Vertex> itr = v.vertices(ruleDirection, rule.getLabel());
+ List<Vertex> verticesList = (List<Vertex>)IteratorUtils.toList(itr);
+ itr = verticesList.stream().filter(item -> {
+ return item.property(AAIProperties.NODE_TYPE).orElse("").equals(childDbName);
+ }).iterator();
+ if (itr.hasNext()) {
+ getList = (List<Object>)obj.getValue(property);
+ }
+ int processed = 0;
+ int removed = 0;
+ while (itr.hasNext()) {
+ Vertex childVertex = itr.next();
+ if (!seen.contains(childVertex)) {
+ Introspector argumentObject = obj.newIntrospectorInstanceOfNestedProperty(property);
+
+ Object result = dbToObjectWithFilters(argumentObject, childVertex, seen, depth, nodeOnly, filterCousinNodes, filterParentNodes);
+ if (result != null) {
+ getList.add(argumentObject.getUnderlyingObject());
+ }
+
+ processed++;
+ } else {
+ removed++;
+ LOGGER.warn("Cycle found while serializing vertex id={}", childVertex.id().toString());
+ }
+ }
+ if (processed == 0) {
+ //vertices were all seen, reset the list
+ getList = null;
+ }
+ if (processed > 0) {
+ modified = true;
+ }
+ }
+ } else if (obj.isSimpleGenericType(property)) {
+ List<Object> temp = this.engine.getListProperty(v, property);
+ if (temp != null) {
+ getList = (List<Object>)obj.getValue(property);
+ getList.addAll(temp);
+ modified = true;
+ }
+
+ }
+
+ }
+
+ }
+ }
+
+ //no changes were made to this obj, discard the instance
+ if (!modified) {
+ return null;
+ }
+ this.enrichData(obj, v);
+ return obj;
+
+ }
+
+ /**
+ * Creates the relationship list with the filtered node types.
+ *
+ * @param v the v
+ * @param obj the obj
+ * @param cleanUp the clean up
+ * @return the object
+ * @throws InstantiationException the instantiation exception
+ * @throws IllegalAccessException the illegal access exception
+ * @throws IllegalArgumentException the illegal argument exception
+ * @throws InvocationTargetException the invocation target exception
+ * @throws NoSuchMethodException the no such method exception
+ * @throws SecurityException the security exception
+ * @throws UnsupportedEncodingException the unsupported encoding exception
+ * @throws AAIException the AAI exception
+ * @throws MalformedURLException the malformed URL exception
+ * @throws URISyntaxException
+ */
+ private Introspector createFilteredRelationshipList(Vertex v, Introspector obj, String cleanUp, List<String> filterNodes) throws UnsupportedEncodingException, AAIException {
+ List<Vertex> allCousins = this.engine.getQueryEngine().findCousinVertices(v);
+
+ Iterator<Vertex> cousinVertices = allCousins.stream().filter(item -> {
+ String node = (String)item.property(AAIProperties.NODE_TYPE).orElse("");
+ return filterNodes.parallelStream().anyMatch(node::contains);
+ }).iterator();
+
+
+ List<Vertex> cousins = (List<Vertex>)IteratorUtils.toList(cousinVertices);
+
+ //items.parallelStream().anyMatch(inputStr::contains)
+ List<Object> relationshipObjList = obj.getValue("relationship");
+ for (Vertex cousin : cousins) {
+
+ Introspector relationshipObj = obj.newIntrospectorInstanceOfNestedProperty("relationship");
+ Object result = processEdgeRelationship(relationshipObj, cousin, cleanUp, null);
+ if (result != null) {
+ relationshipObjList.add(result);
+ }
+
+
+ }
+
+ if (relationshipObjList.isEmpty()) {
+ return null;
+ } else {
+ return obj;
+ }
+ }
+
}