From 5722e90ff54f5fb6bb16ebd1dfb723d08ab80955 Mon Sep 17 00:00:00 2001 From: "Kajur, Venkata Harish (vk250x)" Date: Tue, 2 Apr 2019 11:05:31 -0400 Subject: [PATCH] Add the logging statements for when application is doing implicit delete so if an user did a PUT on cloud region in which they implicitly delete 100 tenants, it would log the specific tenants that got deleted with depth all on tenants because a cloud region could also have thousands of other child objects and there is no reason to log that if we can just log the objects that were lost in the implicit delete process Issue-ID: AAI-2332 Change-Id: I8cd0c876670f8687b6485d3735f9f47e84c38286 Signed-off-by: Kajur, Harish (vk250x) --- .../onap/aai/serialization/db/DBSerializer.java | 100 ++++++++++++++++++--- .../engines/query/GraphTraversalQueryEngine.java | 49 +++++++--- .../serialization/engines/query/QueryEngine.java | 11 ++- .../src/main/java/org/onap/aai/util/AAIConfig.java | 43 +++++---- .../main/java/org/onap/aai/util/AAIConstants.java | 9 ++ 5 files changed, 171 insertions(+), 41 deletions(-) diff --git a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java index 4c9a9b15..575f8191 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/db/DBSerializer.java @@ -23,12 +23,13 @@ package org.onap.aai.serialization.db; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.common.base.CaseFormat; -import com.google.common.collect.Multimap; import org.apache.commons.collections.IteratorUtils; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; -import org.apache.tinkerpop.gremlin.structure.*; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.janusgraph.core.SchemaViolationException; import org.javatuples.Triplet; import org.onap.aai.concurrent.AaiCallable; @@ -62,7 +63,6 @@ import org.onap.aai.serialization.db.exceptions.MultipleEdgeRuleFoundException; import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException; import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.serialization.engines.query.QueryEngine; -import org.onap.aai.serialization.tinkerpop.TreeBackedVertex; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.util.AAIConfig; @@ -81,15 +81,15 @@ import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; public class DBSerializer { private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DBSerializer.class); + private static final String IMPLICIT_DELETE = "Implicit DELETE"; + private static final String MISSING_REQUIRED_NODE_PROPERTY = "Vertex missing required aai-node-type property"; private final TransactionalGraphEngine engine; @@ -417,7 +417,57 @@ public class DBSerializer { for (Vertex toBeRemoved : processedVertexes) { dependentVertexes.remove(toBeRemoved); } - this.deleteItemsWithTraversal(dependentVertexes); + + // If the dependent vertices are not empty, then with + // the current behaviour, it should remove the vertices implicitly + // We are updating the code to properly log which call + // is doing this so the SE can work with the clients making the call to + // tell them not to call this API and can hopefully deprecate this + // functionality in the future releases + if(!dependentVertexes.isEmpty()){ + + LoggingContext.responseDescription(IMPLICIT_DELETE); + + // Find all the deletable vertices from the dependent vertices that should be deleted + // So for each of the following dependent vertices, + // we will use the edge properties and do the cascade delete + List impliedDeleteVertices = this.engine.getQueryEngine().findDeletable(dependentVertexes); + int impliedDeleteCount = impliedDeleteVertices.size(); + + LOGGER.warn( + "For the vertex with id {}, doing an implicit delete on update will delete total of {} vertexes", + v.id(), + impliedDeleteCount + ); + + String impliedDeleteLogEnabled = AAIConfig.get(AAIConstants.AAI_IMPLIED_DELETE_LOG_ENABLED, "true"); + + int impliedDeleteLogLimit = AAIConfig.getInt(AAIConstants.AAI_IMPLIED_DELETE_LOG_LIMIT, "-1"); + + if(impliedDeleteLogLimit == -1){ + impliedDeleteLogLimit = Integer.MAX_VALUE; + } + + // If the logging is enabled for implied delete + // then log the payload in the latest format + if("true".equals(impliedDeleteLogEnabled) && + impliedDeleteCount <= impliedDeleteLogLimit){ + for(Vertex vertex : impliedDeleteVertices){ + Introspector introspector = null; + try { + introspector = getLatestVersionView(vertex); + if(LOGGER.isInfoEnabled()){ + LOGGER.info("Implied delete object in json format {}", introspector.marshal(false)); + } + } catch(Exception ex){ + LOGGER.warn("Encountered an exception during retrieval of vertex properties with vertex-id {} -> {}", v.id(), LogFormatTools.getStackTop(ex)); + } + } + } + + // After all the appropriate logging, calling the delete to delete the affected vertices + this.delete(impliedDeleteVertices); + } this.executePostSideEffects(obj, v); return processedVertexes; @@ -910,9 +960,7 @@ public class DBSerializer { String cleanUp = "false"; boolean nodeOnly = false; StopWatch.conditionalStart(); - Tree tree = this.engine.getQueryEngine().findSubGraph(v, depth, nodeOnly); - TreeBackedVertex treeVertex = new TreeBackedVertex(v, tree); - this.dbToObject(obj, treeVertex, seen, depth, nodeOnly, cleanUp); + this.dbToObject(obj, v, seen, depth, nodeOnly, cleanUp); dbTimeMsecs += StopWatch.stopIfStarted(); return obj; } @@ -1608,7 +1656,6 @@ public class DBSerializer { public void deleteItemsWithTraversal(List vertexes) throws IllegalStateException { for (Vertex v : vertexes) { - LOGGER.debug("About to delete the vertex with id: " + v.id()); deleteWithTraversal(v); } @@ -1624,10 +1671,39 @@ public class DBSerializer { List results = this.engine.getQueryEngine().findDeletable(startVertex); for (Vertex v : results) { - LOGGER.warn("Removing vertex " + v.id().toString()); + LOGGER.debug("Removing vertex {} with label {}", v.id(), v.label()); + v.remove(); + } + dbTimeMsecs += StopWatch.stopIfStarted(); + } + /** + * Removes the list of vertexes from the graph + *

+ * Current the vertex label will just be vertex but + * in the future the aai-node-type property will be replaced + * by using the vertex label as when retrieving an vertex + * and retrieving an single property on an vertex will pre-fetch + * all the properties of that vertex and this is due to the following property + *

+ * + * query.fast-property=true + * + *

+ * JanusGraph doesn't provide the capability to override that + * at a transaction level and there is a plan to move to vertex label + * so it is best to utilize this for now and when the change is applied + * + * @param vertices - list of vertices to delete from the graph + */ + void delete(List vertices){ + StopWatch.conditionalStart(); + + for (Vertex v : vertices) { + LOGGER.debug("Removing vertex {} with label {}", v.id(), v.label()); v.remove(); } + dbTimeMsecs += StopWatch.stopIfStarted(); } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java index d072db55..94557b08 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/GraphTraversalQueryEngine.java @@ -58,7 +58,7 @@ public class GraphTraversalQueryEngine extends QueryEngine { /** * Instantiates a new graph traversal query engine. * - * @param graphEngine the graph engine + * @param g graph traversal source to traverse the graph */ public GraphTraversalQueryEngine(GraphTraversalSource g) { super(g); @@ -146,18 +146,47 @@ public class GraphTraversalQueryEngine extends QueryEngine { */ @Override public List findDeletable(Vertex start) { - @SuppressWarnings("unchecked") - GraphTraversal pipe = this.g - .V(start).emit(v -> true).repeat( - __.union( - __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), - __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() - ) - ).dedup(); + try { + StopWatch.conditionalStart(); + @SuppressWarnings("unchecked") + GraphTraversal pipe = this.g + .V(start).emit(v -> true).repeat( + __.union( + __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), + __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() + ) + ).dedup(); - return pipe.toList(); + return pipe.toList(); + } finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } } + /** + * {@inheritDoc} + */ + @Override + public List findDeletable(List startVertexes) { + try { + StopWatch.conditionalStart(); + Vertex[] vertices = new Vertex[startVertexes.size()]; + vertices = startVertexes.toArray(vertices); + GraphTraversal pipe = this.g + .V(vertices).emit(v -> true).repeat( + __.union( + __.outE().has(DELETE_OTHER_V.toString(), OUT.toString()).inV(), + __.inE().has(DELETE_OTHER_V.toString(), IN.toString()).outV() + ) + ).dedup(); + + return pipe.toList(); + } + finally { + dbTimeMsecs += StopWatch.stopIfStarted(); + } + } + /** * {@inheritDoc} */ diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java index 110f8628..1e2da4b8 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/query/QueryEngine.java @@ -37,7 +37,7 @@ public abstract class QueryEngine { /** * Instantiates a new query engine. * - * @param graphEngine the graph engine + * @param g graph traversal source to traverse the graph */ public QueryEngine (GraphTraversalSource g) { this.g = g; @@ -100,6 +100,15 @@ public abstract class QueryEngine { */ public abstract List findDeletable(Vertex start); + /** + * Find all vertices that should be deleted in a cascade from a delete of start vertexes + * + * @param startVertexes Specifies the list of start vertexes + * + * @return the list of vertices to be deleted when start list of vertexes is deleted + */ + public abstract List findDeletable(List startVertexes); + /** * Finds the subgraph under start, including cousins as well as start's children/grandchildren/etc. * More specifically, this includes start, all its descendants, start's cousins, and start's diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java index 5b48127c..86cb635e 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIConfig.java @@ -24,10 +24,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; -import java.util.ArrayList; -import java.util.HashMap; import java.util.Properties; -import java.util.Timer; import org.onap.aai.logging.LoggingContext; import org.onap.aai.logging.LoggingContext.StatusCode; @@ -47,7 +44,7 @@ public class AAIConfig { private static final String GLOBAL_PROP_FILE_NAME = AAIConstants.AAI_CONFIG_FILENAME; private static Properties serverProps; private static boolean propsInitialized = false; - + /** * Instantiates a new AAI config. */ @@ -71,11 +68,11 @@ public class AAIConfig { LoggingContext.statusCode(StatusCode.COMPLETE); LOGGER.info("Initializing AAIConfig"); - + AAIConfig.getConfigFile(); AAIConfig.reloadConfig(); - - if (AAIConstants.AAI_NODENAME == null || AAIConstants.AAI_NODENAME == "") { + + if (AAIConstants.AAI_NODENAME == null || AAIConstants.AAI_NODENAME == "") { ErrorLogHelper.logError("AAI_4005", " AAI_NODENAME is not defined"); } else { LOGGER.info("A&AI Server Node Name = " + AAIConstants.AAI_NODENAME); @@ -99,9 +96,9 @@ public class AAIConfig { String propFileName = GLOBAL_PROP_FILE_NAME; Properties newServerProps = null; - + LOGGER.debug("Reloading config from " + propFileName); - + try(InputStream is = new FileInputStream(propFileName)) { newServerProps = new Properties(); newServerProps.load(is); @@ -113,7 +110,7 @@ public class AAIConfig { ErrorLogHelper.logError("AAI_4002", " " + propFileName + ". IOException: "+e.getMessage()); } } - + /** * Gets the. * @@ -142,7 +139,7 @@ public class AAIConfig { */ public static String get(String key) throws AAIException { String response = null; - + if (key.equals(AAIConstants.AAI_NODENAME)) { // Get this from InetAddress rather than the properties file String nodeName = getNodeName(); @@ -151,16 +148,16 @@ public class AAIConfig { } // else get from property file } - + if (!propsInitialized || (serverProps == null)) { reloadConfig(); } - + if ((key.endsWith("password") || key.endsWith("passwd") || key.endsWith("apisecret")) && serverProps.containsKey(key+".x")) { String valx = serverProps.getProperty(key+".x"); return Password.deobfuscate(valx); } - + if (!serverProps.containsKey(key)) { throw new AAIException("AAI_4005", "Property key "+key+" cannot be found"); } else { @@ -180,9 +177,19 @@ public class AAIConfig { * @throws AAIException the AAI exception */ public static int getInt(String key) throws AAIException{ - return Integer.valueOf(AAIConfig.get(key)); + return Integer.parseInt(AAIConfig.get(key)); } + /** + * Gets the int. + * + * @param key the key + * @return the int + */ + public static int getInt(String key, String value) { + return Integer.parseInt(AAIConfig.get(key, value)); + } + /** * Gets the server props. * @@ -191,7 +198,7 @@ public class AAIConfig { public static Properties getServerProps() { return serverProps; } - + /** * Gets the node name. * @@ -211,8 +218,8 @@ public class AAIConfig { } return null; } - - + + /** * Check if a null or an Empty string is passed in. * diff --git a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java index b735f722..b9fb08f8 100644 --- a/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java +++ b/aai-core/src/main/java/org/onap/aai/util/AAIConstants.java @@ -69,6 +69,15 @@ public final class AAIConstants { public static final String AAI_NODENAME = "aai.config.nodename"; + /* + * Logs the objects being deleted when an client deletes objects via implied delete during PUT operation + */ + public static final String AAI_IMPLIED_DELETE_LOG_ENABLED = "aai.implied.delete.log.enabled"; + /* + * Specifies how many objects maximum to log + */ + public static final String AAI_IMPLIED_DELETE_LOG_LIMIT = "aai.implied.delete.log.limit"; + public static final String AAI_BULKCONSUMER_LIMIT = "aai.bulkconsumer.payloadlimit"; public static final String AAI_BULKCONSUMER_OVERRIDE_LIMIT = "aai.bulkconsumer.payloadoverride"; -- 2.16.6