Improve the performance of resoures microservice
[aai/aai-common.git] / aai-core / src / main / java / org / onap / aai / rest / db / HttpEntry.java
1 /**
2  * ============LICENSE_START=======================================================
3  * org.onap.aai
4  * ================================================================================
5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20 package org.onap.aai.rest.db;
21
22
23 import java.io.IOException;
24 import java.io.UnsupportedEncodingException;
25 import java.lang.reflect.InvocationTargetException;
26 import java.net.MalformedURLException;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.util.ArrayList;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.Set;
34 import java.util.concurrent.TimeUnit;
35 import java.util.stream.Collectors;
36
37 import javax.ws.rs.core.HttpHeaders;
38 import javax.ws.rs.core.MediaType;
39 import javax.ws.rs.core.MultivaluedMap;
40 import javax.ws.rs.core.Response;
41 import javax.ws.rs.core.Response.Status;
42 import javax.ws.rs.core.UriBuilder;
43
44 import com.att.eelf.configuration.EELFLogger;
45 import com.att.eelf.configuration.EELFManager;
46 import com.fasterxml.jackson.databind.JsonNode;
47 import com.fasterxml.jackson.databind.ObjectMapper;
48 import com.github.fge.jsonpatch.JsonPatchException;
49 import com.github.fge.jsonpatch.mergepatch.JsonMergePatch;
50 import org.apache.commons.lang.StringUtils;
51 import org.apache.tinkerpop.gremlin.structure.Graph;
52 import org.apache.tinkerpop.gremlin.structure.Vertex;
53 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
54 import org.janusgraph.core.JanusGraphException;
55 import org.javatuples.Pair;
56 import org.onap.aai.db.props.AAIProperties;
57 import org.onap.aai.dbmap.DBConnectionType;
58 import org.onap.aai.domain.responseMessage.AAIResponseMessage;
59 import org.onap.aai.domain.responseMessage.AAIResponseMessageDatum;
60 import org.onap.aai.exceptions.AAIException;
61 import org.onap.aai.extensions.AAIExtensionMap;
62 import org.onap.aai.extensions.ExtensionController;
63 import org.onap.aai.introspection.*;
64 import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
65 import org.onap.aai.logging.ErrorLogHelper;
66 import org.onap.aai.logging.LoggingContext;
67 import org.onap.aai.nodes.NodeIngestor;
68 import org.onap.aai.parsers.query.QueryParser;
69 import org.onap.aai.parsers.uri.URIToExtensionInformation;
70
71 import org.onap.aai.parsers.uri.URIToObject;
72 import org.onap.aai.rest.ueb.UEBNotification;
73 import org.onap.aai.restcore.HttpMethod;
74 import org.onap.aai.schema.enums.ObjectMetadata;
75 import org.onap.aai.serialization.db.DBSerializer;
76 import org.onap.aai.serialization.engines.JanusGraphDBEngine;
77 import org.onap.aai.serialization.engines.QueryStyle;
78 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
79 import org.onap.aai.serialization.engines.query.QueryEngine;
80 import org.onap.aai.serialization.queryformats.Format;
81 import org.onap.aai.serialization.queryformats.FormatFactory;
82 import org.onap.aai.serialization.queryformats.Formatter;
83 import org.onap.aai.setup.SchemaVersion;
84 import org.onap.aai.setup.SchemaVersions;
85 import org.onap.aai.util.AAIConfig;
86 import org.springframework.beans.factory.annotation.Autowired;
87 import org.springframework.beans.factory.annotation.Value;
88
89 /**
90  * The Class HttpEntry.
91  */
92 public class HttpEntry {
93
94         private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(HttpEntry.class);
95         private static final String TARGET_ENTITY = "DB";
96
97         private  ModelType introspectorFactoryType;
98
99         private  QueryStyle queryStyle;
100
101         private SchemaVersion version;
102
103         private Loader loader;
104
105         private TransactionalGraphEngine dbEngine;
106
107         private boolean processSingle = true;
108
109         private int paginationBucket = -1;
110         private int paginationIndex = -1;
111         private int totalVertices = 0;
112         private int totalPaginationBuckets = 0;
113
114         @Autowired
115         private NodeIngestor nodeIngestor;
116
117         @Autowired
118         private LoaderFactory loaderFactory;
119
120         @Autowired
121         private SchemaVersions schemaVersions;
122
123         @Value("${schema.uri.base.path}")
124         private String basePath;
125
126         private UEBNotification notification;
127
128         /**
129          * Instantiates a new http entry.
130          *
131          * @param modelType the model type
132          * @param queryStyle the query style
133          */
134         public HttpEntry(ModelType modelType, QueryStyle queryStyle) {
135                 this.introspectorFactoryType = modelType;
136                 this.queryStyle = queryStyle;
137         }
138
139         public HttpEntry setHttpEntryProperties(SchemaVersion version, DBConnectionType connectionType){
140                 this.version = version;
141                 this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
142                 this.dbEngine = new JanusGraphDBEngine(
143                                 queryStyle,
144                                 connectionType,
145                                 loader);
146
147                 getDbEngine().startTransaction();
148                 this.notification = new UEBNotification(loader, loaderFactory, schemaVersions);
149                 return this;
150         }
151
152
153          public HttpEntry setHttpEntryProperties(SchemaVersion version, DBConnectionType connectionType, UEBNotification notification){
154                 this.version = version;
155                 this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
156                 this.dbEngine = new JanusGraphDBEngine(
157                                 queryStyle,
158                                 connectionType,
159                                 loader);
160
161                 this.notification = notification;
162                 //start transaction on creation
163                 getDbEngine().startTransaction();
164                 return this;
165         }
166
167
168         /**
169          * Gets the introspector factory type.
170          *
171          * @return the introspector factory type
172          */
173         public ModelType getIntrospectorFactoryType() {
174                 return introspectorFactoryType;
175         }
176
177         /**
178          * Gets the query style.
179          *
180          * @return the query style
181          */
182         public QueryStyle getQueryStyle() {
183                 return queryStyle;
184         }
185
186         /**
187          * Gets the version.
188          *
189          * @return the version
190          */
191         public SchemaVersion getVersion() {
192                 return version;
193         }
194
195         /**
196          * Gets the loader.
197          *
198          * @return the loader
199          */
200         public Loader getLoader() {
201                 return loader;
202         }
203
204         /**
205          * Gets the db engine.
206          *
207          * @return the db engine
208          */
209         public TransactionalGraphEngine getDbEngine() {
210                 return dbEngine;
211         }
212
213         public Pair<Boolean, List<Pair<URI, Response>>> process (List<DBRequest> requests, String sourceOfTruth) throws AAIException {
214                 return this.process(requests, sourceOfTruth, true);
215         }
216
217
218         /**
219          * Checks the pagination bucket and pagination index variables to determine whether or not the user
220          * requested paginated results
221          *
222          * @return a boolean true/false of whether the user requested paginated results
223          */
224         public boolean isPaginated(){
225                 return this.paginationBucket > -1 && this.paginationIndex > -1;
226         }
227
228         /**
229          * Setter for the pagination bucket variable which stores in this object the size of results to return
230          * @param pb
231          */
232         public void setPaginationBucket(int pb){
233                 this.paginationBucket = pb;
234         }
235
236         /**
237          * Returns the pagination size
238          * @return integer of the size of results to be returned when paginated
239          */
240         public int getPaginationBucket(){
241                 return this.paginationBucket;
242         }
243
244         /**
245          * Sets the pagination index that was passed in by the user, to determine which index or results to retrieve when
246          * paginated
247          * @param pi
248          */
249         public void setPaginationIndex(int pi){
250                 if(pi == 0){
251                         pi = 1;
252                 }
253                 this.paginationIndex = pi;
254         }
255
256         /**
257          * Getter to return the pagination index requested by the user when requesting paginated results
258          * @return
259          */
260         public int getPaginationIndex(){
261                 return this.paginationIndex;
262         }
263
264         /**
265          * Sets the total vertices variables and calculates the amount of pages based on size and total vertices
266          * @param totalVertices
267          * @param paginationBucketSize
268          */
269         public void setTotalsForPaging(int totalVertices, int paginationBucketSize){
270                 this.totalVertices = totalVertices;
271                 //set total number of buckets equal to full pages
272                 this.totalPaginationBuckets = totalVertices/paginationBucketSize;
273                 //conditionally add a page for the remainder
274                 if(totalVertices % paginationBucketSize > 0){
275                         this.totalPaginationBuckets++;
276                 }
277         }
278
279         /**
280          * @return the total amount of pages
281          */
282         public int getTotalPaginationBuckets(){
283                 return this.totalPaginationBuckets;
284         }
285
286         /**
287          *
288          * @return the total number of vertices when paginated
289          */
290         public int getTotalVertices(){
291                 return this.totalVertices;
292         }
293
294         /**
295          * Process.
296          * @param requests the requests
297          * @param sourceOfTruth the source of truth
298          *
299          * @return the pair
300          * @throws AAIException the AAI exception
301          */
302         public Pair<Boolean, List<Pair<URI, Response>>> process (List<DBRequest> requests, String sourceOfTruth, boolean enableResourceVersion) throws AAIException {
303
304                 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, sourceOfTruth);
305                 String methodName = "process";
306                 Response response;
307                 Introspector obj = null;
308                 QueryParser query = null;
309                 URI uri = null;
310                 String transactionId = null;
311                 int depth = AAIProperties.MAXIMUM_DEPTH;
312                 Format format = null;
313                 List<Pair<URI,Response>> responses = new ArrayList<>();
314                 MultivaluedMap<String, String> params = null;
315                 HttpMethod method = null;
316                 String uriTemp = "";
317                 Boolean success = true;
318                 QueryEngine queryEngine = dbEngine.getQueryEngine();
319                 int maxRetries = 10;
320                 int retry = 0;
321
322                 LoggingContext.save();
323                 for (DBRequest request : requests) {
324                         response = null;
325                         Status status = Status.NOT_FOUND;
326                         method = request.getMethod();
327                         try {
328                                 for (retry = 0; retry < maxRetries; ++retry) {
329                                         try {
330
331                                                 LoggingContext.targetEntity(TARGET_ENTITY);
332                                                 LoggingContext.targetServiceName(methodName + " " + method);
333
334                                                 obj = request.getIntrospector();
335                                                 query = request.getParser();
336                                                 transactionId = request.getTransactionId();
337                                                 uriTemp = request.getUri().getRawPath().replaceFirst("^v\\d+/", "");
338                                                 uri = UriBuilder.fromPath(uriTemp).build();
339                                                 LoggingContext.startTime();
340                                                 List<Vertex> vertTemp;
341                                                 List<Vertex> vertices;
342                                                 if(this.isPaginated()) {
343                                                         vertTemp = query.getQueryBuilder().toList();
344                                                         this.setTotalsForPaging(vertTemp.size(), this.paginationBucket);
345                                                         vertices = vertTemp.subList(((this.paginationIndex - 1) * this.paginationBucket), Math.min((this.paginationBucket * this.paginationIndex), vertTemp.size()));
346                                                 }else{
347                                                         vertices = query.getQueryBuilder().toList();
348                                                 }
349                                                 boolean isNewVertex = false;
350                                                 String outputMediaType = getMediaType(request.getHeaders().getAcceptableMediaTypes());
351                                                 String result = null;
352                                                 params = request.getInfo().getQueryParameters(false);
353                                                 depth = setDepth(obj, params.getFirst("depth"));
354                                                 if (params.containsKey("format")) {
355                                                         format = Format.getFormat(params.getFirst("format"));
356                                                 }
357                                                 String cleanUp = params.getFirst("cleanup");
358                                                 String requestContext = "";
359                                                 List<String> requestContextList = request.getHeaders().getRequestHeader("aai-request-context");
360                                                 if (requestContextList != null) {
361                                                         requestContext = requestContextList.get(0);
362                                                 }
363
364                                                 if (cleanUp == null) {
365                                                         cleanUp = "false";
366                                                 }
367                                                 if (vertices.size() > 1 && processSingle && !method.equals(HttpMethod.GET)) {
368                                                         if (method.equals(HttpMethod.DELETE)) {
369                                                                 LoggingContext.restoreIfPossible();
370                                                                 throw new AAIException("AAI_6138");
371                                                         } else {
372                                                                 LoggingContext.restoreIfPossible();
373                                                                 throw new AAIException("AAI_6137");
374                                                         }
375                                                 }
376                                                 if (method.equals(HttpMethod.PUT)) {
377                                                         String resourceVersion = (String)obj.getValue("resource-version");
378                                                         if (vertices.isEmpty()) {
379                                                                 if (enableResourceVersion) {
380                                                                         serializer.verifyResourceVersion("create", query.getResultType(), "", resourceVersion, obj.getURI());
381                                                                 }
382                                                                 isNewVertex = true;
383                                                         } else {
384                                                                 if (enableResourceVersion) {
385                                                                         serializer.verifyResourceVersion("update", query.getResultType(), vertices.get(0).<String>property("resource-version").orElse(null), resourceVersion, obj.getURI());
386                                                                 }
387                                                                 isNewVertex = false;
388                                                         }
389                                                 } else {
390                                                         if (vertices.isEmpty()) {
391                                                                 String msg = createNotFoundMessage(query.getResultType(), request.getUri());
392                                                                 throw new AAIException("AAI_6114", msg);
393                                                         } else {
394                                                                 isNewVertex = false;
395                                                         }
396                                                 }
397                                                 Vertex v = null;
398                                                 if (!isNewVertex) {
399                                                         v = vertices.get(0);
400                                                 }
401                                                 HashMap<String, Introspector> relatedObjects = new HashMap<>();
402                                                 switch (method) {
403                                                         case GET:
404                                                                 String nodeOnly = params.getFirst("nodes-only");
405                                                                 boolean isNodeOnly = nodeOnly != null;
406
407                                                                 if (format == null) {
408                                                                         obj = this.getObjectFromDb(vertices, serializer, query, obj, request.getUri(), depth, isNodeOnly, cleanUp);
409
410
411                                                                         LoggingContext.elapsedTime((long) serializer.getDBTimeMsecs(), TimeUnit.MILLISECONDS);
412                                                                         LOGGER.info ("Completed");
413                                                                         LoggingContext.restoreIfPossible();
414
415                                                                         if (obj != null) {
416                                                                                 status = Status.OK;
417                                                                                 MarshallerProperties properties;
418                                                                                 if (!request.getMarshallerProperties().isPresent()) {
419                                                                                         properties = new MarshallerProperties.Builder(org.onap.aai.restcore.MediaType.getEnum(outputMediaType)).build();
420                                                                                 } else {
421                                                                                         properties = request.getMarshallerProperties().get();
422                                                                                 }
423                                                                                 result = obj.marshal(properties);
424                                                                         }
425                                                                 } else {
426                                                                         FormatFactory ff = new FormatFactory(loader, serializer, schemaVersions, basePath + "/");
427                                                                         Formatter formatter =  ff.get(format, params);
428                                                                         result = formatter.output(vertices.stream().map(vertex -> (Object) vertex).collect(Collectors.toList())).toString();
429                                                                         status = Status.OK;
430                                                                 }
431
432                                                                 break;
433                                                         case PUT:
434                                                                 response = this.invokeExtension(dbEngine, this.dbEngine.tx(), method, request, sourceOfTruth, version, loader, obj, uri, true);
435                                                                 if (isNewVertex) {
436                                                                         v = serializer.createNewVertex(obj);
437                                                                 }
438                                                                 serializer.serializeToDb(obj, v, query, uri.getRawPath(), requestContext);
439                                                                 this.invokeExtension(dbEngine, this.dbEngine.tx(), HttpMethod.PUT, request, sourceOfTruth, version, loader, obj, uri, false);
440                                                                 status = Status.OK;
441                                                                 if (isNewVertex) {
442                                                                         status = Status.CREATED;
443                                                                 }
444                                                                 obj = serializer.getLatestVersionView(v);
445                                                                 if (query.isDependent()) {
446                                                                         relatedObjects = this.getRelatedObjects(serializer, queryEngine, v, obj, this.loader);
447                                                                 }
448                                                                 LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs() +
449                                                                                 (long)queryEngine.getDBTimeMsecs(), TimeUnit.MILLISECONDS);
450                                                                 LOGGER.info ("Completed ");
451                                                                 LoggingContext.restoreIfPossible();
452                                                                 notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects, basePath);
453
454                                                                 break;
455                                                         case PUT_EDGE:
456                                                                 serializer.touchStandardVertexProperties(v, false);
457                                                                 this.invokeExtension(dbEngine, this.dbEngine.tx(), method, request, sourceOfTruth, version, loader, obj, uri, true);
458                                                                 serializer.createEdge(obj, v);
459
460                                                                 LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs(),TimeUnit.MILLISECONDS);
461                                                                 LOGGER.info ("Completed");
462                                                                 LoggingContext.restoreIfPossible();
463                                                                 status = Status.OK;
464                                                                 notification.createNotificationEvent(transactionId, sourceOfTruth, status, new URI(uri.toString().replace("/relationship-list/relationship", "")), serializer.getLatestVersionView(v), relatedObjects, basePath);
465                                                                 break;
466                                                         case MERGE_PATCH:
467                                                                 Introspector existingObj  = loader.introspectorFromName(obj.getDbName());
468                                                                 existingObj = this.getObjectFromDb(vertices, serializer, query, existingObj, request.getUri(), 0, false, cleanUp);
469                                                                 String existingJson = existingObj.marshal(false);
470                                                                 String newJson;
471
472                                                                 if (request.getRawRequestContent().isPresent()) {
473                                                                         newJson = request.getRawRequestContent().get();
474                                                                 } else {
475                                                                         newJson = "";
476                                                                 }
477                                                                 Object relationshipList = request.getIntrospector().getValue("relationship-list");
478                                                                 ObjectMapper mapper = new ObjectMapper();
479                                                                 try {
480                                                                         JsonNode existingNode = mapper.readTree(existingJson);
481                                                                         JsonNode newNode = mapper.readTree(newJson);
482                                                                         JsonMergePatch patch = JsonMergePatch.fromJson(newNode);
483                                                                         JsonNode completed = patch.apply(existingNode);
484                                                                         String patched = mapper.writeValueAsString(completed);
485                                                                         Introspector patchedObj = loader.unmarshal(existingObj.getName(), patched);
486                                                                         if (relationshipList == null) {
487                                                                                 //if the caller didn't touch the relationship-list, we shouldn't either
488                                                                                 patchedObj.setValue("relationship-list", null);
489                                                                         }
490                                                                         serializer.serializeToDb(patchedObj, v, query, uri.getRawPath(), requestContext);
491                                                                         status = Status.OK;
492                                                                         patchedObj = serializer.getLatestVersionView(v);
493                                                                         if (query.isDependent()) {
494                                                                                 relatedObjects = this.getRelatedObjects(serializer, queryEngine, v, patchedObj, this.loader);
495                                                                         }
496                                                                         LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs() +
497                                                                                         (long)queryEngine.getDBTimeMsecs(), TimeUnit.MILLISECONDS);
498                                                                         LOGGER.info ("Completed");
499                                                                         LoggingContext.restoreIfPossible();
500                                                                         notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, patchedObj, relatedObjects, basePath);
501                                                                 } catch (IOException | JsonPatchException e) {
502
503                                                                         LOGGER.info ("Caught exception: " + e.getMessage());
504                                                                         LoggingContext.restoreIfPossible();
505                                                                         throw new AAIException("AAI_3000", "could not perform patch operation");
506                                                                 }
507                                                                 break;
508                                                         case DELETE:
509                                                                 String resourceVersion = params.getFirst("resource-version");
510                                                                 obj = serializer.getLatestVersionView(v);
511                                                                 if (query.isDependent()) {
512                                                                         relatedObjects = this.getRelatedObjects(serializer, queryEngine, v, obj, this.loader);
513                                                                 }
514                                                                 /*
515                                                                  * Find all Delete-other-vertex vertices and create structure for notify
516                                                                  * findDeleatble also returns the startVertex v and we dont want to create
517                                                                  * duplicate notification events for the same
518                                                                  * So remove the startvertex first
519                                                                  */
520
521                                                                 List<Vertex> deletableVertices = dbEngine.getQueryEngine().findDeletable(v);
522                                                                 Long vId = (Long) v.id();
523
524                                                                 /*
525                                                                  * I am assuming vertexId cant be null
526                                                                  */
527                                                                 deletableVertices.removeIf(s -> vId.equals(s.id()));
528                                                                 boolean isDelVerticesPresent = !deletableVertices.isEmpty();
529                                                                 Map<Vertex, Introspector> deleteObjects = new HashMap<>();
530                                                                 Map<String, URI> uriMap = new HashMap<>();
531                                                                 Map<String, HashMap<String, Introspector>> deleteRelatedObjects = new HashMap<>();
532
533                                                                 if(isDelVerticesPresent){
534                                                                         deleteObjects = this.buildIntrospectorObjects(serializer, deletableVertices);
535
536                                                                         uriMap = this.buildURIMap(serializer, deleteObjects);
537                                                                         deleteRelatedObjects = this.buildRelatedObjects(serializer, queryEngine, deleteObjects);
538                                                                 }
539
540                                                                 this.invokeExtension(dbEngine, this.dbEngine.tx(), method, request, sourceOfTruth, version, loader, obj, uri, true);
541                                                                 serializer.delete(v, deletableVertices, resourceVersion, enableResourceVersion);
542                                                                 this.invokeExtension(dbEngine, this.dbEngine.tx(), method, request, sourceOfTruth, version, loader, obj, uri, false);
543
544                                                                 LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs() +
545                                                                                 (long)queryEngine.getDBTimeMsecs(), TimeUnit.MILLISECONDS);
546                                                                 LOGGER.info ("Completed");
547                                                                 LoggingContext.restoreIfPossible();
548                                                                 status = Status.NO_CONTENT;
549                                                                 notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects, basePath);
550
551                                                                 /*
552                                                                  * Notify delete-other-v candidates
553                                                                  */
554
555                                                                 if(isDelVerticesPresent){
556                                                                         this.buildNotificationEvent(sourceOfTruth, status, transactionId, notification, deleteObjects,
557                                                                         uriMap, deleteRelatedObjects, basePath);
558                                                                 }
559
560                                                                 break;
561                                                         case DELETE_EDGE:
562                                                                 serializer.touchStandardVertexProperties(v, false);
563                                                                 serializer.deleteEdge(obj, v);
564
565                                                                 LoggingContext.elapsedTime((long)serializer.getDBTimeMsecs(),TimeUnit.MILLISECONDS);
566                                                                 LOGGER.info ("Completed");
567                                                                 LoggingContext.restoreIfPossible();
568                                                                 status = Status.NO_CONTENT;
569                                                                 notification.createNotificationEvent(transactionId, sourceOfTruth, Status.OK, new URI(uri.toString().replace("/relationship-list/relationship", "")), serializer.getLatestVersionView(v), relatedObjects, basePath);
570                                                                 break;
571                                                         default:
572                                                                 break;
573                                                 }
574
575
576                                                 /* temporarily adding vertex id to the headers
577                                                  * to be able to use for testing the vertex id endpoint functionality
578                                                  * since we presently have no other way of generating those id urls
579                                                 */
580                                                 if (response == null && v != null && (
581                                                         method.equals(HttpMethod.PUT)
582                                                         || method.equals(HttpMethod.GET)
583                                                         || method.equals(HttpMethod.MERGE_PATCH))
584                                                 ) {
585                                                         String myvertid = v.id().toString();
586                                                         if(this.isPaginated()){
587                                                                 response = Response.status(status)
588                                                                                 .header("vertex-id", myvertid)
589                                                                                 .header("total-results", this.getTotalVertices())
590                                                                                 .header("total-pages", this.getTotalPaginationBuckets())
591                                                                                 .entity(result)
592                                                                                 .type(outputMediaType).build();
593                                                         }else {
594                                                                 response = Response.status(status)
595                                                                                 .header("vertex-id", myvertid)
596                                                                                 .entity(result)
597                                                                                 .type(outputMediaType).build();
598                                                         }
599                                                 } else if (response == null) {
600                                                         response = Response.status(status)
601                                                                         .type(outputMediaType).build();
602                                                 } else {
603                                                         //response already set to something
604                                                 }
605                                                 Pair<URI,Response> pairedResp = Pair.with(request.getUri(), response);
606                                                 responses.add(pairedResp);
607                                                 //break out of retry loop
608                                                 break;
609                                         } catch (JanusGraphException e) {
610                                                 this.dbEngine.rollback();
611
612                                                 LOGGER.info ("Caught exception: " + e.getMessage());
613                                                 LoggingContext.restoreIfPossible();
614                                                 AAIException ex = new AAIException("AAI_6142", e);
615                                                 ErrorLogHelper.logException(ex);
616                                                 Thread.sleep((retry + 1) * 20L);
617                                                 this.dbEngine.startTransaction();
618                                                 queryEngine = dbEngine.getQueryEngine();
619                                                 serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, sourceOfTruth);
620                                         }
621                                         if (retry == maxRetries) {
622                                                 throw new AAIException("AAI_6134");
623                                         }
624                                 }
625                         } catch (AAIException e) {
626                                 success = false;
627                                 ArrayList<String> templateVars = new ArrayList<>();
628                                 templateVars.add(request.getMethod().toString()); //GET, PUT, etc
629                                 templateVars.add(request.getUri().getPath());
630                                 templateVars.addAll(e.getTemplateVars());
631                                 ErrorLogHelper.logException(e);
632                                 response = Response
633                                                 .status(e.getErrorObject().getHTTPResponseCode())
634                                                 .entity(ErrorLogHelper.getRESTAPIErrorResponse(request.getHeaders().getAcceptableMediaTypes(), e, templateVars))
635                                                 .build();
636                                 Pair<URI,Response> pairedResp = Pair.with(request.getUri(), response);
637                                 responses.add(pairedResp);
638                                 continue;
639                         } catch (Exception e) {
640                                 success = false;
641                                 e.printStackTrace();
642                                 AAIException ex = new AAIException("AAI_4000", e);
643                                 ArrayList<String> templateVars = new ArrayList<String>();
644                                 templateVars.add(request.getMethod().toString()); //GET, PUT, etc
645                                 templateVars.add(request.getUri().getPath().toString());
646                                 ErrorLogHelper.logException(ex);
647                                 response = Response
648                                                 .status(ex.getErrorObject().getHTTPResponseCode())
649                                                 .entity(ErrorLogHelper.getRESTAPIErrorResponse(request.getHeaders().getAcceptableMediaTypes(), ex, templateVars))
650                                                 .build();
651                                 Pair<URI, Response> pairedResp = Pair.with(request.getUri(), response);
652                                 responses.add(pairedResp);
653                                 continue;
654                         }
655                 }
656                 notification.triggerEvents();
657                 return Pair.with(success, responses);
658         }
659
660
661         /**
662          * Gets the media type.
663          *
664          * @param mediaTypeList the media type list
665          * @return the media type
666          */
667         private String getMediaType(List <MediaType> mediaTypeList) {
668                 String mediaType = MediaType.APPLICATION_JSON;  // json is the default
669                 for (MediaType mt : mediaTypeList) {
670                         if (MediaType.APPLICATION_XML_TYPE.isCompatible(mt)) {
671                                 mediaType = MediaType.APPLICATION_XML;
672                         }
673                 }
674                 return mediaType;
675         }
676
677         /**
678          * Gets the object from db.
679          *
680          * @param serializer the serializer
681          * @param query the query
682          * @param obj the obj
683          * @param uri the uri
684          * @param depth the depth
685          * @param cleanUp the clean up
686          * @return the object from db
687          * @throws AAIException the AAI exception
688          * @throws IllegalAccessException the illegal access exception
689          * @throws IllegalArgumentException the illegal argument exception
690          * @throws InvocationTargetException the invocation target exception
691          * @throws SecurityException the security exception
692          * @throws InstantiationException the instantiation exception
693          * @throws NoSuchMethodException the no such method exception
694          * @throws UnsupportedEncodingException the unsupported encoding exception
695          * @throws MalformedURLException the malformed URL exception
696          * @throws AAIUnknownObjectException
697          * @throws URISyntaxException
698          */
699         private Introspector getObjectFromDb(List<Vertex> results, DBSerializer serializer, QueryParser query, Introspector obj, URI uri, int depth, boolean nodeOnly, String cleanUp) throws AAIException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, SecurityException, InstantiationException, NoSuchMethodException, UnsupportedEncodingException, AAIUnknownObjectException, URISyntaxException {
700
701                 //nothing found
702                 if (results.isEmpty()) {
703                         String msg = createNotFoundMessage(query.getResultType(), uri);
704                         throw new AAIException("AAI_6114", msg);
705                 }
706
707                 return serializer.dbToObject(results, obj, depth, nodeOnly, cleanUp);
708
709         }
710
711         /**
712          * Invoke extension.
713          *
714          * @param dbEngine the db engine
715          * @param g the g
716          * @param httpMethod the http method
717          * @param fromAppId the from app id
718          * @param apiVersion the api version
719          * @param loader the loader
720          * @param obj the obj
721          * @param uri the uri
722          * @param isPreprocess the is preprocess
723          * @return the response
724          * @throws IllegalArgumentException the illegal argument exception
725          * @throws UnsupportedEncodingException the unsupported encoding exception
726          * @throws AAIException the AAI exception
727          */
728         private Response invokeExtension(TransactionalGraphEngine dbEngine, Graph g, HttpMethod httpMethod, DBRequest request, String fromAppId, SchemaVersion apiVersion, Loader loader, Introspector obj, URI uri, boolean isPreprocess) throws IllegalArgumentException, UnsupportedEncodingException, AAIException {
729                 AAIExtensionMap aaiExtMap = new AAIExtensionMap();
730                 //ModelInjestor injestor = ModelInjestor.getInstance();
731                 Response response = null;
732                 URIToExtensionInformation extensionInformation = new URIToExtensionInformation(loader, uri);
733                 aaiExtMap.setHttpEntry(this);
734                 aaiExtMap.setDbRequest(request);
735                 aaiExtMap.setTransId(request.getTransactionId());
736                 aaiExtMap.setFromAppId(fromAppId);
737                 aaiExtMap.setGraph(g);
738                 aaiExtMap.setApiVersion(apiVersion.toString());
739                 aaiExtMap.setObjectFromRequest(obj);
740                 aaiExtMap.setObjectFromRequestType(obj.getJavaClassName());
741                 aaiExtMap.setObjectFromResponse(obj);
742                 aaiExtMap.setObjectFromResponseType(obj.getJavaClassName());
743                 aaiExtMap.setJaxbContext(nodeIngestor.getContextForVersion(apiVersion));
744                 aaiExtMap.setUri(uri.getRawPath());
745                 aaiExtMap.setTransactionalGraphEngine(dbEngine);
746                 aaiExtMap.setLoader(loader);
747                 aaiExtMap.setNamespace(extensionInformation.getNamespace());
748
749                 ExtensionController ext = new ExtensionController();
750                 ext.runExtension(aaiExtMap.getApiVersion(),
751                                 extensionInformation.getNamespace(),
752                                 extensionInformation.getTopObject(),
753                                 extensionInformation.getMethodName(httpMethod, isPreprocess),
754                                 aaiExtMap,
755                                 isPreprocess);
756
757                 if (aaiExtMap.getPrecheckAddedList().size() > 0) {
758                         response = notifyOnSkeletonCreation(aaiExtMap, obj, request.getHeaders());
759                 }
760
761                 return response;
762         }
763
764         /**
765          * Notify on skeleton creation.
766          *
767          * @param aaiExtMap the aai ext map
768          * @param input the input
769          * @param headers the headers
770          * @return the response
771          */
772         //Legacy support
773         private Response notifyOnSkeletonCreation(AAIExtensionMap aaiExtMap, Introspector input, HttpHeaders headers) {
774                 Response response = null;
775                 HashMap<AAIException, ArrayList<String>> exceptionList = new HashMap<AAIException, ArrayList<String>>();
776
777                 StringBuilder keyString = new StringBuilder();
778
779                 Set<String> resourceKeys = input.getKeys();
780                 for (String key : resourceKeys) {
781                         keyString.append(key).append("=").append(input.getValue(key).toString()).append(" ");
782                 }
783
784                 for (AAIResponseMessage msg : aaiExtMap.getPrecheckResponseMessages().getAAIResponseMessage()) {
785                         ArrayList<String> templateVars = new ArrayList<>();
786
787                         templateVars.add("PUT " + input.getDbName());
788                         templateVars.add(keyString.toString());
789                         List<String> keys = new ArrayList<>();
790                         templateVars.add(msg.getAaiResponseMessageResourceType());
791                         for (AAIResponseMessageDatum dat : msg.getAaiResponseMessageData().getAAIResponseMessageDatum()) {
792                                 keys.add(dat.getAaiResponseMessageDatumKey() + "=" + dat.getAaiResponseMessageDatumValue());
793                         }
794                         templateVars.add(StringUtils.join(keys, ", "));
795                         exceptionList.put(new AAIException("AAI_0004", msg.getAaiResponseMessageResourceType()),
796                                         templateVars);
797                 }
798                 response = Response
799                                 .status(Status.ACCEPTED).entity(ErrorLogHelper
800                                                 .getRESTAPIInfoResponse(headers.getAcceptableMediaTypes(), exceptionList))
801                                                 .build();
802
803                 return response;
804         }
805
806         /**
807          * Creates the not found message.
808          *
809          * @param resultType the result type
810          * @param uri the uri
811          * @return the string
812          */
813         private String createNotFoundMessage(String resultType, URI uri) {
814
815                 String msg = "No Node of type " + resultType + " found at: " + uri.getPath();
816
817                 return msg;
818         }
819
820         /**
821          * Sets the depth.
822          *
823          * @param depthParam the depth param
824          * @return the int
825          * @throws AAIException the AAI exception
826          */
827         protected int setDepth(Introspector obj, String depthParam) throws AAIException {
828                 int depth = AAIProperties.MAXIMUM_DEPTH;
829
830                 String getAllRandomStr = AAIConfig.get("aai.rest.getall.depthparam", "");
831                 if (depthParam != null && getAllRandomStr != null && !getAllRandomStr.isEmpty()
832                                 && getAllRandomStr.equals(depthParam)) {
833                         return depth;
834                 }
835
836                 if(depthParam == null){
837                         if(this.version.compareTo(schemaVersions.getDepthVersion()) >= 0){
838                                 depth = 0;
839                         } else {
840                                 depth = AAIProperties.MAXIMUM_DEPTH;
841                         }
842                 } else {
843                         if (!depthParam.isEmpty() && !"all".equals(depthParam)){
844                                 try {
845                                         depth = Integer.parseInt(depthParam);
846                                 } catch (Exception e) {
847                                         throw new AAIException("AAI_4016");
848                                 }
849
850                         }
851                 }
852                 String maxDepth = obj.getMetadata(ObjectMetadata.MAXIMUM_DEPTH);
853
854                 int maximumDepth = AAIProperties.MAXIMUM_DEPTH;
855
856                 if(maxDepth != null){
857                         try {
858                                 maximumDepth = Integer.parseInt(maxDepth);
859                         } catch(Exception ex){
860                                 throw new AAIException("AAI_4018");
861                         }
862                 }
863
864                 if(depth > maximumDepth){
865                         throw new AAIException("AAI_3303");
866                 }
867
868                 return depth;
869         }
870
871         /**
872          * Checks if is modification method.
873          *
874          * @param method the method
875          * @return true, if is modification method
876          */
877         private boolean isModificationMethod(HttpMethod method) {
878                 boolean result = false;
879
880                 if (method.equals(HttpMethod.PUT) || method.equals(HttpMethod.PUT_EDGE) || method.equals(HttpMethod.DELETE_EDGE) || method.equals(HttpMethod.MERGE_PATCH)) {
881                         result = true;
882                 }
883
884                 return result;
885
886         }
887
888         /**
889          * Given an uri, introspector object and loader object
890          * it will check if the obj is top level object if it is,
891          * it will return immediately returning the uri passed in
892          * If it isn't, it will go through, get the uriTemplate
893          * from the introspector object and get the count of "/"s
894          * and remove that part of the uri using substring
895          * and keep doing that until the current object is top level
896          * Also added the max depth just so worst case scenario
897          * Then keep adding aai-uri to the list include the aai-uri passed in
898          * Convert that list into an array and return it
899          * <p>
900          *
901          * Example:
902          *
903          * <blockquote>
904          * aai-uri -> /cloud-infrastructure/cloud-regions/cloud-region/cloud-owner/cloud-region-id/tenants/tenant/tenant1/vservers/vserver/v1
905          *
906          * Given the uriTemplate vserver -> /vservers/vserver/{vserver-id}
907          * it converts to /vservers/vserver
908          *
909          * lastIndexOf /vservers/vserver in /cloud-infrastructure/cloud-regions/cloud-region/cloud-owner/cloud-region-id/tenants/tenant/tenant1/vservers/vserver/v1
910          *                                                                                                                                                                                                                                                                      ^
911          *                                                                                                                                                                                                                                                                      |
912          *                                                                                                                                                                                                                                                                      |
913          *                                                                                                                                                                                                                                                              lastIndexOf
914          * Use substring to get the string from 0 to that lastIndexOf
915          * aai-uri -> /cloud-infrastructure/cloud-regions/cloud-region/cloud-owner/cloud-region-id/tenants/tenant/tenant1
916          *
917          * From this new aai-uri, generate a introspector from the URITOObject class
918          * and keep doing this until you
919          *
920          * </blockquote>
921          *
922          * @param aaiUri - aai-uri of the vertex representating the unique id of a given vertex
923          * @param obj   - introspector object of the given starting vertex
924          * @param loader - Type of loader which will always be MoxyLoader to support model driven
925          * @return an array of strings which can be used to get the vertexes of parent and grand parents from a given vertex
926          * @throws UnsupportedEncodingException
927          * @throws AAIException
928          */
929         String[] convertIntrospectorToUriList(String aaiUri, Introspector obj, Loader loader) throws UnsupportedEncodingException, AAIException {
930
931                 List<String> uriList = new ArrayList<>();
932                 String template = StringUtils.EMPTY;
933                 String truncatedUri = aaiUri;
934                 int depth = AAIProperties.MAXIMUM_DEPTH;
935                 uriList.add(truncatedUri);
936
937                 while (depth >= 0 && !obj.isTopLevel()) {
938                         template = obj.getMetadata(ObjectMetadata.URI_TEMPLATE);
939
940                         if(template == null){
941                                 LOGGER.warn("Unable to find the uriTemplate for the object {}", obj.getDbName());
942                                 return null;
943                         }
944
945                         int templateCount = StringUtils.countMatches(template, "/");
946                         int truncatedUriCount = StringUtils.countMatches(truncatedUri, "/");
947
948                         if(templateCount > truncatedUriCount){
949                                 LOGGER.warn("Template uri {} contains more slashes than truncatedUri {}", template, truncatedUri);
950                                 return null;
951                         }
952
953                         int cutIndex = StringUtils.ordinalIndexOf(truncatedUri, "/", truncatedUriCount-templateCount+1);
954                         truncatedUri = StringUtils.substring(truncatedUri, 0, cutIndex);
955                         uriList.add(truncatedUri);
956                         obj = new URIToObject(loader, UriBuilder.fromPath(truncatedUri).build()).getEntity();
957                         depth--;
958                 }
959
960                 return uriList.toArray(new String[uriList.size()]);
961         }
962
963         /**
964          *
965          * @param serializer
966          * @param queryEngine
967          * @param v
968          * @param obj
969          * @param loader
970          * @return
971          * @throws IllegalAccessException
972          * @throws IllegalArgumentException
973          * @throws InvocationTargetException
974          * @throws SecurityException
975          * @throws InstantiationException
976          * @throws NoSuchMethodException
977          * @throws UnsupportedEncodingException
978          * @throws AAIException
979          * @throws URISyntaxException
980          */
981         private HashMap<String, Introspector> getRelatedObjects(DBSerializer serializer, QueryEngine queryEngine, Vertex v, Introspector obj, Loader loader) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException, SecurityException, InstantiationException, NoSuchMethodException, UnsupportedEncodingException, AAIException, URISyntaxException {
982
983                 HashMap<String, Introspector> relatedVertices = new HashMap<>();
984                 VertexProperty aaiUriProperty = v.property(AAIProperties.AAI_URI);
985
986                 if(!aaiUriProperty.isPresent()){
987                         if(LOGGER.isDebugEnabled()){
988                                 LOGGER.debug("For the given vertex {}, it seems aai-uri is not present so not getting related objects", v.id().toString());
989                         } else {
990                                 LOGGER.info("It seems aai-uri is not present in vertex, so not getting related objects, for more info enable debug log");
991                         }
992                         return relatedVertices;
993                 }
994
995                 String aaiUri = aaiUriProperty.value().toString();
996
997                 if(!obj.isTopLevel()){
998                         String[] uriList = convertIntrospectorToUriList(aaiUri, obj, loader);
999                         List<Vertex> vertexChain = null;
1000                         // If the uriList is null then there is something wrong with converting the uri
1001                         // into a list of aai-uris so falling back to the old mechanism for finding parents
1002                         if(uriList == null){
1003                                 LOGGER.info("Falling back to the old mechanism due to unable to convert aai-uri to list of uris but this is not optimal");
1004                                 vertexChain = queryEngine.findParents(v);
1005                         } else {
1006                                 vertexChain = queryEngine.findParents(uriList);
1007                         }
1008                         for(Vertex vertex : vertexChain){
1009                                 try {
1010                                         final Introspector vertexObj = serializer.getVertexProperties(vertex);
1011                                         relatedVertices.put(vertexObj.getObjectId(), vertexObj);
1012                                 } catch (AAIUnknownObjectException e) {
1013                                         LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
1014                                 }
1015                         }
1016                 } else {
1017                         try {
1018                                 final Introspector vertexObj = serializer.getVertexProperties(v);
1019                                 relatedVertices.put(vertexObj.getObjectId(), vertexObj);
1020                         } catch (AAIUnknownObjectException e) {
1021                                 LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
1022                         }
1023                 }
1024
1025                 return relatedVertices;
1026         }
1027
1028         private Map<Vertex, Introspector> buildIntrospectorObjects(DBSerializer serializer, Iterable<Vertex> vertices) {
1029                 Map<Vertex, Introspector> deleteObjectMap = new HashMap<>();
1030                 for (Vertex vertex : vertices) {
1031                         try {
1032                                 // deleteObjectMap.computeIfAbsent(vertex, s ->
1033                                 // serializer.getLatestVersionView(vertex));
1034                                 Introspector deleteObj = serializer.getLatestVersionView(vertex);
1035                                 deleteObjectMap.put(vertex, deleteObj);
1036                         } catch (UnsupportedEncodingException | AAIException e) {
1037                                 LOGGER.warn("Unable to get Introspctor Objects, Just continue");
1038                                 continue;
1039                         }
1040
1041                 }
1042
1043                 return deleteObjectMap;
1044
1045         }
1046
1047         private Map<String, URI> buildURIMap(DBSerializer serializer, Map<Vertex, Introspector> introSpector) {
1048                 Map<String, URI> uriMap = new HashMap<>();
1049                 for (Map.Entry<Vertex, Introspector> entry : introSpector.entrySet()) {
1050                         URI uri;
1051                         try {
1052                                 uri = serializer.getURIForVertex(entry.getKey());
1053                                 if (null != entry.getValue())
1054                                         uriMap.put(entry.getValue().getObjectId(), uri);
1055                         } catch (UnsupportedEncodingException e) {
1056                                 LOGGER.warn("Unable to get URIs, Just continue");
1057                                 continue;
1058                         }
1059
1060                 }
1061
1062                 return uriMap;
1063
1064         }
1065
1066         private Map<String, HashMap<String, Introspector>> buildRelatedObjects(DBSerializer serializer,
1067                         QueryEngine queryEngine, Map<Vertex, Introspector> introSpector) {
1068
1069                 Map<String, HashMap<String, Introspector>> relatedObjectsMap = new HashMap<>();
1070                 for (Map.Entry<Vertex, Introspector> entry : introSpector.entrySet()) {
1071                         try {
1072                                 HashMap<String, Introspector> relatedObjects = this.getRelatedObjects(serializer, queryEngine,
1073                                                 entry.getKey(), entry.getValue(), this.loader);
1074                                 if (null != entry.getValue())
1075                                         relatedObjectsMap.put(entry.getValue().getObjectId(), relatedObjects);
1076                         } catch (IllegalAccessException | IllegalArgumentException
1077                                         | InvocationTargetException | SecurityException | InstantiationException | NoSuchMethodException
1078                                         | UnsupportedEncodingException | AAIException | URISyntaxException e) {
1079                                 LOGGER.warn("Unable to get realted Objects, Just continue");
1080                                 continue;
1081                         }
1082
1083                 }
1084
1085                 return relatedObjectsMap;
1086
1087         }
1088
1089         private void buildNotificationEvent(String sourceOfTruth, Status status, String transactionId,
1090                         UEBNotification notification, Map<Vertex, Introspector> deleteObjects, Map<String, URI> uriMap,
1091                         Map<String, HashMap<String, Introspector>> deleteRelatedObjects, String basePath) {
1092                 for (Map.Entry<Vertex, Introspector> entry : deleteObjects.entrySet()) {
1093                         try {
1094                                 String vertexObjectId = "";
1095
1096                                 if (null != entry.getValue()) {
1097                                         vertexObjectId = entry.getValue().getObjectId();
1098
1099                                         if (uriMap.containsKey(vertexObjectId) && deleteRelatedObjects.containsKey(vertexObjectId))
1100                                                 notification.createNotificationEvent(transactionId, sourceOfTruth, status,
1101                                                                 uriMap.get(vertexObjectId), entry.getValue(), deleteRelatedObjects.get(vertexObjectId), basePath);
1102                                 }
1103                         } catch (UnsupportedEncodingException | AAIException e) {
1104
1105                                 LOGGER.warn("Error in sending notification");
1106                         }
1107                 }
1108         }
1109
1110         public void setPaginationParameters(String resultIndex, String resultSize) {
1111                 if (resultIndex != null && resultIndex != "-1" && resultSize != null && resultSize != "-1") {
1112                         this.setPaginationIndex(Integer.parseInt(resultIndex));
1113                         this.setPaginationBucket(Integer.parseInt(resultSize));
1114                 }
1115         }
1116
1117         public List<Object> getPaginatedVertexList(List<Object> vertexList) throws AAIException{
1118                 List<Object> vertices;
1119                 if(this.isPaginated()) {
1120                         this.setTotalsForPaging(vertexList.size(), this.getPaginationBucket());
1121                         int startIndex = (this.getPaginationIndex() - 1) * this.getPaginationBucket();
1122                         int endIndex = Math.min((this.getPaginationBucket() * this.getPaginationIndex()), vertexList.size());
1123                         if(startIndex > endIndex){
1124                                 throw new AAIException("AAI_6150"," ResultIndex is not appropriate for the result set, Needs to be <= "+ endIndex);
1125                         }
1126                         vertices = vertexList.subList(startIndex, endIndex);
1127                 }else{
1128                         vertices = vertexList;
1129                 }
1130                 return vertices;
1131         }
1132 }