Performance Improvements for Gizmo bulk API
[aai/champ.git] / champ-service / src / main / java / org / onap / champ / service / ChampDataService.java
1 /**
2  * ============LICENSE_START==========================================
3  * org.onap.aai
4  * ===================================================================
5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6  * Copyright © 2017-2018 Amdocs
7  * ===================================================================
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  *
12  *     http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  * ============LICENSE_END============================================
20  */
21 package org.onap.champ.service;
22
23 import org.onap.aai.champcore.ChampGraph;
24 import org.onap.aai.champcore.ChampTransaction;
25 import org.onap.aai.champcore.exceptions.ChampMarshallingException;
26 import org.onap.aai.champcore.exceptions.ChampObjectNotExistsException;
27 import org.onap.aai.champcore.exceptions.ChampRelationshipNotExistsException;
28 import org.onap.aai.champcore.exceptions.ChampSchemaViolationException;
29 import org.onap.aai.champcore.exceptions.ChampTransactionException;
30 import org.onap.aai.champcore.exceptions.ChampUnmarshallingException;
31 import org.onap.aai.champcore.model.ChampElement;
32 import org.onap.aai.champcore.model.ChampField;
33 import org.onap.aai.champcore.model.ChampObject;
34 import org.onap.aai.champcore.model.ChampObjectIndex;
35 import org.onap.aai.champcore.model.ChampRelationship;
36 import org.onap.aai.champcore.model.fluent.object.ObjectBuildOrPropertiesStep;
37 import org.onap.aai.cl.api.Logger;
38 import org.onap.aai.cl.eelf.LoggerFactory;
39 import org.onap.champ.entity.ChampBulkEdgeResponse;
40 import org.onap.champ.entity.ChampBulkOp;
41 import org.onap.champ.entity.ChampBulkPayload;
42 import org.onap.champ.entity.ChampBulkResponse;
43 import org.onap.champ.entity.ChampBulkVertexResponse;
44 import org.onap.champ.exception.ChampServiceException;
45 import org.onap.champ.service.logging.ChampMsgs;
46 import org.onap.champ.util.ChampProperties;
47 import org.onap.champ.util.ChampServiceConstants;
48
49 import java.util.ArrayList;
50 import java.util.HashMap;
51 import java.util.HashSet;
52 import java.util.List;
53 import java.util.Map;
54 import java.util.Optional;
55 import java.util.stream.Collectors;
56 import java.util.stream.Stream;
57 import javax.ws.rs.core.Response.Status;
58
59 public class ChampDataService {
60   private ChampUUIDService champUUIDService;
61
62   private ChampGraph graphImpl;
63   private ChampTransactionCache cache;
64   private static final String KEY_NAME = ChampProperties.get(ChampServiceConstants.CHAMP_KEY_NAME);
65   private static final String SOT_NAME = ChampProperties.get(ChampServiceConstants.CHAMP_SOT_NAME);
66   private static final String CREATED_TS_NAME = ChampProperties.get(ChampServiceConstants.CHAMP_CREATED_TS_NAME);
67   private static final String LAST_MOD_TS_NAME = ChampProperties.get(ChampServiceConstants.CHAMP_LAST_MOD_TS_NAME);
68   private Logger logger = LoggerFactory.getInstance().getLogger(ChampDataService.class);
69
70
71   public ChampDataService(ChampUUIDService champUUIDService, ChampGraph graphImpl, ChampTransactionCache cache) {
72
73     this.champUUIDService = champUUIDService;
74     this.graphImpl = graphImpl;
75
76     ChampField field = new ChampField.Builder(ChampProperties.get("keyName"))
77             .type(ChampField.Type.STRING)
78             .build();
79     ChampObjectIndex index = new ChampObjectIndex.Builder(ChampProperties.get("keyName"), "STRING", field).build();
80
81     graphImpl.storeObjectIndex(index);
82
83     this.cache = cache;
84   }
85
86   public ChampObject getObject(String id, Optional<ChampTransaction> transaction) throws ChampServiceException {
87
88     Optional<ChampObject> retrieved = Optional.empty();
89     try {
90       retrieved = champUUIDService.getObjectbyUUID(id, transaction.orElse(null));
91     } catch (ChampUnmarshallingException | ChampTransactionException e) {
92       throw new ChampServiceException("Error: " + e.getMessage(), Status.INTERNAL_SERVER_ERROR);
93     }
94     if (retrieved.isPresent()) {
95       return (ChampObject) champUUIDService.populateUUIDKey(retrieved.get());
96     } else {
97       return null;
98     }
99   }
100
101   public ChampObject storeObject(ChampObject object, Optional<ChampTransaction> transaction)
102           throws ChampMarshallingException, ChampSchemaViolationException, ChampObjectNotExistsException,
103           ChampTransactionException, ChampServiceException {
104
105     if (object.getProperty(KEY_NAME).isPresent() || object.getKey().isPresent()) {
106       throw new ChampServiceException(KEY_NAME + " can't be updated", Status.BAD_REQUEST);
107     }
108
109     champUUIDService.populateUUIDProperty(object, java.util.UUID.randomUUID().toString());
110     addTimestamps(object, null);
111     ChampObject created = graphImpl.storeObject(object, transaction);
112     return (ChampObject) champUUIDService.populateUUIDKey(created);
113   }
114
115   public ChampObject replaceObject(ChampObject object, String objectId, Optional<ChampTransaction> transaction)
116           throws ChampServiceException, ChampUnmarshallingException, ChampTransactionException, ChampMarshallingException,
117           ChampSchemaViolationException, ChampObjectNotExistsException {
118     if (object.getKey().isPresent() && (!object.getKeyValue().equals(objectId))) {
119       throw new ChampServiceException("Object Id in the URI doesn't match the body.", Status.BAD_REQUEST);
120     }
121
122     if (object.getProperty(KEY_NAME).isPresent() && !object.getProperty(KEY_NAME).get().toString().equals(objectId)) {
123       throw new ChampServiceException(KEY_NAME + " can't be updated", Status.BAD_REQUEST);
124     }
125
126     Optional<ChampObject> retrieved = champUUIDService.getObjectbyUUID(objectId, transaction.orElse(null));
127     if (!retrieved.isPresent()) {
128       throw new ChampServiceException(objectId + " not found", Status.NOT_FOUND);
129     }
130     ObjectBuildOrPropertiesStep payloadBuilder = ChampObject.create().from(object).withKey(retrieved.get().getKey().get())
131             .withProperty(KEY_NAME, objectId);
132     if (retrieved.get().getProperty(SOT_NAME).isPresent()){
133       payloadBuilder = payloadBuilder.withProperty(SOT_NAME, retrieved.get().getProperty(SOT_NAME).get());
134     }
135
136     if (object.getProperty(CREATED_TS_NAME).isPresent() && retrieved.get().getProperty(CREATED_TS_NAME).isPresent()) {
137       // the timestamps in object are parsed as strings regardless of how the input json is. Convert retrieved to string for easy comparison
138       if (!retrieved.get().getProperty(CREATED_TS_NAME).get().toString().equals(object.getProperty(CREATED_TS_NAME).get())) {
139         throw new ChampServiceException(CREATED_TS_NAME + " can't be updated", Status.BAD_REQUEST);
140       }
141     }
142
143     if (object.getProperty(LAST_MOD_TS_NAME).isPresent() && retrieved.get().getProperty(LAST_MOD_TS_NAME).isPresent()) {
144       if (!retrieved.get().getProperty(LAST_MOD_TS_NAME).get().toString().equals(object.getProperty(LAST_MOD_TS_NAME).get())) {
145         throw new ChampServiceException(LAST_MOD_TS_NAME + " can't be updated", Status.BAD_REQUEST);
146       }
147     }
148
149     ChampObject payload = payloadBuilder.build();
150     addTimestamps(payload, (Long)retrieved.get().getProperty(CREATED_TS_NAME).orElse(null));
151     ChampObject updated = graphImpl.replaceObject(payload, transaction);
152     return (ChampObject) champUUIDService.populateUUIDKey(updated);
153   }
154
155   public void deleteObject(String objectId, Optional<ChampTransaction> transaction) throws ChampServiceException,
156           ChampObjectNotExistsException, ChampTransactionException, ChampUnmarshallingException {
157     Optional<ChampObject> retrieved = champUUIDService.getObjectbyUUID(objectId, transaction.orElse(null));
158     if (!retrieved.isPresent()) {
159       throw new ChampServiceException(objectId + " not found", Status.NOT_FOUND);
160     }
161     Stream<ChampRelationship> relationships = graphImpl.retrieveRelationships(retrieved.get(), transaction);
162
163     if (relationships.count() > 0) {
164       throw new ChampServiceException("Attempt to delete vertex with id " + objectId + " which has incident edges.",
165               Status.BAD_REQUEST);
166     }
167     graphImpl.deleteObject(retrieved.get().getKey().get(), transaction);
168
169   }
170
171   public ChampRelationship storeRelationship(ChampRelationship r, Optional<ChampTransaction> transaction)
172           throws ChampMarshallingException, ChampObjectNotExistsException, ChampSchemaViolationException,
173           ChampRelationshipNotExistsException, ChampUnmarshallingException, ChampTransactionException,
174           ChampServiceException {
175
176     if (r.getSource() == null || !r.getSource().getKey().isPresent() || r.getTarget() == null
177             || !r.getTarget().getKey().isPresent()) {
178       logger.error(ChampMsgs.CHAMP_DATA_SERVICE_ERROR, "Source/Target Object key must be provided");
179       throw new ChampServiceException("Source/Target Object key must be provided", Status.BAD_REQUEST);
180     }
181
182     if (r.getProperty(KEY_NAME).isPresent() || r.getKey().isPresent()) {
183       logger.error(ChampMsgs.CHAMP_DATA_SERVICE_ERROR, "key or " + KEY_NAME + " not allowed while creating new Objects");
184       throw new ChampServiceException("key or " + KEY_NAME + " not allowed while creating new Objects", Status.BAD_REQUEST);
185
186     }
187
188     Optional<ChampObject> source = champUUIDService.getObjectbyUUID(r.getSource().getKey().get().toString(),
189             transaction.orElse(null));
190     Optional<ChampObject> target = champUUIDService.getObjectbyUUID(r.getTarget().getKey().get().toString(),
191             transaction.orElse(null));
192
193     if (!source.isPresent() || !target.isPresent()) {
194       logger.error(ChampMsgs.CHAMP_DATA_SERVICE_ERROR, "Source/Target object not found");
195       throw new ChampServiceException("Source/Target object not found", Status.BAD_REQUEST);
196     }
197
198     champUUIDService.populateUUIDProperty(r, java.util.UUID.randomUUID().toString());
199
200     ChampRelationship payload = new ChampRelationship.Builder(source.get(), target.get(), r.getType())
201             .properties(r.getProperties()).build();
202     addTimestamps(payload, null);
203     ChampRelationship created = graphImpl.storeRelationship(payload, transaction);
204     return (ChampRelationship) champUUIDService.populateUUIDKey(created);
205   }
206
207   public ChampRelationship updateRelationship(ChampRelationship r, String rId, Optional<ChampTransaction> transaction)
208           throws ChampServiceException, ChampUnmarshallingException, ChampTransactionException, ChampMarshallingException,
209           ChampSchemaViolationException, ChampRelationshipNotExistsException {
210     if (r.getKey().isPresent() && (!r.getKeyValue().equals(rId))) {
211
212       throw new ChampServiceException("Relationship Id in the URI \"" + rId + "\" doesn't match the URI in the body"
213               + " \"" + r.getKeyValue() + "\"", Status.BAD_REQUEST);
214
215     }
216
217     if (r.getProperty(KEY_NAME).isPresent() && !r.getProperty(KEY_NAME).get().toString().equals(rId)) {
218       throw new ChampServiceException(KEY_NAME + " can't be updated", Status.BAD_REQUEST);
219     }
220
221     Optional<ChampRelationship> retrieved = champUUIDService.getRelationshipbyUUID(rId, transaction.orElse(null));
222     if (!retrieved.isPresent()) {
223       throw new ChampServiceException(rId + " not found", Status.NOT_FOUND);
224     }
225     // check if key is present or if it equals the key that is in the URI
226     if (r.getSource() == null || !r.getSource().getKey().isPresent() || r.getTarget() == null
227             || !r.getTarget().getKey().isPresent()) {
228       throw new ChampServiceException("Source/Target Object key must be provided", Status.BAD_REQUEST);
229     }
230     ChampObject source = retrieved.get().getSource();
231     ChampObject target = retrieved.get().getTarget();
232
233     if (!source.getProperty(KEY_NAME).get().toString().equals(r.getSource().getKey().get().toString())
234             || !target.getProperty(KEY_NAME).get().toString().equals(r.getTarget().getKey().get().toString())) {
235       throw new ChampServiceException("Source/Target cannot be updated", Status.BAD_REQUEST);
236     }
237
238     if (r.getProperty(CREATED_TS_NAME).isPresent() && retrieved.get().getProperty(CREATED_TS_NAME).isPresent()) {
239       if (!retrieved.get().getProperty(CREATED_TS_NAME).get().toString().equals(r.getProperty(CREATED_TS_NAME).get())) {
240         throw new ChampServiceException(CREATED_TS_NAME + " can't be updated", Status.BAD_REQUEST);
241       }
242     }
243
244     if (r.getProperty(LAST_MOD_TS_NAME).isPresent() && retrieved.get().getProperty(LAST_MOD_TS_NAME).isPresent()) {
245       if (!retrieved.get().getProperty(LAST_MOD_TS_NAME).get().toString().equals(r.getProperty(LAST_MOD_TS_NAME).get())) {
246         throw new ChampServiceException(LAST_MOD_TS_NAME + " can't be updated", Status.BAD_REQUEST);
247       }
248     }
249
250     ChampRelationship payload = new ChampRelationship.Builder(source, target, r.getType())
251             .key(retrieved.get().getKey().get()).properties(r.getProperties()).property(KEY_NAME, rId).build();
252     addTimestamps(payload, (Long)retrieved.get().getProperty(CREATED_TS_NAME).orElse(null));
253     ChampRelationship updated = graphImpl.replaceRelationship(payload, transaction);
254     return (ChampRelationship) champUUIDService.populateUUIDKey(updated);
255   }
256
257   public void deleteRelationship(String relationshipId, Optional<ChampTransaction> transaction)
258           throws ChampServiceException, ChampRelationshipNotExistsException, ChampTransactionException,
259           ChampUnmarshallingException {
260     Optional<ChampRelationship> retrieved = champUUIDService.getRelationshipbyUUID(relationshipId,
261             transaction.orElse(null));
262     if (!retrieved.isPresent()) {
263       throw new ChampServiceException(relationshipId + " not found", Status.NOT_FOUND);
264     }
265
266     graphImpl.deleteRelationship(retrieved.get(), transaction);
267
268   }
269
270
271   public List<ChampRelationship> getRelationshipsByObject(String objectId, Optional<ChampTransaction> transaction)
272           throws ChampServiceException {
273     try {
274       Optional<ChampObject> retrievedObject = champUUIDService.getObjectbyUUID(objectId, transaction.orElse(null));
275       if (!retrievedObject.isPresent()) {
276         throw new ChampServiceException(objectId + " not found", Status.NOT_FOUND);
277       }
278       List<ChampRelationship> relations = new ArrayList<ChampRelationship>();
279
280       Stream<ChampRelationship> retrieved = graphImpl.retrieveRelationships(retrievedObject.get(), transaction);
281       relations = champUUIDService.populateUUIDKey(retrieved.collect(Collectors.toList()));
282       return relations;
283     } catch (ChampObjectNotExistsException e) {
284       throw new ChampServiceException(" obj not found", Status.NOT_FOUND);
285     } catch (ChampUnmarshallingException | ChampTransactionException e) {
286       throw new ChampServiceException("Internal Error", Status.INTERNAL_SERVER_ERROR);
287     }
288
289   }
290
291   /**
292    * Gets the ChampObjects that pass filter
293    * @param filter key/value pairs that must be present in the returned objects
294    * @param properties properties that will show up in the object
295    * @return
296    * @throws ChampServiceException
297    */
298   public List<ChampObject> queryObjects(Map<String, Object> filter, HashSet<String> properties) throws ChampServiceException {
299     try {
300
301       Stream<ChampObject> retrieved = graphImpl.queryObjects(filter);
302       List<ChampObject> objects = champUUIDService.populateUUIDKey(retrieved.collect(Collectors.toList()));
303
304       if (!properties.contains("all")) {
305         for (ChampObject champObject : objects) {
306           champObject.dropProperties(properties);
307         }
308       }
309
310       return objects;
311     } catch (ChampTransactionException e) {
312       throw new ChampServiceException("Internal Error", Status.INTERNAL_SERVER_ERROR);
313     }
314   }
315
316   public List<ChampRelationship> queryRelationships(Map<String, Object> filter) throws ChampServiceException {
317     try {
318       List<ChampRelationship> relations = new ArrayList<ChampRelationship>();
319       Stream<ChampRelationship> retrieved;
320
321       retrieved = graphImpl.queryRelationships(filter);
322
323       relations = champUUIDService.populateUUIDKey(retrieved.collect(Collectors.toList()));
324       return relations;
325     } catch (ChampTransactionException e) {
326       throw new ChampServiceException("Internal Error", Status.INTERNAL_SERVER_ERROR);
327     }
328   }
329
330   public ChampRelationship getRelationship(String id, Optional<ChampTransaction> transaction)
331           throws ChampServiceException {
332
333     Optional<ChampRelationship> retrieved = Optional.empty();
334     try {
335       retrieved = champUUIDService.getRelationshipbyUUID(id, transaction.orElse(null));
336     } catch (ChampUnmarshallingException | ChampTransactionException e) {
337       throw new ChampServiceException("Error: " + e.getMessage(), Status.INTERNAL_SERVER_ERROR);
338     }
339     if (retrieved.isPresent()) {
340       return (ChampRelationship) champUUIDService.populateUUIDKey(retrieved.get());
341     } else {
342       return null;
343     }
344   }
345
346   public String openTransaction() {
347     ChampTransaction transaction = graphImpl.openTransaction();
348     String transacId = transaction.id();
349     cache.put(transacId, transaction);
350     return transacId;
351
352   }
353
354   public void commitTransaction(String tId) throws ChampServiceException, ChampTransactionException {
355     ChampTransaction transaction = cache.get(tId);
356     if (transaction == null) {
357       throw new ChampServiceException("Transaction Not found: " + tId, Status.NOT_FOUND);
358     }
359     graphImpl.commitTransaction(transaction);
360     cache.invalidate(tId);
361     cache.invalidate(transaction.id());
362
363   }
364
365   public void rollbackTransaction(String tId) throws ChampServiceException, ChampTransactionException {
366     ChampTransaction transaction = cache.get(tId);
367     if (transaction == null) {
368       throw new ChampServiceException("Transaction Not found: " + tId, Status.NOT_FOUND);
369     }
370     graphImpl.rollbackTransaction(transaction);
371     cache.invalidate(tId);
372     cache.invalidate(transaction.id());
373
374   }
375
376   public ChampTransaction getTransaction(String id) {
377     return cache.get(id);
378   }
379
380   public ChampBulkResponse processBulkRequest(ChampBulkPayload bulkPayload) throws ChampServiceException, ChampRelationshipNotExistsException, ChampTransactionException, ChampUnmarshallingException, ChampObjectNotExistsException, ChampMarshallingException, ChampSchemaViolationException {
381     // Open a transaction.  If any operations fail, we want to rollback
382     ChampTransaction transaction = graphImpl.openTransaction();
383     if (transaction == null) {
384       throw new ChampServiceException("Unable to open transaction", Status.INTERNAL_SERVER_ERROR);
385     }
386
387     ChampBulkResponse responsePayload = new ChampBulkResponse();
388     Map<String,ChampObject> addedObjects = new HashMap<String,ChampObject>();
389     List<ChampBulkVertexResponse> addedObjectsResp = new ArrayList<ChampBulkVertexResponse>();
390     List<ChampBulkEdgeResponse> addedEdgesResp = new ArrayList<ChampBulkEdgeResponse>();
391
392     try {
393       // 1. Process edge deletes
394       for (ChampBulkOp op : bulkPayload.getEdgeDeleteOps()) {
395         deleteRelationship(op.getId(), Optional.ofNullable(transaction));
396       }
397
398       // 2. Process vertex deletes
399       for (ChampBulkOp op : bulkPayload.getVertexDeleteOps()) {
400         deleteObject(op.getId(), Optional.ofNullable(transaction));
401       }
402
403       // 3. Add/modify vertexes
404       for (ChampBulkOp op : bulkPayload.getVertexAddModifyOps()) {
405         if (op.getOperation().equals(ChampBulkPayload.ADD_OP)) {
406           ChampObject addedObj = storeObject(op.toChampObject(), Optional.ofNullable(transaction));
407           addedObjects.put(op.getLabel(), addedObj);
408           addedObjectsResp.add(new ChampBulkVertexResponse(op.getLabel(), addedObj));
409         }
410         else {
411           ChampObject addedObj = replaceObject(op.toChampObject(), op.getId(), Optional.ofNullable(transaction));
412           addedObjects.put(op.getLabel(), addedObj);
413           addedObjectsResp.add(new ChampBulkVertexResponse(op.getLabel(), addedObj));
414         }
415       }
416
417       // 4. Add/modify edges
418       for (ChampBulkOp op : bulkPayload.getEdgeAddModifyOps()) {
419         // If the edge references a newly added vertex, we need to replace the reference with the real ID
420         op.setSource(resolveVertex(op.getSource(), addedObjects));
421         op.setTarget(resolveVertex(op.getTarget(), addedObjects));
422
423         if (op.getOperation().equals(ChampBulkPayload.ADD_OP)) {
424           ChampRelationship addedRel = storeRelationship(op.toChampRelationship(), Optional.ofNullable(transaction));
425           addedEdgesResp.add(new ChampBulkEdgeResponse(op.getLabel(), addedRel));
426         }
427         else {
428           ChampRelationship addedRel = updateRelationship(op.toChampRelationship(), op.getId(), Optional.ofNullable(transaction));
429           addedEdgesResp.add(new ChampBulkEdgeResponse(op.getLabel(), addedRel));
430         }
431       }
432     }
433     catch (Exception ex) {
434       // Rollback the transaction
435       graphImpl.rollbackTransaction(transaction);
436       throw ex;
437     }
438
439     // Commit transaction
440     graphImpl.commitTransaction(transaction);
441
442     responsePayload.setObjects(addedObjectsResp);
443     responsePayload.setRelationships(addedEdgesResp);
444     return responsePayload;
445   }
446
447
448   private String resolveVertex(String vertexId, Map<String, ChampObject> addedObjects) throws ChampServiceException {
449     if (vertexId.startsWith("$")) {
450       String key = vertexId.substring(1);
451       if (addedObjects.get(key) != null) {
452         return addedObjects.get(key).getKey().get().toString();
453       }
454
455       throw new ChampServiceException("Unable to resolve vertex " + key, Status.BAD_REQUEST);
456     }
457
458     return vertexId;
459   }
460
461   private void addTimestamps(ChampElement e, Long oldCreated) {
462     Long timestamp = System.currentTimeMillis();
463
464     if (oldCreated == null) {
465       e.getProperties().put(CREATED_TS_NAME, timestamp);
466     } else {
467       e.getProperties().put(CREATED_TS_NAME, oldCreated);
468     }
469
470     e.getProperties().put(LAST_MOD_TS_NAME, timestamp);
471   }
472 }