aai-core/src/main/java/org/onap/aai/serialization/queryformats/LifecycleFormat.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.aai.serialization.queryformats;
  22
  23 import com.google.gson.JsonArray;
  24 import com.google.gson.JsonObject;
  25
  26 import java.util.*;
  27 import java.util.stream.Collectors;
  28 import java.util.stream.Stream;
  29
  30 import org.apache.tinkerpop.gremlin.structure.Edge;
  31 import org.apache.tinkerpop.gremlin.structure.Vertex;
  32 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
  33 import org.onap.aai.db.props.AAIProperties;
  34 import org.onap.aai.logging.LogFormatTools;
  35 import org.onap.aai.serialization.queryformats.exceptions.AAIFormatQueryResultFormatNotSupported;
  36 import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException;
  37 import org.slf4j.Logger;
  38 import org.slf4j.LoggerFactory;
  39 import org.springframework.util.LinkedMultiValueMap;
  40 import org.springframework.util.MultiValueMap;
  41
  42 public class LifecycleFormat extends HistoryFormat {
  43
  44     private static final Logger LOGGER = LoggerFactory.getLogger(LifecycleFormat.class);
  45
  46     protected LifecycleFormat(Builder builder) {
  47         super(builder);
  48     }
  49
  50     protected JsonArray createPropertiesObject(Vertex v) {
  51         JsonArray jsonArray = new JsonArray();
  52         Iterator<VertexProperty<Object>> iter = v.properties();
  53         List<JsonObject> jsonList = new ArrayList<>();
  54
  55         Map<String, Set<Long>> propStartTimes = new HashMap<>(); // vertex end
  56         while (iter.hasNext()) {
  57             JsonObject json = new JsonObject();
  58             VertexProperty<Object> prop = iter.next();
  59             if (prop.key() != null && ignoredKeys.contains(prop.key())) {
  60                 continue;
  61             }
  62             if (!propStartTimes.containsKey(prop.key())) {
  63                 propStartTimes.put(prop.key(), new HashSet<>());
  64                 if (v.property(AAIProperties.END_TS).isPresent()) {
  65                     propStartTimes.get(prop.key()).add(v.<Long>value(AAIProperties.END_TS));
  66                 }
  67             }
  68
  69             json.addProperty(KEY, prop.key());
  70             json = mapPropertyValues(json, VALUE, prop.value());
  71             JsonObject metaProperties = createMetaPropertiesObject(prop);
  72             if (isTsInRange(metaProperties.get(AAIProperties.START_TS).getAsLong())) {
  73                 JsonObject jo = new JsonObject();
  74                 jo.add(KEY, json.get(KEY));
  75                 jo.add(VALUE, json.get(VALUE));
  76                 jo.add(TIMESTAMP, metaProperties.get(AAIProperties.START_TS));
  77                 jo.add(SOT, metaProperties.get(AAIProperties.SOURCE_OF_TRUTH));
  78                 jo.add(TX_ID, metaProperties.get(AAIProperties.START_TX_ID));
  79                 jsonList.add(jo);
  80                 propStartTimes.get(prop.key()).add(metaProperties.get(AAIProperties.START_TS).getAsLong());
  81             }
  82             if (!AAIProperties.RESOURCE_VERSION.equals(prop.key()) && metaProperties.has(AAIProperties.END_TS)
  83                     && isTsInRange(metaProperties.get(AAIProperties.END_TS).getAsLong())
  84                     && !propStartTimes.get(prop.key()).contains(metaProperties.get(AAIProperties.END_TS).getAsLong())) {
  85                 JsonObject jo = new JsonObject();
  86                 jo.add(KEY, json.get(KEY));
  87                 jo.add(VALUE, null);
  88                 jo.add(TIMESTAMP, metaProperties.get(AAIProperties.END_TS));
  89                 jo.add(SOT, metaProperties.get(AAIProperties.END_SOT));
  90                 jo.add(TX_ID, metaProperties.get(AAIProperties.END_TX_ID));
  91                 jsonList.add(jo);
  92             }
  93
  94         }
  95         jsonList.stream()
  96                 // remove all the null values that is the start time for another value
  97                 .filter(jo -> !jo.get(VALUE).isJsonNull()
  98                         || !propStartTimes.get(jo.get(KEY).getAsString()).contains(jo.get(TIMESTAMP).getAsLong()))
  99                 // sort by ts in decreasing order
 100                 .sorted((o1, o2) -> {
 101                     if (o1.get(TIMESTAMP).getAsLong() == o2.get(TIMESTAMP).getAsLong()) {
 102                         return o1.get(KEY).getAsString().compareTo(o2.get(KEY).getAsString());
 103                     } else {
 104                         return Long.compare(o2.get(TIMESTAMP).getAsLong(), o1.get(TIMESTAMP).getAsLong());
 105                     }
 106                 }).forEach(jsonArray::add);
 107
 108         return jsonArray;
 109     }
 110
 111     private boolean isTsInRange(long ts) {
 112         return ts >= startTs && ts <= endTs;
 113     }
 114
 115     @Override
 116     protected boolean isValidEdge(Edge e) {
 117         if (e.property(AAIProperties.END_TS).isPresent()) {
 118             long edgeStartTs = e.<Long>value(AAIProperties.START_TS);
 119             long edgeEndTs = e.<Long>value(AAIProperties.END_TS);
 120             return isTsInRange(edgeStartTs) || isTsInRange(edgeEndTs);
 121         } else {
 122             long edgeStartTs = e.<Long>value(AAIProperties.START_TS);
 123             return isTsInRange(edgeStartTs);
 124         }
 125     }
 126
 127     @Override
 128     protected JsonObject getRelatedObject(Edge e, Vertex related) throws AAIFormatVertexException {
 129
 130         JsonObject json = new JsonObject();
 131         json.addProperty("relationship-label", e.label());
 132         json.addProperty("node-type", related.<String>value(AAIProperties.NODE_TYPE));
 133         json.addProperty("url", this.urlBuilder.pathed(related));
 134         if (related.property(AAIProperties.AAI_URI).isPresent()) {
 135             json.addProperty("uri", related.<String>value(AAIProperties.AAI_URI));
 136         } else {
 137             LOGGER.warn("Vertex {} is missing aai-uri", related.id());
 138             json.addProperty("uri", "NA");
 139         }
 140
 141         if (e.property(AAIProperties.START_TS).isPresent()) {
 142             long edgeStartTimestamp = e.<Long>value(AAIProperties.START_TS);
 143             if (isTsInRange(edgeStartTimestamp)) {
 144                 json.addProperty(TIMESTAMP,
 145                         e.property(AAIProperties.START_TS).isPresent() ? e.<Long>value(AAIProperties.START_TS) : 0);
 146                 json.addProperty(SOT,
 147                         e.property(AAIProperties.SOURCE_OF_TRUTH).isPresent() ? e.value(AAIProperties.SOURCE_OF_TRUTH)
 148                                 : "");
 149                 json.addProperty(TX_ID,
 150                         e.property(AAIProperties.START_TX_ID).isPresent() ? e.value(AAIProperties.START_TX_ID) : "N/A");
 151             }
 152         }
 153
 154         if (e.property(AAIProperties.END_TS).isPresent()) {
 155             long edgeEndTimestamp = e.<Long>value(AAIProperties.END_TS);
 156             if (isTsInRange(edgeEndTimestamp)) {
 157                 json.addProperty(END_TIMESTAMP, edgeEndTimestamp);
 158                 json.addProperty(END_SOT,
 159                         e.property(AAIProperties.END_SOT).isPresent() ? e.value(AAIProperties.END_SOT) : "");
 160                 json.addProperty(END_TX_ID,
 161                         e.property(AAIProperties.END_TX_ID).isPresent() ? e.value(AAIProperties.END_TX_ID) : "N/A");
 162             }
 163         }
 164
 165         return json;
 166     }
 167
 168     @Override
 169     protected Optional<JsonObject> getJsonFromVertex(Vertex v) throws AAIFormatVertexException {
 170         JsonObject json = new JsonObject();
 171         json.addProperty(NODE_TYPE, v.<String>value(AAIProperties.NODE_TYPE));
 172         json.addProperty("url", this.urlBuilder.pathed(v));
 173         if (v.property(AAIProperties.AAI_URI).isPresent()) {
 174             json.addProperty("uri", v.<String>value(AAIProperties.AAI_URI));
 175         } else {
 176             LOGGER.warn("Vertex {} is missing aai-uri", v.id());
 177             json.addProperty("uri", "NA");
 178         }
 179         json.addProperty(TIMESTAMP, v.<Long>value(AAIProperties.START_TS));
 180
 181         json.add(PROPERTIES, this.createPropertiesObject(v));
 182
 183         if (!nodesOnly) {
 184             json.add(RELATED_TO, this.createRelationshipObject(v));
 185         }
 186
 187         json.add(NODE_ACTIONS, getNodeActions(v, json));
 188
 189         if (json.getAsJsonObject().get(PROPERTIES).getAsJsonArray().size() == 0
 190                 && json.getAsJsonObject().get(RELATED_TO).getAsJsonArray().size() == 0
 191                 && json.getAsJsonObject().get(NODE_ACTIONS).getAsJsonArray().size() == 0) {
 192             return Optional.empty();
 193         } else if (json.getAsJsonObject().get(PROPERTIES).getAsJsonArray().size() == 1
 194                 && (json.getAsJsonObject().get(RELATED_TO).getAsJsonArray().size() > 0
 195                         || json.getAsJsonObject().get(NODE_ACTIONS).getAsJsonArray().size() > 0)) {
 196             if (json.getAsJsonObject().get(PROPERTIES).getAsJsonArray().get(0).getAsJsonObject().get("key")
 197                     .getAsString().equals(AAIProperties.END_TS)) {
 198                 json.getAsJsonObject().add(PROPERTIES, new JsonArray());
 199             }
 200         }
 201
 202         return Optional.of(json);
 203     }
 204
 205     @Override
 206     protected Optional<JsonObject> getJsonFromVertex(Vertex input, Map<String, List<String>> properties)
 207             throws AAIFormatVertexException {
 208         return Optional.empty();
 209     }
 210
 211     private JsonArray getNodeActions(Vertex v, JsonObject json) {
 212         JsonArray nodeActions = new JsonArray();
 213         JsonObject action;
 214         if (v.property(AAIProperties.END_TS).isPresent()) {
 215             long deletedTs = (Long) v.property(AAIProperties.END_TS).value();
 216             if (isTsInRange(deletedTs)) {
 217                 action = new JsonObject();
 218                 action.addProperty("action", "DELETED");
 219                 action.addProperty(TIMESTAMP, deletedTs);
 220                 if (v.property(AAIProperties.END_TS).property(AAIProperties.SOURCE_OF_TRUTH).isPresent()) {
 221                     action.addProperty(SOT,
 222                             v.property(AAIProperties.END_TS).<String>value(AAIProperties.SOURCE_OF_TRUTH));
 223                 }
 224                 if (v.property(AAIProperties.END_TS).property(AAIProperties.END_TX_ID).isPresent()) {
 225                     action.addProperty(TX_ID, v.property(AAIProperties.END_TS).<String>value(AAIProperties.END_TX_ID));
 226                 } else {
 227                     action.addProperty(TX_ID, "N/A");
 228                 }
 229                 nodeActions.add(action);
 230             }
 231         }
 232         long createdTs = json.get(TIMESTAMP).getAsLong();
 233         if (isTsInRange(createdTs)) {
 234             action = new JsonObject();
 235             action.addProperty("action", "CREATED");
 236             action.addProperty(TIMESTAMP, createdTs);
 237             action.addProperty(SOT, v.<String>value(AAIProperties.SOURCE_OF_TRUTH));
 238             if (v.property(AAIProperties.SOURCE_OF_TRUTH).property(AAIProperties.START_TX_ID).isPresent()) {
 239                 action.addProperty(TX_ID,
 240                         v.property(AAIProperties.SOURCE_OF_TRUTH).<String>value(AAIProperties.START_TX_ID));
 241             } else {
 242                 action.addProperty(TX_ID, "N/A");
 243             }
 244             nodeActions.add(action);
 245         }
 246         return nodeActions;
 247     }
 248
 249     public JsonArray process(List<Object> queryResults) {
 250         JsonArray body = new JsonArray();
 251         Stream<Object> stream;
 252         if (queryResults.size() >= this.parallelThreshold()) {
 253             stream = queryResults.parallelStream();
 254         } else {
 255             stream = queryResults.stream();
 256         }
 257
 258         final boolean isParallel = stream.isParallel();
 259
 260         stream.map(o -> {
 261             try {
 262                 return this.formatObject(o);
 263             } catch (AAIFormatVertexException e) {
 264                 LOGGER.warn("Failed to format vertex, returning a partial list " + LogFormatTools.getStackTop(e));
 265             } catch (AAIFormatQueryResultFormatNotSupported e) {
 266                 LOGGER.warn("Failed to format result type of the query " + LogFormatTools.getStackTop(e));
 267             }
 268
 269             return Optional.<JsonObject>empty();
 270         }).filter(Optional::isPresent).map(Optional::get).forEach(json -> {
 271             if (isParallel) {
 272                 synchronized (body) {
 273                     body.add(json);
 274                 }
 275             } else {
 276                 body.add(json);
 277             }
 278         });
 279         JsonArray result = organizeBody(body);
 280         result.forEach(jsonElement -> jsonElement.getAsJsonObject().remove(TIMESTAMP));
 281         return result;
 282     }
 283
 284     private JsonArray organizeBody(JsonArray body) {
 285
 286         final MultiValueMap<String, Integer> toBeMerged = new LinkedMultiValueMap<>();
 287         for (int i = 0; i < body.size(); i++) {
 288             toBeMerged.add(body.get(i).getAsJsonObject().get("uri").getAsString(), i);
 289         }
 290
 291         final List<List<Integer>> dupes =
 292                 toBeMerged.values().stream().filter(l -> l.size() > 1).collect(Collectors.toList());
 293         if (dupes.isEmpty()) {
 294             return body;
 295         } else {
 296             Set<Integer> remove = new HashSet<>();
 297             for (List<Integer> dupe : dupes) {
 298                 dupe.sort((a, b) -> Long.compare(body.get(b).getAsJsonObject().get(TIMESTAMP).getAsLong(),
 299                         body.get(a).getAsJsonObject().get(TIMESTAMP).getAsLong()));
 300                 int keep = dupe.remove(0);
 301                 for (Integer idx : dupe) {
 302                     body.get(keep).getAsJsonObject().getAsJsonArray(NODE_ACTIONS)
 303                             .addAll(body.get(idx).getAsJsonObject().getAsJsonArray(NODE_ACTIONS));
 304                     body.get(keep).getAsJsonObject().getAsJsonArray(PROPERTIES)
 305                             .addAll(body.get(idx).getAsJsonObject().getAsJsonArray(PROPERTIES));
 306                     body.get(keep).getAsJsonObject().getAsJsonArray(RELATED_TO)
 307                             .addAll(body.get(idx).getAsJsonObject().getAsJsonArray(RELATED_TO));
 308                     remove.add(idx);
 309                 }
 310             }
 311             final JsonArray newBody = new JsonArray();
 312             for (int i = 0; i < body.size(); i++) {
 313                 if (!remove.contains(i)) {
 314                     newBody.add(body.get(i));
 315                 }
 316             }
 317             return newBody;
 318         }
 319     }
 320
 321 }