aai-core/src/main/java/org/onap/aai/serialization/queryformats/LifecycleFormat.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.aai.serialization.queryformats;
  22
  23 import org.slf4j.Logger;
  24 import org.slf4j.LoggerFactory;
  25 import com.google.gson.JsonArray;
  26 import com.google.gson.JsonObject;
  27 import org.apache.tinkerpop.gremlin.structure.Edge;
  28 import org.apache.tinkerpop.gremlin.structure.Vertex;
  29 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
  30 import org.onap.aai.db.props.AAIProperties;
  31 import org.onap.aai.logging.LogFormatTools;
  32 import org.onap.aai.serialization.queryformats.exceptions.AAIFormatQueryResultFormatNotSupported;
  33 import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException;
  34 import org.springframework.util.LinkedMultiValueMap;
  35 import org.springframework.util.MultiValueMap;
  36
  37 import java.util.*;
  38 import java.util.stream.Collectors;
  39 import java.util.stream.Stream;
  40
  41
  42 public class LifecycleFormat extends HistoryFormat {
  43
  44     private static final Logger LOGGER = LoggerFactory.getLogger(LifecycleFormat.class);
  45
  46     protected LifecycleFormat(Builder builder) {
  47         super(builder);
  48     }
  49
  50     protected JsonArray createPropertiesObject(Vertex v) {
  51         JsonArray jsonArray = new JsonArray();
  52         Iterator<VertexProperty<Object>> iter = v.properties();
  53         List<JsonObject> jsonList = new ArrayList<>();
  54
  55         Map<String, Set<Long>> propStartTimes = new HashMap<>(); //vertex end
  56         while (iter.hasNext()) {
  57             JsonObject json = new JsonObject();
  58             VertexProperty<Object> prop = iter.next();
  59             if(prop.key() != null && ignoredKeys.contains(prop.key())){
  60                 continue;
  61             }
  62             if (!propStartTimes.containsKey(prop.key())) {
  63                 propStartTimes.put(prop.key(), new HashSet<>());
  64                 if (v.property(AAIProperties.END_TS).isPresent()) {
  65                     propStartTimes.get(prop.key()).add(v.<Long>value(AAIProperties.END_TS));
  66                 }
  67             }
  68
  69             json.addProperty(KEY, prop.key());
  70             json = mapPropertyValues(json, VALUE, prop.value());
  71             JsonObject metaProperties = createMetaPropertiesObject(prop);
  72             if (isTsInRange(metaProperties.get(AAIProperties.START_TS).getAsLong())) {
  73                 JsonObject jo = new JsonObject();
  74                 jo.add(KEY, json.get(KEY));
  75                 jo.add(VALUE, json.get(VALUE));
  76                 jo.add(TIMESTAMP, metaProperties.get(AAIProperties.START_TS));
  77                 jo.add(SOT, metaProperties.get(AAIProperties.SOURCE_OF_TRUTH));
  78                 jo.add(TX_ID, metaProperties.get(AAIProperties.START_TX_ID));
  79                 jsonList.add(jo);
  80                 propStartTimes.get(prop.key()).add(metaProperties.get(AAIProperties.START_TS).getAsLong());
  81             }
  82             if (!AAIProperties.RESOURCE_VERSION.equals(prop.key())
  83                 && metaProperties.has(AAIProperties.END_TS)
  84                 && isTsInRange(metaProperties.get(AAIProperties.END_TS).getAsLong())
  85                 && !propStartTimes.get(prop.key()).contains(metaProperties.get(AAIProperties.END_TS).getAsLong())) {
  86                 JsonObject jo = new JsonObject();
  87                 jo.add(KEY, json.get(KEY));
  88                 jo.add(VALUE, null);
  89                 jo.add(TIMESTAMP, metaProperties.get(AAIProperties.END_TS));
  90                 jo.add(SOT, metaProperties.get(AAIProperties.END_SOT));
  91                 jo.add(TX_ID, metaProperties.get(AAIProperties.END_TX_ID));
  92                 jsonList.add(jo);
  93             }
  94
  95         }
  96         jsonList.stream()
  97             // remove all the null values that is the start time for another value
  98             .filter(jo -> !jo.get(VALUE).isJsonNull() || !propStartTimes.get(jo.get(KEY).getAsString()).contains(jo.get(TIMESTAMP).getAsLong()))
  99             // sort by ts in decreasing order
 100             .sorted((o1, o2) -> {
 101                 if (o1.get(TIMESTAMP).getAsLong() == o2.get(TIMESTAMP).getAsLong()) {
 102                     return o1.get(KEY).getAsString().compareTo(o2.get(KEY).getAsString());
 103                 } else {
 104                     return Long.compare(o2.get(TIMESTAMP).getAsLong(), o1.get(TIMESTAMP).getAsLong());
 105                 }
 106             }).forEach(jsonArray::add);
 107
 108         return jsonArray;
 109     }
 110
 111     private boolean isTsInRange(long ts) {
 112         return ts >= startTs && ts <= endTs;
 113     }
 114
 115
 116     @Override
 117     protected boolean isValidEdge(Edge e) {
 118         if (e.property(AAIProperties.END_TS).isPresent()) {
 119             long edgeStartTs = e.<Long>value(AAIProperties.START_TS);
 120             long edgeEndTs = e.<Long>value(AAIProperties.END_TS);
 121             return isTsInRange(edgeStartTs) || isTsInRange(edgeEndTs);
 122         } else {
 123             long edgeStartTs = e.<Long>value(AAIProperties.START_TS);
 124             return isTsInRange(edgeStartTs);
 125         }
 126     }
 127
 128     @Override
 129     protected JsonObject getRelatedObject(Edge e, Vertex related) throws AAIFormatVertexException {
 130
 131         JsonObject json = new JsonObject();
 132         json.addProperty("relationship-label", e.label());
 133         json.addProperty("node-type", related.<String>value(AAIProperties.NODE_TYPE));
 134         json.addProperty("url", this.urlBuilder.pathed(related));
 135         if (related.property(AAIProperties.AAI_URI).isPresent()) {
 136             json.addProperty("uri", related.<String>value(AAIProperties.AAI_URI));
 137         } else {
 138             LOGGER.warn("Vertex {} is missing aai-uri", related.id());
 139             json.addProperty("uri", "NA");
 140         }
 141
 142         if(e.property(AAIProperties.START_TS).isPresent()) {
 143             long edgeStartTimestamp = e.<Long>value(AAIProperties.START_TS);
 144             if (isTsInRange(edgeStartTimestamp)) {
 145                 json.addProperty(TIMESTAMP,  e.property(AAIProperties.START_TS).isPresent()? e.<Long>value(AAIProperties.START_TS) : 0);
 146                 json.addProperty(SOT, e.property(AAIProperties.SOURCE_OF_TRUTH).isPresent()? e.value(AAIProperties.SOURCE_OF_TRUTH) : "");
 147                 json.addProperty(TX_ID, e.property(AAIProperties.START_TX_ID).isPresent()? e.value(AAIProperties.START_TX_ID) : "N/A");
 148             }
 149         }
 150
 151         if(e.property(AAIProperties.END_TS).isPresent()) {
 152             long edgeEndTimestamp = e.<Long>value(AAIProperties.END_TS);
 153             if (isTsInRange(edgeEndTimestamp)) {
 154                 json.addProperty(END_TIMESTAMP, edgeEndTimestamp);
 155                 json.addProperty(END_SOT, e.property(AAIProperties.END_SOT).isPresent() ? e.value(AAIProperties.END_SOT) : "");
 156                 json.addProperty(END_TX_ID, e.property(AAIProperties.END_TX_ID).isPresent() ? e.value(AAIProperties.END_TX_ID) : "N/A");
 157             }
 158         }
 159
 160         return json;
 161     }
 162
 163     @Override
 164     protected Optional<JsonObject> getJsonFromVertex(Vertex v) throws AAIFormatVertexException {
 165         JsonObject json = new JsonObject();
 166         json.addProperty(NODE_TYPE, v.<String>value(AAIProperties.NODE_TYPE));
 167         json.addProperty("url", this.urlBuilder.pathed(v));
 168         if (v.property(AAIProperties.AAI_URI).isPresent()) {
 169             json.addProperty("uri", v.<String>value(AAIProperties.AAI_URI));
 170         } else {
 171             LOGGER.warn("Vertex {} is missing aai-uri", v.id());
 172             json.addProperty("uri", "NA");
 173         }
 174         json.addProperty(TIMESTAMP, v.<Long>value(AAIProperties.START_TS));
 175
 176         json.add(PROPERTIES, this.createPropertiesObject(v));
 177
 178         if (!nodesOnly) {
 179             json.add(RELATED_TO, this.createRelationshipObject(v));
 180         }
 181
 182         json.add(NODE_ACTIONS, getNodeActions(v, json));
 183
 184         if (json.getAsJsonObject().get(PROPERTIES).getAsJsonArray().size() == 0
 185             && json.getAsJsonObject().get(RELATED_TO).getAsJsonArray().size() == 0
 186             && json.getAsJsonObject().get(NODE_ACTIONS).getAsJsonArray().size() == 0) {
 187             return Optional.empty();
 188         } else if (json.getAsJsonObject().get(PROPERTIES).getAsJsonArray().size() == 1
 189             && (json.getAsJsonObject().get(RELATED_TO).getAsJsonArray().size() > 0
 190             || json.getAsJsonObject().get(NODE_ACTIONS).getAsJsonArray().size() > 0)) {
 191             if (json.getAsJsonObject().get(PROPERTIES).getAsJsonArray()
 192                 .get(0).getAsJsonObject().get("key").getAsString().equals(AAIProperties.END_TS)) {
 193                 json.getAsJsonObject().add(PROPERTIES, new JsonArray());
 194             }
 195         }
 196
 197         return Optional.of(json);
 198     }
 199
 200     @Override
 201     protected Optional<JsonObject> getJsonFromVertex(Vertex input, Map<String, List<String>> properties) throws AAIFormatVertexException {
 202         return Optional.empty();
 203     }
 204
 205     private JsonArray getNodeActions(Vertex v, JsonObject json) {
 206         JsonArray nodeActions = new JsonArray();
 207         JsonObject action;
 208         if (v.property(AAIProperties.END_TS).isPresent()) {
 209             long deletedTs = (Long) v.property(AAIProperties.END_TS).value();
 210             if (isTsInRange(deletedTs)) {
 211                 action = new JsonObject();
 212                 action.addProperty("action", "DELETED");
 213                 action.addProperty(TIMESTAMP, deletedTs);
 214                 if (v.property(AAIProperties.END_TS).property(AAIProperties.SOURCE_OF_TRUTH).isPresent()) {
 215                     action.addProperty(SOT, v.property(AAIProperties.END_TS).<String>value(AAIProperties.SOURCE_OF_TRUTH));
 216                 }
 217                 if (v.property(AAIProperties.END_TS).property(AAIProperties.END_TX_ID).isPresent()) {
 218                     action.addProperty(TX_ID, v.property(AAIProperties.END_TS).<String>value(AAIProperties.END_TX_ID));
 219                 } else {
 220                     action.addProperty(TX_ID, "N/A");
 221                 }
 222                 nodeActions.add(action);
 223             }
 224         }
 225         long createdTs = json.get(TIMESTAMP).getAsLong();
 226         if (isTsInRange(createdTs)) {
 227             action = new JsonObject();
 228             action.addProperty("action", "CREATED");
 229             action.addProperty(TIMESTAMP, createdTs);
 230             action.addProperty(SOT, v.<String>value(AAIProperties.SOURCE_OF_TRUTH));
 231             if (v.property(AAIProperties.SOURCE_OF_TRUTH).property(AAIProperties.START_TX_ID).isPresent()) {
 232                 action.addProperty(TX_ID, v.property(AAIProperties.SOURCE_OF_TRUTH).<String>value(AAIProperties.START_TX_ID));
 233             } else {
 234                 action.addProperty(TX_ID, "N/A");
 235             }
 236             nodeActions.add(action);
 237         }
 238         return nodeActions;
 239     }
 240
 241     public JsonArray process(List<Object> queryResults) {
 242         JsonArray body = new JsonArray();
 243         Stream<Object> stream;
 244         if (queryResults.size() >= this.parallelThreshold()) {
 245             stream = queryResults.parallelStream();
 246         } else {
 247             stream = queryResults.stream();
 248         }
 249
 250         final boolean isParallel = stream.isParallel();
 251
 252         stream.map(o -> {
 253             try {
 254                 return this.formatObject(o);
 255             } catch (AAIFormatVertexException e) {
 256                 LOGGER.warn("Failed to format vertex, returning a partial list " + LogFormatTools.getStackTop(e));
 257             } catch (AAIFormatQueryResultFormatNotSupported e) {
 258                 LOGGER.warn("Failed to format result type of the query " + LogFormatTools.getStackTop(e));
 259             }
 260
 261             return Optional.<JsonObject>empty();
 262         }).filter(Optional::isPresent)
 263             .map(Optional::get)
 264             .forEach(json -> {
 265                 if (isParallel) {
 266                     synchronized (body) {
 267                         body.add(json);
 268                     }
 269                 } else {
 270                     body.add(json);
 271                 }
 272             });
 273         JsonArray result = organizeBody(body);
 274         result.forEach(jsonElement -> jsonElement.getAsJsonObject().remove(TIMESTAMP));
 275         return result;
 276     }
 277
 278     private JsonArray organizeBody(JsonArray body) {
 279
 280         final MultiValueMap<String, Integer> toBeMerged = new LinkedMultiValueMap<>();
 281         for (int i = 0; i < body.size(); i++) {
 282             toBeMerged.add(body.get(i).getAsJsonObject().get("uri").getAsString(), i);
 283         }
 284
 285         final List<List<Integer>> dupes = toBeMerged.values().stream().filter(l -> l.size() > 1).collect(Collectors.toList());
 286         if (dupes.isEmpty()) {
 287             return body;
 288         } else {
 289             Set<Integer> remove = new HashSet<>();
 290             for (List<Integer> dupe : dupes) {
 291                 dupe.sort((a,b) -> Long.compare(body.get(b).getAsJsonObject().get(TIMESTAMP).getAsLong(), body.get(a).getAsJsonObject().get(TIMESTAMP).getAsLong()));
 292                 int keep = dupe.remove(0);
 293                 for (Integer idx : dupe) {
 294                     body.get(keep).getAsJsonObject().getAsJsonArray(NODE_ACTIONS)
 295                         .addAll(body.get(idx).getAsJsonObject().getAsJsonArray(NODE_ACTIONS));
 296                     body.get(keep).getAsJsonObject().getAsJsonArray(PROPERTIES)
 297                         .addAll(body.get(idx).getAsJsonObject().getAsJsonArray(PROPERTIES));
 298                     body.get(keep).getAsJsonObject().getAsJsonArray(RELATED_TO)
 299                         .addAll(body.get(idx).getAsJsonObject().getAsJsonArray(RELATED_TO));
 300                     remove.add(idx);
 301                 }
 302             }
 303             final JsonArray newBody = new JsonArray();
 304             for (int i = 0; i < body.size(); i++) {
 305                 if (!remove.contains(i)) {
 306                     newBody.add(body.get(i));
 307                 }
 308             }
 309             return newBody;
 310         }
 311     }
 312
 313 }