champ-lib/champ-core/src/main/java/org/onap/aai/champcore/ie/GraphMLImporterExporter.java

   1 /**
   2  * ============LICENSE_START==========================================
   3  * org.onap.aai
   4  * ===================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * Copyright © 2017-2018 Amdocs
   7  * ===================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END============================================
  20  */
  21 package org.onap.aai.champcore.ie;
  22
  23 import java.io.IOException;
  24 import java.io.InputStream;
  25 import java.io.OutputStream;
  26 import java.util.ArrayList;
  27 import java.util.Collections;
  28 import java.util.HashMap;
  29 import java.util.HashSet;
  30 import java.util.LinkedList;
  31 import java.util.List;
  32 import java.util.Map;
  33 import java.util.Map.Entry;
  34 import java.util.Optional;
  35 import java.util.Set;
  36 import java.util.concurrent.atomic.AtomicInteger;
  37
  38 import javax.xml.parsers.DocumentBuilder;
  39 import javax.xml.parsers.DocumentBuilderFactory;
  40 import javax.xml.parsers.ParserConfigurationException;
  41 import javax.xml.stream.XMLOutputFactory;
  42 import javax.xml.stream.XMLStreamException;
  43 import javax.xml.stream.XMLStreamWriter;
  44
  45 import org.onap.aai.champcore.ChampAPI;
  46 import org.onap.aai.champcore.ChampGraph;
  47 import org.onap.aai.champcore.ChampTransaction;
  48 import org.onap.aai.champcore.exceptions.ChampMarshallingException;
  49 import org.onap.aai.champcore.exceptions.ChampObjectNotExistsException;
  50 import org.onap.aai.champcore.exceptions.ChampRelationshipNotExistsException;
  51 import org.onap.aai.champcore.exceptions.ChampSchemaViolationException;
  52 import org.onap.aai.champcore.exceptions.ChampTransactionException;
  53 import org.onap.aai.champcore.exceptions.ChampUnmarshallingException;
  54 import org.onap.aai.champcore.model.ChampObject;
  55 import org.onap.aai.champcore.model.ChampObjectIndex;
  56 import org.onap.aai.champcore.model.ChampRelationship;
  57 import org.slf4j.Logger;
  58 import org.slf4j.LoggerFactory;
  59 import org.w3c.dom.Document;
  60 import org.w3c.dom.NamedNodeMap;
  61 import org.w3c.dom.Node;
  62 import org.w3c.dom.NodeList;
  63 import org.xml.sax.InputSource;
  64 import org.xml.sax.SAXException;
  65
  66 public class GraphMLImporterExporter implements Importer, Exporter {
  67
  68         private static final Logger LOGGER = LoggerFactory.getLogger(GraphMLImporterExporter.class);
  69
  70         private static class GraphMLKey {
  71                 private final String id;
  72                 private final String attrName;
  73                 private final String attrType;
  74
  75                 public GraphMLKey(String id, String attrName, Class<?> attrType) {
  76                         this.id = id;
  77                         this.attrName = attrName;
  78
  79                         if (attrType.equals(Boolean.class)) {
  80                                 this.attrType = "boolean";
  81                         } else if (attrType.equals(Integer.class)) {
  82                                 this.attrType = "int";
  83                         } else if (attrType.equals(Long.class)) {
  84                                 this.attrType = "long";
  85                         } else if (attrType.equals(Float.class)) {
  86                                 this.attrType = "float";
  87                         } else if (attrType.equals(Double.class)) {
  88                                 this.attrType = "double";
  89                         } else if (attrType.equals(String.class)) {
  90                                 this.attrType = "string";
  91                         } else {
  92                                 throw new RuntimeException("Cannot handle type " + attrType + " in GraphML");
  93                         }
  94                 }
  95         }
  96
  97         public void importData(ChampAPI api, InputStream is) {
  98
  99                 try {
 100                         final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
 101                         factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
 102                         final DocumentBuilder builder = factory.newDocumentBuilder();
 103                         final InputSource inputSource = new InputSource(is);
 104                     final Document doc = builder.parse(inputSource);
 105
 106                         final Map<String, Map<String, String>> nodePropertyDefinitions = new HashMap<String, Map<String, String>> ();
 107                         final Map<String, Map<String, String>> edgePropertyDefinitions = new HashMap<String, Map<String, String>> ();
 108                         final Set<Map<String, String>> nodeDefaults = new HashSet<Map<String, String>> ();
 109                         final Set<Map<String, String>> edgeDefaults = new HashSet<Map<String, String>> ();
 110
 111                         final NodeList keys = doc.getElementsByTagName("key");
 112
 113                         for (int i = 0; i < keys.getLength(); i++) {
 114                                 final Node key = keys.item(i);
 115                                 final String id = key.getAttributes().getNamedItem("id").getNodeValue();
 116                                 final String attrName = key.getAttributes().getNamedItem("attr.name").getNodeValue();
 117                                 final String attrType = key.getAttributes().getNamedItem("attr.type").getNodeValue();
 118                                 final String elementType = key.getAttributes().getNamedItem("for").getNodeValue();
 119                                 final Map<String, String> propertyDefinitions = new HashMap<String, String> ();
 120
 121                                 propertyDefinitions.put("attr.name", attrName);
 122                                 propertyDefinitions.put("attr.type",  attrType);
 123
 124                                 final NodeList keyChildren = key.getChildNodes();
 125
 126                                 for (int j = 0; j < keyChildren.getLength(); j++) {
 127                                         final Node keyChild = keyChildren.item(j);
 128
 129                                         if (keyChild.getNodeType() != Node.ELEMENT_NODE) continue;
 130
 131                                         if (keyChild.getNodeName().equals("default")) {
 132                                                 propertyDefinitions.put("default", keyChild.getFirstChild().getNodeValue());
 133
 134                                                 if (elementType.equals("node")) nodeDefaults.add(propertyDefinitions);
 135                                                 else if (elementType.equals("edge")) edgeDefaults.add(propertyDefinitions);
 136                                         }
 137                                 }
 138
 139                                 if (elementType.equals("node")) {
 140                                         nodePropertyDefinitions.put(id, propertyDefinitions);
 141                                 } else if (elementType.equals("edge")) {
 142                                         edgePropertyDefinitions.put(id, propertyDefinitions);
 143                                 } else {
 144                                         LOGGER.warn("Unknown element type {}, skipping", elementType);
 145                                 }
 146                         }
 147
 148                         final NodeList graphs = doc.getElementsByTagName("graph");
 149
 150                         for (int i = 0; i < graphs.getLength(); i++) {
 151                                 final Node graph = graphs.item(i);
 152                                 final String graphName = graph.getAttributes().getNamedItem("id").getNodeValue();
 153                                 final NodeList nodesAndEdges = graph.getChildNodes();
 154
 155                                 List<String> fields = new ArrayList<String>();
 156                             fields.add("importAssignedId");
 157                                 api.getGraph(graphName).storeObjectIndex(ChampObjectIndex.create()
 158                                                                                                                                                         .ofName("importAssignedId")
 159                                                                                                                                                         .onAnyType()
 160                                                                                                                                                         .forFields(fields)
 161                                                                                                                                                         .build());
 162
 163                                 for (int j = 0; j < nodesAndEdges.getLength(); j++) {
 164                                         final Node nodeOrEdge = nodesAndEdges.item(j);
 165
 166                                         if (nodeOrEdge.getNodeType() != Node.ELEMENT_NODE) continue;
 167
 168                                         if (nodeOrEdge.getNodeName().equals("node")) {
 169                                                 writeNode(api.getGraph(graphName), nodeOrEdge, nodePropertyDefinitions, nodeDefaults);
 170                                         } else if (nodeOrEdge.getNodeName().equals("edge")) {
 171                                                 writeEdge(api.getGraph(graphName), nodeOrEdge, edgePropertyDefinitions, edgeDefaults);
 172                                         } else {
 173                                                 LOGGER.warn("Unknown object {} found in graphML, skipping", nodeOrEdge.getNodeName());
 174                                         }
 175                                 }
 176                         }
 177                 } catch (ParserConfigurationException e) {
 178                         throw new RuntimeException("Failed to setup DocumentBuilder", e);
 179                 } catch (SAXException e) {
 180                         throw new RuntimeException("Failed to parse input stream", e);
 181                 } catch (IOException e) {
 182                         throw new RuntimeException("Failed to parse input stream", e);
 183                 }
 184         }
 185
 186         private void writeEdge(ChampGraph graph, Node edge, Map<String, Map<String, String>> edgePropertyDefinitions, Set<Map<String, String>> edgeDefaults) {
 187                 final NamedNodeMap edgeAttributes = edge.getAttributes();
 188                 final NodeList data = edge.getChildNodes();
 189                 final Object sourceKey = edgeAttributes.getNamedItem("source").getNodeValue();
 190                 final Object targetKey = edgeAttributes.getNamedItem("target").getNodeValue();
 191                 ChampObject sourceObject=null;
 192                 ChampObject targetObject=null;
 193
 194                 try {
 195                         final Optional<ChampObject> source = graph.queryObjects(Collections.singletonMap("importAssignedId", sourceKey), Optional.empty()).findFirst();
 196                         final Optional<ChampObject> target = graph.queryObjects(Collections.singletonMap("importAssignedId", targetKey), Optional.empty()).findFirst();
 197
 198                         if (!source.isPresent()) {
 199                                 sourceObject = graph.storeObject(ChampObject.create()
 200                                                                                                                 .ofType("undefined")
 201                                                                                                                 .withoutKey()
 202                                                                                                                 .build(),
 203                                                                                                  Optional.empty());
 204                         } else sourceObject = source.get();
 205
 206                         if (!target.isPresent()) {
 207                                 targetObject = graph.storeObject(ChampObject.create()
 208                                                                                                                 .ofType("undefined")
 209                                                                                                                 .withoutKey()
 210                                                                                                                 .build(),
 211                                                                                                  Optional.empty());
 212                         } else targetObject = target.get();
 213
 214                 } catch (ChampMarshallingException e) {
 215                         LOGGER.error("Failed to marshall object to backend type, skipping this edge", e);
 216                         return;
 217                 } catch (ChampSchemaViolationException e) {
 218                         LOGGER.error("Source/target object violates schema constraint(s)", e);
 219                         return;
 220                 } catch (ChampObjectNotExistsException e) {
 221                         LOGGER.error("Failed to update existing source/target ChampObject", e);
 222                         return;
 223                 } catch (ChampTransactionException e) {
 224             LOGGER.error("Failed to commit or rollback transaction", e);
 225                 }
 226
 227                 final ChampRelationship.Builder champRelBuilder = new ChampRelationship.Builder(sourceObject, targetObject, "undefined");
 228
 229                 for (Map<String, String> defaultProperty : edgeDefaults) {
 230                         champRelBuilder.property(defaultProperty.get("attr.name"), defaultProperty.get("default"));
 231                 }
 232
 233                 for (int k = 0; k < data.getLength(); k++) {
 234                         final Node datum = data.item(k);
 235
 236                         if (datum.getNodeType() != Node.ELEMENT_NODE) continue;
 237
 238                         final String nodeProperty = datum.getAttributes().getNamedItem("key").getNodeValue();
 239                         final Map<String, String> nodePropertyDefinition = edgePropertyDefinitions.get(nodeProperty);
 240
 241                         switch (nodePropertyDefinition.get("attr.type")) {
 242                         case "boolean":
 243                                 champRelBuilder.property(nodePropertyDefinition.get("attr.name"), Boolean.valueOf(datum.getFirstChild().getNodeValue()));
 244                                 break;
 245                         case "int":
 246                                 champRelBuilder.property(nodePropertyDefinition.get("attr.name"), Integer.valueOf(datum.getFirstChild().getNodeValue()));
 247                                 break;
 248                         case "long":
 249                                 champRelBuilder.property(nodePropertyDefinition.get("attr.name"), Long.valueOf(datum.getFirstChild().getNodeValue()));
 250                                 break;
 251                         case "float":
 252                                 champRelBuilder.property(nodePropertyDefinition.get("attr.name"), Float.valueOf(datum.getFirstChild().getNodeValue()));
 253                                 break;
 254                         case "double":
 255                                 champRelBuilder.property(nodePropertyDefinition.get("attr.name"), Double.valueOf(datum.getFirstChild().getNodeValue()));
 256                                 break;
 257                         case "string":
 258                                 champRelBuilder.property(nodePropertyDefinition.get("attr.name"), datum.getFirstChild().getNodeValue());
 259                                 break;
 260                         default:
 261                                 throw new RuntimeException("Unknown node property attr.type " + nodePropertyDefinition.get("attr.type"));
 262                         }
 263                 }
 264
 265                 final ChampRelationship relToStore = champRelBuilder.build();
 266
 267                 try {
 268                         graph.storeRelationship(relToStore, Optional.empty());
 269                 } catch (ChampMarshallingException e) {
 270                         LOGGER.warn("Failed to marshall ChampObject to backend type", e);
 271                 } catch (ChampSchemaViolationException e) {
 272                         LOGGER.error("Failed to store object (schema violated): " + relToStore, e);
 273                 } catch (ChampRelationshipNotExistsException e) {
 274                         LOGGER.error("Failed to update existing ChampRelationship", e);
 275                 } catch (ChampObjectNotExistsException e) {
 276                         LOGGER.error("Objects bound to relationship do not exist (should never happen)");
 277                 } catch (ChampUnmarshallingException e) {
 278                         LOGGER.error("Failed to unmarshall ChampObject to backend type");
 279                 } catch (ChampTransactionException e) {
 280                     LOGGER.error("Failed to commit or rollback transaction");
 281                 }
 282
 283         }
 284
 285         private void writeNode(ChampGraph graph, Node node, Map<String, Map<String, String>> nodePropertyDefinitions, Set<Map<String, String>> nodeDefaults) {
 286                 final NamedNodeMap nodeAttributes = node.getAttributes();
 287                 final Object importAssignedId = nodeAttributes.getNamedItem("id").getNodeValue();
 288                 final NodeList data = node.getChildNodes();
 289                 final Map<String, Object> properties = new HashMap<String, Object> ();
 290
 291                 for (int k = 0; k < data.getLength(); k++) {
 292                         final Node datum = data.item(k);
 293
 294                         if (datum.getNodeType() != Node.ELEMENT_NODE) continue;
 295
 296                         final String nodeProperty = datum.getAttributes().getNamedItem("key").getNodeValue();
 297                         final Map<String, String> nodePropertyDefinition = nodePropertyDefinitions.get(nodeProperty);
 298
 299                         switch (nodePropertyDefinition.get("attr.type")) {
 300                         case "boolean":
 301                                 properties.put(nodePropertyDefinition.get("attr.name"), Boolean.valueOf(datum.getFirstChild().getNodeValue()));
 302                                 break;
 303                         case "int":
 304                                 properties.put(nodePropertyDefinition.get("attr.name"), Integer.valueOf(datum.getFirstChild().getNodeValue()));
 305                                 break;
 306                         case "long":
 307                                 properties.put(nodePropertyDefinition.get("attr.name"), Long.valueOf(datum.getFirstChild().getNodeValue()));
 308                                 break;
 309                         case "float":
 310                                 properties.put(nodePropertyDefinition.get("attr.name"), Float.valueOf(datum.getFirstChild().getNodeValue()));
 311                                 break;
 312                         case "double":
 313                                 properties.put(nodePropertyDefinition.get("attr.name"), Double.valueOf(datum.getFirstChild().getNodeValue()));
 314                                 break;
 315                         case "string":
 316                                 properties.put(nodePropertyDefinition.get("attr.name"), datum.getFirstChild().getNodeValue());
 317                                 break;
 318                         default:
 319                                 throw new RuntimeException("Unknown node property attr.type " + nodePropertyDefinition.get("attr.type"));
 320                         }
 321                 }
 322
 323                 if (!properties.containsKey("type")) throw new RuntimeException("No type provided for object (was this GraphML exported by Champ?)");
 324
 325                 final ChampObject.Builder champObjBuilder = new ChampObject.Builder((String) properties.get("type"));
 326
 327                 for (Map<String, String> defaultProperty : nodeDefaults) {
 328                         champObjBuilder.property(defaultProperty.get("attr.name"), defaultProperty.get("default"));
 329                 }
 330
 331                 properties.remove("type");
 332
 333                 champObjBuilder.properties(properties)
 334                                                 .property("importAssignedId", importAssignedId);
 335
 336                 final ChampObject objectToStore = champObjBuilder.build();
 337
 338                 try {
 339                   graph.storeObject(objectToStore, Optional.empty());
 340                 } catch (ChampMarshallingException e) {
 341                         LOGGER.warn("Failed to marshall ChampObject to backend type", e);
 342                 } catch (ChampSchemaViolationException e) {
 343                         LOGGER.error("Failed to store object (schema violated): " + objectToStore, e);
 344                 } catch (ChampObjectNotExistsException e) {
 345                         LOGGER.error("Failed to update existing ChampObject", e);
 346                 } catch (ChampTransactionException e) {
 347           LOGGER.error("Failed to commit or rollback transaction");
 348         }
 349         }
 350
 351         @Override
 352         public void exportData(ChampGraph graph, OutputStream os) {
 353
 354                 final XMLOutputFactory output = XMLOutputFactory.newInstance();
 355
 356                 try {
 357                         final XMLStreamWriter writer = output.createXMLStreamWriter(os);
 358
 359                         writer.writeStartDocument();
 360                         writer.writeStartElement("graphml");
 361                         writer.writeDefaultNamespace("http://graphml.graphdrawing.org/xmlns");
 362                         writer.writeNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
 363                         writer.writeAttribute("http://www.w3.org/2001/XMLSchema-instance", "schemaLocation", "http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd");
 364
 365                         final List<ChampObject> nodes = new LinkedList<ChampObject> ();
 366                         final List<ChampRelationship> edges = new LinkedList<ChampRelationship> ();
 367                         final Map<String, GraphMLKey> nodeKeys = new HashMap<String, GraphMLKey> ();
 368                         final Map<String, GraphMLKey> edgeKeys = new HashMap<String, GraphMLKey> ();
 369                         final AtomicInteger elementCount = new AtomicInteger();
 370
 371                         graph.queryObjects(Collections.emptyMap(), Optional.empty()).forEach(object -> {
 372                                 nodes.add(object);
 373
 374                                 for (Map.Entry<String, Object> property : object.getProperties().entrySet()) {
 375                                         if (nodeKeys.containsKey(property.getKey())) continue;
 376
 377                                         nodeKeys.put(property.getKey(), new GraphMLKey("d" + elementCount.incrementAndGet(), property.getKey(), property.getValue().getClass()));
 378                                 }
 379
 380                                 nodeKeys.put("type", new GraphMLKey("d" + elementCount.incrementAndGet(), "type", String.class));
 381                         });
 382
 383                         graph.queryRelationships(Collections.emptyMap(), Optional.empty()).forEach(relationship -> {
 384                                 edges.add(relationship);
 385
 386                                 for (Map.Entry<String, Object> property : relationship.getProperties().entrySet()) {
 387                                         if (nodeKeys.containsKey(property.getKey())) continue;
 388
 389                                         edgeKeys.put(property.getKey(), new GraphMLKey("d" + elementCount.incrementAndGet(), property.getKey(), property.getValue().getClass()));
 390                                 }
 391
 392                                 edgeKeys.put("type", new GraphMLKey("d" + elementCount.incrementAndGet(), "type", String.class));
 393                         });
 394
 395                         for (Entry<String, GraphMLKey> nodeKey : nodeKeys.entrySet()) {
 396                                 final GraphMLKey graphMlKey = nodeKey.getValue();
 397
 398                                 writer.writeStartElement("key");
 399                                 writer.writeAttribute("id", graphMlKey.id);
 400                                 writer.writeAttribute("for", "node");
 401                                 writer.writeAttribute("attr.name", graphMlKey.attrName);
 402                                 writer.writeAttribute("attr.type", graphMlKey.attrType);
 403                                 writer.writeEndElement();
 404                         }
 405
 406                         for (Entry<String, GraphMLKey> edgeKey : edgeKeys.entrySet()) {
 407                                 final GraphMLKey graphMlKey = edgeKey.getValue();
 408
 409                                 writer.writeStartElement("key");
 410                                 writer.writeAttribute("id", graphMlKey.id);
 411                                 writer.writeAttribute("for", "edge");
 412                                 writer.writeAttribute("attr.name", graphMlKey.attrName);
 413                                 writer.writeAttribute("attr.type", graphMlKey.attrType);
 414                                 writer.writeEndElement();
 415                         }
 416
 417                         for (ChampObject object : nodes) {
 418                                 try {
 419                                         writer.writeStartElement("node");
 420                                         writer.writeAttribute("id", String.valueOf(object.getKey().get()));
 421
 422                                         writer.writeStartElement("data");
 423                                         writer.writeAttribute("key", nodeKeys.get("type").id);
 424                                         writer.writeCharacters(object.getType());
 425                                         writer.writeEndElement();
 426
 427                                         for (Entry<String, Object> property : object.getProperties().entrySet()) {
 428                                                 final GraphMLKey key = nodeKeys.get(property.getKey());
 429
 430                                                 writer.writeStartElement("data");
 431                                                 writer.writeAttribute("key", key.id);
 432                                                 writer.writeCharacters(String.valueOf(property.getValue()));
 433                                                 writer.writeEndElement();
 434                                         }
 435
 436                                         writer.writeEndElement();
 437                                 } catch (XMLStreamException e) {
 438                                         throw new RuntimeException("Failed to write edge to output stream", e);
 439                                 }
 440                         }
 441
 442                         for (ChampRelationship relationship : edges) {
 443                                 try {
 444                                         writer.writeStartElement("edge");
 445                                         writer.writeAttribute("id", String.valueOf(relationship.getKey().get()));
 446
 447                                         writer.writeStartElement("data");
 448                                         writer.writeAttribute("key", edgeKeys.get("type").id);
 449                                         writer.writeCharacters(relationship.getType());
 450                                         writer.writeEndElement();
 451
 452                                         for (Entry<String, Object> property : relationship.getProperties().entrySet()) {
 453                                                 final GraphMLKey key = edgeKeys.get(property.getKey());
 454
 455                                                 writer.writeStartElement("data");
 456                                                 writer.writeAttribute("key", key.id);
 457                                                 writer.writeCharacters(String.valueOf(property.getValue()));
 458                                                 writer.writeEndElement();
 459                                         }
 460
 461                                         writer.writeEndElement();
 462                                 } catch (XMLStreamException e) {
 463                                         throw new RuntimeException("Failed to write edge to output stream", e);
 464                                 }
 465                         }
 466
 467                         writer.writeEndElement();
 468                         writer.writeEndDocument();
 469                         writer.flush();
 470                 } catch (XMLStreamException | ChampTransactionException e) {
 471                         throw new RuntimeException(e);
 472                 }
 473         }
 474 }