asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/GraphMLConverter.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23 import java.io.BufferedInputStream;
  24 import java.io.BufferedOutputStream;
  25 import java.io.File;
  26 import java.io.FileInputStream;
  27 import java.io.FileOutputStream;
  28 import java.io.FileWriter;
  29 import java.io.IOException;
  30 import java.io.InputStream;
  31 import java.io.OutputStream;
  32 import java.util.ArrayList;
  33 import java.util.HashMap;
  34 import java.util.Iterator;
  35 import java.util.List;
  36 import java.util.Map;
  37 import java.util.Map.Entry;
  38
  39 import org.apache.commons.configuration.BaseConfiguration;
  40 import org.apache.commons.lang3.tuple.ImmutablePair;
  41 import org.apache.tinkerpop.gremlin.structure.Direction;
  42 import org.apache.tinkerpop.gremlin.structure.Edge;
  43 import org.apache.tinkerpop.gremlin.structure.Element;
  44 import org.apache.tinkerpop.gremlin.structure.Graph;
  45 import org.apache.tinkerpop.gremlin.structure.Property;
  46 import org.apache.tinkerpop.gremlin.structure.Vertex;
  47 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
  48 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
  49 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
  50 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
  51 import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
  52 import org.openecomp.sdc.asdctool.Utils;
  53 import org.openecomp.sdc.be.dao.neo4j.GraphPropertiesDictionary;
  54 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
  55 import org.slf4j.Logger;
  56 import org.slf4j.LoggerFactory;
  57
  58 import com.google.gson.Gson;
  59 import com.thinkaurelius.titan.core.TitanEdge;
  60 import com.thinkaurelius.titan.core.TitanFactory;
  61 import com.thinkaurelius.titan.core.TitanGraph;
  62 import com.thinkaurelius.titan.core.TitanGraphQuery;
  63 import com.thinkaurelius.titan.core.TitanVertex;
  64
  65 public class GraphMLConverter {
  66
  67         private static Logger log = LoggerFactory.getLogger(GraphMLConverter.class.getName());
  68
  69         private Gson gson = new Gson();
  70
  71         public boolean importGraph(String[] args) {
  72
  73                 TitanGraph graph = null;
  74                 try {
  75                         String titanFileLocation = args[1];
  76                         String inputFile = args[2];
  77                         graph = openGraph(titanFileLocation);
  78
  79                         List<ImmutablePair<String, String>> propertiesCriteriaToDelete = new ArrayList<>();
  80                         ImmutablePair<String, String> immutablePair1 = new ImmutablePair<String, String>("healthcheckis", "GOOD");
  81                         ImmutablePair<String, String> immutablePair2 = new ImmutablePair<String, String>("nodeLabel", "user");
  82                         ImmutablePair<String, String> immutablePair3 = new ImmutablePair<String, String>("nodeLabel",
  83                                         "resourceCategory");
  84                         ImmutablePair<String, String> immutablePair4 = new ImmutablePair<String, String>("nodeLabel",
  85                                         "serviceCategory");
  86
  87                         propertiesCriteriaToDelete.add(immutablePair1);
  88                         propertiesCriteriaToDelete.add(immutablePair2);
  89                         propertiesCriteriaToDelete.add(immutablePair3);
  90                         propertiesCriteriaToDelete.add(immutablePair4);
  91
  92                         boolean result = importJsonGraph(graph, inputFile, propertiesCriteriaToDelete);
  93
  94                         return result;
  95
  96                 } catch (Exception e) {
  97                         e.printStackTrace();
  98                         return false;
  99                 } finally {
 100                         if (graph != null) {
 101                                 // graph.shutdown();
 102                                 graph.close();
 103                         }
 104                 }
 105
 106         }
 107
 108         public boolean exportGraph(String[] args) {
 109
 110                 TitanGraph graph = null;
 111                 try {
 112                         String titanFileLocation = args[1];
 113                         String outputDirectory = args[2];
 114                         graph = openGraph(titanFileLocation);
 115
 116                         String result = exportJsonGraph(graph, outputDirectory);
 117
 118                         if (result == null) {
 119                                 return false;
 120                         }
 121
 122                         System.out.println("Exported file=" + result);
 123                 } catch (Exception e) {
 124                         e.printStackTrace();
 125                         return false;
 126                 } finally {
 127                         if (graph != null) {
 128                                 // graph.shutdown();
 129                                 graph.close();
 130                         }
 131                 }
 132
 133                 return true;
 134         }
 135
 136         public String exportGraphMl(String[] args) {
 137
 138                 TitanGraph graph = null;
 139                 String result = null;
 140                 try {
 141                         String titanFileLocation = args[1];
 142                         String outputDirectory = args[2];
 143                         graph = openGraph(titanFileLocation);
 144
 145                         result = exportGraphMl(graph, outputDirectory);
 146
 147                         System.out.println("Exported file=" + result);
 148                 } catch (Exception e) {
 149                         e.printStackTrace();
 150                         return null;
 151                 } finally {
 152                         if (graph != null) {
 153                                 graph.close();
 154                         }
 155                 }
 156
 157                 return result;
 158         }
 159
 160         public boolean findErrorInJsonGraph(String[] args) {
 161
 162                 TitanGraph graph = null;
 163                 try {
 164                         String titanFileLocation = args[1];
 165                         String outputDirectory = args[2];
 166                         graph = openGraph(titanFileLocation);
 167
 168                         String result = findErrorInJsonGraph(graph, outputDirectory);
 169
 170                         if (result == null) {
 171                                 return false;
 172                         }
 173
 174                         System.out.println("Exported file=" + result);
 175                 } catch (Exception e) {
 176                         e.printStackTrace();
 177                         return false;
 178                 } finally {
 179                         if (graph != null) {
 180                                 // graph.shutdown();
 181                                 graph.close();
 182                         }
 183                 }
 184
 185                 return true;
 186         }
 187
 188         public TitanGraph openGraph(String titanFileLocation) {
 189
 190                 TitanGraph graph = TitanFactory.open(titanFileLocation);
 191
 192                 return graph;
 193
 194         }
 195
 196         public String exportJsonGraph(TitanGraph graph, String outputDirectory) {
 197
 198                 String result = null;
 199
 200                 // GraphMLWriter graphMLWriter = new GraphMLWriter(graph);
 201
 202                 String outputFile = outputDirectory + File.separator + "exportGraph." + System.currentTimeMillis() + ".json";
 203
 204                 OutputStream out = null;
 205                 try {
 206                         out = new BufferedOutputStream(new FileOutputStream(outputFile));
 207
 208                         // GraphSONWriter.outputGraph(graph, outputFile);
 209                         final GraphSONWriter.Builder builder = GraphSONWriter.build();
 210                         final GraphSONMapper mapper = newGraphSONMapper(graph);
 211                         builder.mapper(mapper);
 212                         final GraphSONWriter writer = builder.create();
 213                         writer.writeGraph(out, graph);
 214
 215                         // GraphSONWriter create = GraphSONWriter.build(). create();
 216                         // create.writeGraph(out, graph);
 217
 218                         // graph.commit();
 219                         graph.tx().commit();
 220
 221                         result = outputFile;
 222
 223                 } catch (Exception e) {
 224                         e.printStackTrace();
 225                         graph.tx().rollback();
 226                 } finally {
 227                         try {
 228                                 if (out != null) {
 229                                         out.close();
 230                                 }
 231                         } catch (IOException e) {
 232                                 e.printStackTrace();
 233                         }
 234                 }
 235                 return result;
 236
 237         }
 238
 239         public String exportGraphMl(TitanGraph graph, String outputDirectory) {
 240                 String result = null;
 241                 String outputFile = outputDirectory + File.separator + "exportGraph." + System.currentTimeMillis() + ".graphml";
 242                 try {
 243                         try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(outputFile))) {
 244                                 graph.io(IoCore.graphml()).writer().normalize(true).create().writeGraph(os, graph);
 245                         }
 246                         result = outputFile;
 247                         graph.tx().commit();
 248                 } catch (Exception e) {
 249                         graph.tx().rollback();
 250                         e.printStackTrace();
 251                 }
 252                 return result;
 253
 254         }
 255
 256         private static GraphSONMapper newGraphSONMapper(final Graph graph) {
 257                 final GraphSONMapper.Builder builder = graph.io(IoCore.graphson()).mapper();
 258                 // Different failure with embedded type info.
 259                 // builder.embedTypes(true);
 260                 return builder.create();
 261         }
 262
 263         public boolean importJsonGraph(TitanGraph graph, String graphJsonFile,
 264                         List<ImmutablePair<String, String>> propertiesCriteriaToDelete) {
 265
 266                 boolean result = false;
 267
 268                 InputStream is = null;
 269
 270                 try {
 271
 272                         if (propertiesCriteriaToDelete != null) {
 273                                 for (Entry<String, String> entry : propertiesCriteriaToDelete
 274
 275                                 ) {
 276
 277                                         String key = entry.getKey();
 278                                         String value = entry.getValue();
 279                                         Iterator iterator = graph.query().has(key, value).vertices().iterator();
 280                                         while (iterator.hasNext()) {
 281                                                 Vertex vertex = (Vertex) iterator.next();
 282                                                 vertex.remove();
 283                                                 System.out.println("Remove vertex of type " + key + " and value " + value);
 284                                         }
 285
 286                                 }
 287                         }
 288                         File file = new File(graphJsonFile);
 289                         if (false == file.isFile()) {
 290                                 System.out.println("File " + graphJsonFile + " cannot be found.");
 291                                 return result;
 292                         }
 293
 294                         is = new BufferedInputStream(new FileInputStream(graphJsonFile));
 295                         System.out.println("Before importing file " + graphJsonFile);
 296
 297                         // GraphSONReader.inputGraph(graph, graphJsonFile);
 298                         GraphSONReader create = GraphSONReader.build().create();
 299                         create.readGraph(is, graph);
 300
 301                         // graph.commit();
 302                         graph.tx().commit();
 303
 304                         result = true;
 305
 306                 } catch (Exception e) {
 307                         System.out.println("Failed to import graph " + e.getMessage());
 308                         e.printStackTrace();
 309                         // graph.rollback();
 310                         graph.tx().rollback();
 311                 } finally {
 312                         try {
 313                                 if (is != null) {
 314                                         is.close();
 315                                 }
 316                         } catch (IOException e) {
 317                                 e.printStackTrace();
 318                         }
 319                 }
 320
 321                 return result;
 322
 323         }
 324
 325         public String findErrorInJsonGraph(TitanGraph graph, String outputDirectory) {
 326
 327                 boolean runVertexScan = false;
 328                 boolean runEdgeScan = false;
 329
 330                 String result = null;
 331
 332                 // GraphMLWriter graphMLWriter = new GraphMLWriter(graph);
 333
 334                 String outputFile = outputDirectory + File.separator + "exportGraph." + System.currentTimeMillis() + ".json";
 335
 336                 OutputStream out = null;
 337                 try {
 338                         out = new BufferedOutputStream(new FileOutputStream(outputFile));
 339
 340                         if (runEdgeScan) {
 341
 342                                 Vertex vertexFrom = null;
 343                                 Vertex vertexTo = null;
 344                                 Edge edge = null;
 345
 346                                 // Iterable<Edge> edges = graph.getEdges();
 347                                 // Iterable<Edge> edges = graph.query().edges();
 348                                 Iterable<TitanEdge> edges = graph.query().edges();
 349                                 // Iterator<Edge> iterator = edges.iterator();
 350                                 Iterator<TitanEdge> iterator = edges.iterator();
 351                                 while (iterator.hasNext()) {
 352
 353                                         try {
 354
 355                                                 edge = iterator.next();
 356
 357                                                 // vertexFrom = edge.getVertex(Direction.OUT);
 358                                                 // vertexTo = edge.getVertex(Direction.IN);
 359                                                 vertexFrom = edge.outVertex();
 360                                                 vertexTo = edge.inVertex();
 361
 362                                                 BaseConfiguration conf = new BaseConfiguration();
 363                                                 conf.setProperty("storage.backend", "inmemory");
 364                                                 TitanGraph openGraph = Utils.openGraph(conf);
 365
 366                                                 TitanVertex addVertexFrom = openGraph.addVertex();
 367                                                 // ElementHelper.setProperties(addVertexFrom,
 368                                                 // ElementHelper.getProperties(vertexFrom));
 369                                                 Utils.setProperties(addVertexFrom, Utils.getProperties(vertexFrom));
 370
 371                                                 TitanVertex addVertexTo = openGraph.addVertex();
 372                                                 // ElementHelper.setProperties(addVertexTo,
 373                                                 // ElementHelper.getProperties(vertexTo));
 374                                                 Utils.setProperties(addVertexTo, Utils.getProperties(vertexTo));
 375
 376                                                 // Edge addEdge = openGraph.addEdge(null, addVertexFrom,
 377                                                 // addVertexTo, edge.getLabel());
 378
 379                                                 // Edge edge = tGraph.addEdge(null,
 380                                                 // fromV.left().value(), toV.left().value(), type);
 381
 382                                                 Edge addEdge = addVertexFrom.addEdge(edge.label(), addVertexTo);
 383                                                 // ElementHelper.setProperties(addEdge,
 384                                                 // ElementHelper.getProperties(edge));
 385                                                 Utils.setProperties(addEdge, Utils.getProperties(edge));
 386
 387                                                 log.info("fromVertex={}", Utils.getProperties(vertexFrom));
 388                                                 log.info("toVertex={}", Utils.getProperties(vertexTo));
 389                                                 log.info("edge={} {} ",edge.label(),Utils.getProperties(edge));
 390
 391                                                 // GraphSONWriter.outputGraph(openGraph, outputFile);
 392                                                 GraphSONWriter create = GraphSONWriter.build().create();
 393                                                 create.writeGraph(out, openGraph);
 394
 395                                                 // openGraph.rollback();
 396                                                 openGraph.tx().rollback();
 397
 398                                         } catch (Exception e) {
 399                                                 e.printStackTrace();
 400
 401                                                 log.error("fromVertex={}", Utils.getProperties(vertexFrom));
 402                                                 log.error("toVertex={}", Utils.getProperties(vertexTo));
 403                                                 log.error("edge={} {} ",edge.label(),Utils.getProperties(edge));
 404
 405                                                 break;
 406
 407                                         }
 408                                 }
 409
 410                                 // graph.rollback();
 411                                 graph.tx().rollback();
 412
 413                         }
 414
 415                         if (runVertexScan) {
 416
 417                                 Vertex vertex = null;
 418                                 // Iterable<Vertex> vertices = graph.getVertices();
 419
 420                                 // Iterator<Vertex> iteratorVertex = vertices.iterator();
 421                                 Iterator<Vertex> iteratorVertex = graph.vertices();
 422                                 while (iteratorVertex.hasNext()) {
 423
 424                                         try {
 425
 426                                                 vertex = iteratorVertex.next();
 427
 428                                                 // Iterable<Edge> edges2 =
 429                                                 // vertex.getEdges(Direction.BOTH);
 430
 431                                                 // Iterator<Edge> iterator2 = edges2.iterator();
 432                                                 Iterator<Edge> iterator2 = vertex.edges(Direction.BOTH);
 433                                                 if (false == iterator2.hasNext()) {
 434
 435                                                         BaseConfiguration conf = new BaseConfiguration();
 436                                                         conf.setProperty("storage.backend", "inmemory");
 437                                                         TitanGraph openGraph = Utils.openGraph(conf);
 438
 439                                                         TitanVertex addVertexFrom = openGraph.addVertex();
 440                                                         Utils.setProperties(addVertexFrom, Utils.getProperties(vertex));
 441
 442                                                         log.info("fromVertex={}", Utils.getProperties(addVertexFrom));
 443
 444                                                         GraphSONWriter create = GraphSONWriter.build().create();
 445                                                         create.writeGraph(out, openGraph);
 446
 447                                                         openGraph.tx().rollback();
 448
 449                                                 }
 450
 451                                         } catch (Exception e) {
 452                                                 e.printStackTrace();
 453
 454                                                 Object property1 = vertex.value(GraphPropertiesDictionary.HEALTH_CHECK.getProperty());
 455                                                 System.out.println(property1);
 456
 457                                                 Object property2 = vertex.value("healthcheck");
 458                                                 System.out.println(property2);
 459
 460                                                 break;
 461
 462                                         }
 463                                 }
 464
 465                                 // graph.rollback();
 466                                 graph.tx().rollback();
 467
 468                         }
 469
 470                         // Iterable<Vertex> vertices2 =
 471                         // graph.getVertices(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(),
 472                         // "GOOD");
 473                         Iterable<TitanVertex> vertices2 = graph.query()
 474                                         .has(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(), "GOOD").vertices();
 475                         ;
 476                         Vertex next = vertices2.iterator().next();
 477
 478                         BaseConfiguration conf = new BaseConfiguration();
 479                         conf.setProperty("storage.backend", "inmemory");
 480                         TitanGraph openGraph = Utils.openGraph(conf);
 481
 482                         // TitanVertex addVertexFrom = openGraph.addVertex();
 483                         //
 484                         // addVertexFrom.setProperty(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(),
 485                         // "GOOD");
 486                         // addVertexFrom.setProperty("healthcheck",
 487                         // next.getProperty("healthcheck"));
 488                         //
 489                         // //next.remove();
 490                         //
 491                         // next.removeProperty("healthcheck");
 492                         // next.removeProperty("healthcheckis");
 493                         //
 494                         // next.remove();
 495
 496                         // GraphSONWriter.outputGraph(openGraph, outputFile);
 497
 498                         for (NodeTypeEnum nodeTypeEnum : NodeTypeEnum.values()) {
 499                                 removeNodesByLabel(graph, nodeTypeEnum.getName());
 500                         }
 501
 502                         // GraphSONWriter.outputGraph(graph, outputFile);
 503
 504                         GraphSONWriter create = GraphSONWriter.build().create();
 505                         create.writeGraph(out, graph);
 506
 507                         // graph.rollback();
 508                         graph.tx().rollback();
 509
 510                 } catch (Exception e) {
 511                         e.printStackTrace();
 512                         // graph.rollback();
 513                         graph.tx().rollback();
 514                 } finally {
 515                         try {
 516                                 if (out != null) {
 517                                         out.close();
 518                                 }
 519                         } catch (IOException e) {
 520                                 e.printStackTrace();
 521                         }
 522                 }
 523                 return result;
 524
 525         }
 526
 527         private void removeNodesByLabel(TitanGraph graph, String label) {
 528                 Iterable<TitanVertex> vertices = graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(), label)
 529                                 .vertices();
 530                 Iterator<TitanVertex> iterator = vertices.iterator();
 531                 while (iterator.hasNext()) {
 532                         Vertex next2 = iterator.next();
 533                         next2.remove();
 534                 }
 535         }
 536
 537         public static void clearGraph(TitanGraph graph) {
 538
 539                 Iterable<TitanVertex> vertices = graph.query().vertices();
 540
 541                 long erased = 0;
 542
 543                 if (vertices != null) {
 544                         Iterator<TitanVertex> iterator = vertices.iterator();
 545                         while (iterator.hasNext()) {
 546                                 Vertex vertex = iterator.next();
 547                                 // graph.removeVertex(vertex);
 548                                 vertex.remove();
 549                                 erased++;
 550                         }
 551
 552                 }
 553
 554                 System.out.println("After erasing " + erased + " vertices.");
 555                 // graph.commit();
 556                 graph.tx().commit();
 557         }
 558
 559         public String exportUsers(TitanGraph graph, String outputDirectory) {
 560
 561                 List<Map<String, Object>> users = new ArrayList<>();
 562                 String result = null;
 563
 564                 // GraphMLWriter graphMLWriter = new GraphMLWriter(graph);
 565
 566                 String outputFile = outputDirectory + File.separator + "users." + System.currentTimeMillis() + ".json";
 567
 568                 FileWriter fileWriter = null;
 569                 try {
 570
 571                         TitanGraphQuery graphQuery = graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(),
 572                                         NodeTypeEnum.User.getName());
 573
 574                         @SuppressWarnings("unchecked")
 575                         Iterable<TitanVertex> vertices = graphQuery.vertices();
 576
 577                         if (vertices != null) {
 578                                 for (Vertex v : vertices) {
 579                                         Map<String, Object> properties = getProperties(v);
 580                                         properties.remove(GraphPropertiesDictionary.LABEL.getProperty());
 581                                         users.add(properties);
 582                                 }
 583                         }
 584
 585                         graph.tx().commit();
 586
 587                         String jsonUsers = gson.toJson(users);
 588
 589                         fileWriter = new FileWriter(outputFile);
 590                         fileWriter.write(jsonUsers);
 591
 592                         result = outputFile;
 593
 594                 } catch (Exception e) {
 595                         e.printStackTrace();
 596                         graph.tx().rollback();
 597                 } finally {
 598                         try {
 599                                 if (fileWriter != null) {
 600                                         fileWriter.close();
 601                                 }
 602                         } catch (IOException e) {
 603                                 e.printStackTrace();
 604                         }
 605                 }
 606                 return result;
 607
 608         }
 609
 610         public Map<String, Object> getProperties(Element element) {
 611
 612                 Map<String, Object> result = new HashMap<String, Object>();
 613                 ;
 614
 615                 if (element.keys() != null && element.keys().size() > 0) {
 616                         Map<String, Property> propertyMap = ElementHelper.propertyMap(element,
 617                                         element.keys().toArray(new String[element.keys().size()]));
 618
 619                         for (Entry<String, Property> entry : propertyMap.entrySet()) {
 620                                 String key = entry.getKey();
 621                                 Object value = entry.getValue().value();
 622
 623                                 result.put(key, value);
 624                         }
 625                 }
 626                 return result;
 627         }
 628
 629         public boolean exportUsers(String[] args) {
 630
 631                 TitanGraph graph = null;
 632                 try {
 633                         String titanFileLocation = args[1];
 634                         String outputDirectory = args[2];
 635                         graph = openGraph(titanFileLocation);
 636
 637                         String result = exportUsers(graph, outputDirectory);
 638
 639                         if (result == null) {
 640                                 return false;
 641                         }
 642
 643                         System.out.println("Exported file=" + result);
 644                 } catch (Exception e) {
 645                         e.printStackTrace();
 646                         return false;
 647                 } finally {
 648                         if (graph != null) {
 649                                 // graph.shutdown();
 650                                 graph.close();
 651                         }
 652                 }
 653
 654                 return true;
 655         }
 656 }