asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/GraphMLConverter.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23 import com.google.gson.Gson;
  24 import com.thinkaurelius.titan.core.*;
  25 import org.apache.commons.configuration.BaseConfiguration;
  26 import org.apache.commons.lang3.tuple.ImmutablePair;
  27 import org.apache.tinkerpop.gremlin.structure.*;
  28 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
  29 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
  30 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
  31 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
  32 import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
  33 import org.openecomp.sdc.asdctool.Utils;
  34 import org.openecomp.sdc.be.dao.neo4j.GraphPropertiesDictionary;
  35 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
  36 import org.openecomp.sdc.common.log.wrappers.Logger;
  37
  38 import java.io.*;
  39 import java.util.*;
  40 import java.util.Map.Entry;
  41
  42 public class GraphMLConverter {
  43
  44         private static Logger log = Logger.getLogger(GraphMLConverter.class.getName());
  45
  46         private Gson gson = new Gson();
  47
  48         public boolean importGraph(String[] args) {
  49
  50                 TitanGraph graph = null;
  51                 try {
  52                         String titanFileLocation = args[1];
  53                         String inputFile = args[2];
  54                         graph = openGraph(titanFileLocation);
  55
  56                         List<ImmutablePair<String, String>> propertiesCriteriaToDelete = new ArrayList<>();
  57                         ImmutablePair<String, String> immutablePair1 = new ImmutablePair<String, String>("healthcheckis", "GOOD");
  58                         ImmutablePair<String, String> immutablePair2 = new ImmutablePair<String, String>("nodeLabel", "user");
  59                         ImmutablePair<String, String> immutablePair3 = new ImmutablePair<String, String>("nodeLabel",
  60                                         "resourceCategory");
  61                         ImmutablePair<String, String> immutablePair4 = new ImmutablePair<String, String>("nodeLabel",
  62                                         "serviceCategory");
  63
  64                         propertiesCriteriaToDelete.add(immutablePair1);
  65                         propertiesCriteriaToDelete.add(immutablePair2);
  66                         propertiesCriteriaToDelete.add(immutablePair3);
  67                         propertiesCriteriaToDelete.add(immutablePair4);
  68
  69                         boolean result = importJsonGraph(graph, inputFile, propertiesCriteriaToDelete);
  70
  71                         return result;
  72
  73                 } catch (Exception e) {
  74                         log.info("import graph failed - {} " , e);
  75                         return false;
  76                 } finally {
  77                         if (graph != null) {
  78                                 // graph.shutdown();
  79                                 graph.close();
  80                         }
  81                 }
  82
  83         }
  84
  85         public boolean exportGraph(String[] args) {
  86
  87                 TitanGraph graph = null;
  88                 try {
  89                         String titanFileLocation = args[1];
  90                         String outputDirectory = args[2];
  91                         graph = openGraph(titanFileLocation);
  92
  93                         String result = exportJsonGraph(graph, outputDirectory);
  94
  95                         if (result == null) {
  96                                 return false;
  97                         }
  98
  99                         System.out.println("Exported file=" + result);
 100                 } catch (Exception e) {
 101                         log.info("export graph failed -{}" , e);
 102                         return false;
 103                 } finally {
 104                         if (graph != null) {
 105                                 // graph.shutdown();
 106                                 graph.close();
 107                         }
 108                 }
 109
 110                 return true;
 111         }
 112
 113         public String exportGraphMl(String[] args) {
 114
 115                 TitanGraph graph = null;
 116                 String result = null;
 117                 try {
 118                         String titanFileLocation = args[1];
 119                         String outputDirectory = args[2];
 120                         graph = openGraph(titanFileLocation);
 121
 122                         result = exportGraphMl(graph, outputDirectory);
 123
 124                         System.out.println("Exported file=" + result);
 125                 } catch (Exception e) {
 126                         log.info("export exportGraphMl failed - {}" , e);
 127                         return null;
 128                 } finally {
 129                         if (graph != null) {
 130                                 graph.close();
 131                         }
 132                 }
 133
 134                 return result;
 135         }
 136
 137         public boolean findErrorInJsonGraph(String[] args) {
 138
 139                 TitanGraph graph = null;
 140                 try {
 141                         String titanFileLocation = args[1];
 142                         String outputDirectory = args[2];
 143                         graph = openGraph(titanFileLocation);
 144
 145                         String result = findErrorInJsonGraph(graph, outputDirectory);
 146
 147                         if (result == null) {
 148                                 return false;
 149                         }
 150
 151                         System.out.println("Exported file=" + result);
 152                 } catch (Exception e) {
 153                         log.info("find Error In Json Graph failed - {}" , e);
 154                         return false;
 155                 } finally {
 156                         if (graph != null) {
 157                                 // graph.shutdown();
 158                                 graph.close();
 159                         }
 160                 }
 161
 162                 return true;
 163         }
 164
 165         public TitanGraph openGraph(String titanFileLocation) {
 166
 167                 TitanGraph graph = TitanFactory.open(titanFileLocation);
 168
 169                 return graph;
 170
 171         }
 172
 173         public String exportJsonGraph(TitanGraph graph, String outputDirectory) {
 174
 175                 String result = null;
 176
 177                 // GraphMLWriter graphMLWriter = new GraphMLWriter(graph);
 178
 179                 String outputFile = outputDirectory + File.separator + "exportGraph." + System.currentTimeMillis() + ".json";
 180
 181                 OutputStream out = null;
 182                 try {
 183                         out = new BufferedOutputStream(new FileOutputStream(outputFile));
 184
 185                         // GraphSONWriter.outputGraph(graph, outputFile);
 186                         final GraphSONWriter.Builder builder = GraphSONWriter.build();
 187                         final GraphSONMapper mapper = newGraphSONMapper(graph);
 188                         builder.mapper(mapper);
 189                         final GraphSONWriter writer = builder.create();
 190                         writer.writeGraph(out, graph);
 191
 192                         // GraphSONWriter create = GraphSONWriter.build(). create();
 193                         // create.writeGraph(out, graph);
 194
 195                         // graph.commit();
 196                         graph.tx().commit();
 197
 198                         result = outputFile;
 199
 200                 } catch (Exception e) {
 201                         log.info("export Json Graph failed - {}" , e);
 202                         graph.tx().rollback();
 203                 } finally {
 204                         try {
 205                                 if (out != null) {
 206                                         out.close();
 207                                 }
 208                         } catch (IOException e) {
 209                                 log.info("close FileOutputStream failed - {}" , e);
 210                         }
 211                 }
 212                 return result;
 213
 214         }
 215
 216         public String exportGraphMl(TitanGraph graph, String outputDirectory) {
 217                 String result = null;
 218                 String outputFile = outputDirectory + File.separator + "exportGraph." + System.currentTimeMillis() + ".graphml";
 219                 try {
 220                         try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(outputFile))) {
 221                                 graph.io(IoCore.graphml()).writer().normalize(true).create().writeGraph(os, graph);
 222                         }
 223                         result = outputFile;
 224                         graph.tx().commit();
 225                 } catch (Exception e) {
 226                         graph.tx().rollback();
 227                         log.info("export Graph Ml failed - {}" , e);
 228                 }
 229                 return result;
 230
 231         }
 232
 233         private static GraphSONMapper newGraphSONMapper(final Graph graph) {
 234                 final GraphSONMapper.Builder builder = graph.io(IoCore.graphson()).mapper();
 235                 // Different failure with embedded type info.
 236                 // builder.embedTypes(true);
 237                 return builder.create();
 238         }
 239
 240         public boolean importJsonGraph(TitanGraph graph, String graphJsonFile,
 241                         List<ImmutablePair<String, String>> propertiesCriteriaToDelete) {
 242
 243                 boolean result = false;
 244
 245                 InputStream is = null;
 246
 247                 try {
 248
 249                         if (propertiesCriteriaToDelete != null) {
 250                                 for (Entry<String, String> entry : propertiesCriteriaToDelete
 251
 252                                 ) {
 253
 254                                         String key = entry.getKey();
 255                                         String value = entry.getValue();
 256                                         Iterator iterator = graph.query().has(key, value).vertices().iterator();
 257                                         while (iterator.hasNext()) {
 258                                                 Vertex vertex = (Vertex) iterator.next();
 259                                                 vertex.remove();
 260                                                 System.out.println("Remove vertex of type " + key + " and value " + value);
 261                                         }
 262
 263                                 }
 264                         }
 265                         File file = new File(graphJsonFile);
 266                         if (false == file.isFile()) {
 267                                 System.out.println("File " + graphJsonFile + " cannot be found.");
 268                                 return result;
 269                         }
 270
 271                         is = new BufferedInputStream(new FileInputStream(graphJsonFile));
 272                         System.out.println("Before importing file " + graphJsonFile);
 273
 274                         // GraphSONReader.inputGraph(graph, graphJsonFile);
 275                         GraphSONReader create = GraphSONReader.build().create();
 276                         create.readGraph(is, graph);
 277
 278                         // graph.commit();
 279                         graph.tx().commit();
 280
 281                         result = true;
 282
 283                 } catch (Exception e) {
 284                         System.out.println("Failed to import graph " + e.getMessage());
 285                         log.info("Failed to import graph - {}" , e);
 286                         // graph.rollback();
 287                         graph.tx().rollback();
 288                 } finally {
 289                         try {
 290                                 if (is != null) {
 291                                         is.close();
 292                                 }
 293                         } catch (IOException e) {
 294                                 log.info("close FileOutputStream failed - {}" , e);
 295                         }
 296                 }
 297
 298                 return result;
 299
 300         }
 301
 302         public String findErrorInJsonGraph(TitanGraph graph, String outputDirectory) {
 303
 304                 boolean runVertexScan = false;
 305                 boolean runEdgeScan = false;
 306
 307                 String result = null;
 308
 309                 // GraphMLWriter graphMLWriter = new GraphMLWriter(graph);
 310
 311                 String outputFile = outputDirectory + File.separator + "exportGraph." + System.currentTimeMillis() + ".json";
 312
 313                 OutputStream out = null;
 314                 try {
 315                         out = new BufferedOutputStream(new FileOutputStream(outputFile));
 316
 317                         if (runEdgeScan) {
 318
 319                                 Vertex vertexFrom = null;
 320                                 Vertex vertexTo = null;
 321                                 Edge edge = null;
 322
 323                                 // Iterable<Edge> edges = graph.getEdges();
 324                                 // Iterable<Edge> edges = graph.query().edges();
 325                                 Iterable<TitanEdge> edges = graph.query().edges();
 326                                 // Iterator<Edge> iterator = edges.iterator();
 327                                 Iterator<TitanEdge> iterator = edges.iterator();
 328                                 while (iterator.hasNext()) {
 329
 330                                         try {
 331
 332                                                 edge = iterator.next();
 333
 334                                                 // vertexFrom = edge.getVertex(Direction.OUT);
 335                                                 // vertexTo = edge.getVertex(Direction.IN);
 336                                                 vertexFrom = edge.outVertex();
 337                                                 vertexTo = edge.inVertex();
 338
 339                                                 BaseConfiguration conf = new BaseConfiguration();
 340                                                 conf.setProperty("storage.backend", "inmemory");
 341                                                 TitanGraph openGraph = Utils.openGraph(conf);
 342
 343                                                 TitanVertex addVertexFrom = openGraph.addVertex();
 344                                                 // ElementHelper.setProperties(addVertexFrom,
 345                                                 // ElementHelper.getProperties(vertexFrom));
 346                                                 Utils.setProperties(addVertexFrom, Utils.getProperties(vertexFrom));
 347
 348                                                 TitanVertex addVertexTo = openGraph.addVertex();
 349                                                 // ElementHelper.setProperties(addVertexTo,
 350                                                 // ElementHelper.getProperties(vertexTo));
 351                                                 Utils.setProperties(addVertexTo, Utils.getProperties(vertexTo));
 352
 353                                                 // Edge addEdge = openGraph.addEdge(null, addVertexFrom,
 354                                                 // addVertexTo, edge.getLabel());
 355
 356                                                 // Edge edge = tGraph.addEdge(null,
 357                                                 // fromV.left().value(), toV.left().value(), type);
 358
 359                                                 Edge addEdge = addVertexFrom.addEdge(edge.label(), addVertexTo);
 360                                                 // ElementHelper.setProperties(addEdge,
 361                                                 // ElementHelper.getProperties(edge));
 362                                                 Utils.setProperties(addEdge, Utils.getProperties(edge));
 363
 364                                                 log.info("fromVertex={}", Utils.getProperties(vertexFrom));
 365                                                 log.info("toVertex={}", Utils.getProperties(vertexTo));
 366                                                 log.info("edge={} {} ",edge.label(),Utils.getProperties(edge));
 367
 368                                                 // GraphSONWriter.outputGraph(openGraph, outputFile);
 369                                                 GraphSONWriter create = GraphSONWriter.build().create();
 370                                                 create.writeGraph(out, openGraph);
 371
 372                                                 // openGraph.rollback();
 373                                                 openGraph.tx().rollback();
 374
 375                                         } catch (Exception e) {
 376                                                 log.info("run Edge Scan failed - {}" , e);
 377
 378                                                 log.error("fromVertex={}", Utils.getProperties(vertexFrom));
 379                                                 log.error("toVertex={}", Utils.getProperties(vertexTo));
 380                                                 log.error("edge={} {} ",edge.label(),Utils.getProperties(edge));
 381
 382                                                 break;
 383
 384                                         }
 385                                 }
 386
 387                                 // graph.rollback();
 388                                 graph.tx().rollback();
 389
 390                         }
 391
 392                         if (runVertexScan) {
 393
 394                                 Vertex vertex = null;
 395                                 // Iterable<Vertex> vertices = graph.getVertices();
 396
 397                                 // Iterator<Vertex> iteratorVertex = vertices.iterator();
 398                                 Iterator<Vertex> iteratorVertex = graph.vertices();
 399                                 while (iteratorVertex.hasNext()) {
 400
 401                                         try {
 402
 403                                                 vertex = iteratorVertex.next();
 404
 405                                                 // Iterable<Edge> edges2 =
 406                                                 // vertex.getEdges(Direction.BOTH);
 407
 408                                                 // Iterator<Edge> iterator2 = edges2.iterator();
 409                                                 Iterator<Edge> iterator2 = vertex.edges(Direction.BOTH);
 410                                                 if (false == iterator2.hasNext()) {
 411
 412                                                         BaseConfiguration conf = new BaseConfiguration();
 413                                                         conf.setProperty("storage.backend", "inmemory");
 414                                                         TitanGraph openGraph = Utils.openGraph(conf);
 415
 416                                                         TitanVertex addVertexFrom = openGraph.addVertex();
 417                                                         Utils.setProperties(addVertexFrom, Utils.getProperties(vertex));
 418
 419                                                         log.info("fromVertex={}", Utils.getProperties(addVertexFrom));
 420
 421                                                         GraphSONWriter create = GraphSONWriter.build().create();
 422                                                         create.writeGraph(out, openGraph);
 423
 424                                                         openGraph.tx().rollback();
 425
 426                                                 }
 427
 428                                         } catch (Exception e) {
 429                                                 log.info("run Vertex Scan failed - {}" , e);
 430
 431                                                 Object property1 = vertex.value(GraphPropertiesDictionary.HEALTH_CHECK.getProperty());
 432                                                 System.out.println(property1);
 433
 434                                                 Object property2 = vertex.value("healthcheck");
 435                                                 System.out.println(property2);
 436
 437                                                 break;
 438
 439                                         }
 440                                 }
 441
 442                                 // graph.rollback();
 443                                 graph.tx().rollback();
 444
 445                         }
 446
 447                         // Iterable<Vertex> vertices2 =
 448                         // graph.getVertices(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(),
 449                         // "GOOD");
 450                         Iterable<TitanVertex> vertices2 = graph.query()
 451                                         .has(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(), "GOOD").vertices();
 452                         ;
 453                         Vertex next = vertices2.iterator().next();
 454
 455                         BaseConfiguration conf = new BaseConfiguration();
 456                         conf.setProperty("storage.backend", "inmemory");
 457                         TitanGraph openGraph = Utils.openGraph(conf);
 458
 459                         // TitanVertex addVertexFrom = openGraph.addVertex();
 460                         //
 461                         // addVertexFrom.setProperty(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(),
 462                         // "GOOD");
 463                         // addVertexFrom.setProperty("healthcheck",
 464                         // next.getProperty("healthcheck"));
 465                         //
 466                         // //next.remove();
 467                         //
 468                         // next.removeProperty("healthcheck");
 469                         // next.removeProperty("healthcheckis");
 470                         //
 471                         // next.remove();
 472
 473                         // GraphSONWriter.outputGraph(openGraph, outputFile);
 474
 475                         for (NodeTypeEnum nodeTypeEnum : NodeTypeEnum.values()) {
 476                                 removeNodesByLabel(graph, nodeTypeEnum.getName());
 477                         }
 478
 479                         // GraphSONWriter.outputGraph(graph, outputFile);
 480
 481                         GraphSONWriter create = GraphSONWriter.build().create();
 482                         create.writeGraph(out, graph);
 483
 484                         // graph.rollback();
 485                         graph.tx().rollback();
 486
 487                 } catch (Exception e) {
 488                         log.info("find Error In Json Graph failed - {}" , e);
 489                         // graph.rollback();
 490                         graph.tx().rollback();
 491                 } finally {
 492                         try {
 493                                 if (out != null) {
 494                                         out.close();
 495                                 }
 496                         } catch (IOException e) {
 497                                 log.info("close FileOutputStream failed - {}" , e);
 498                         }
 499                 }
 500                 return result;
 501
 502         }
 503
 504         private void removeNodesByLabel(TitanGraph graph, String label) {
 505                 Iterable<TitanVertex> vertices = graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(), label)
 506                                 .vertices();
 507                 Iterator<TitanVertex> iterator = vertices.iterator();
 508                 while (iterator.hasNext()) {
 509                         Vertex next2 = iterator.next();
 510                         next2.remove();
 511                 }
 512         }
 513
 514         public static void clearGraph(TitanGraph graph) {
 515
 516                 Iterable<TitanVertex> vertices = graph.query().vertices();
 517
 518                 long erased = 0;
 519
 520                 if (vertices != null) {
 521                         Iterator<TitanVertex> iterator = vertices.iterator();
 522                         while (iterator.hasNext()) {
 523                                 Vertex vertex = iterator.next();
 524                                 // graph.removeVertex(vertex);
 525                                 vertex.remove();
 526                                 erased++;
 527                         }
 528
 529                 }
 530
 531                 System.out.println("After erasing " + erased + " vertices.");
 532                 // graph.commit();
 533                 graph.tx().commit();
 534         }
 535
 536         public String exportUsers(TitanGraph graph, String outputDirectory) {
 537
 538                 List<Map<String, Object>> users = new ArrayList<>();
 539                 String result = null;
 540
 541                 // GraphMLWriter graphMLWriter = new GraphMLWriter(graph);
 542
 543                 String outputFile = outputDirectory + File.separator + "users." + System.currentTimeMillis() + ".json";
 544
 545                 FileWriter fileWriter = null;
 546                 try {
 547
 548                         TitanGraphQuery graphQuery = graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(),
 549                                         NodeTypeEnum.User.getName());
 550
 551                         @SuppressWarnings("unchecked")
 552                         Iterable<TitanVertex> vertices = graphQuery.vertices();
 553
 554                         if (vertices != null) {
 555                                 for (Vertex v : vertices) {
 556                                         Map<String, Object> properties = getProperties(v);
 557                                         properties.remove(GraphPropertiesDictionary.LABEL.getProperty());
 558                                         users.add(properties);
 559                                 }
 560                         }
 561
 562                         graph.tx().commit();
 563
 564                         String jsonUsers = gson.toJson(users);
 565
 566                         fileWriter = new FileWriter(outputFile);
 567                         fileWriter.write(jsonUsers);
 568
 569                         result = outputFile;
 570
 571                 } catch (Exception e) {
 572                         log.info("export Users failed - {}" , e);
 573                         graph.tx().rollback();
 574                 } finally {
 575                         try {
 576                                 if (fileWriter != null) {
 577                                         fileWriter.close();
 578                                 }
 579                         } catch (IOException e) {
 580                                 log.info("close FileOutputStream failed - {}" , e);
 581                         }
 582                 }
 583                 return result;
 584
 585         }
 586
 587         public Map<String, Object> getProperties(Element element) {
 588
 589                 Map<String, Object> result = new HashMap<String, Object>();
 590                 ;
 591
 592                 if (element.keys() != null && element.keys().size() > 0) {
 593                         Map<String, Property> propertyMap = ElementHelper.propertyMap(element,
 594                                         element.keys().toArray(new String[element.keys().size()]));
 595
 596                         for (Entry<String, Property> entry : propertyMap.entrySet()) {
 597                                 String key = entry.getKey();
 598                                 Object value = entry.getValue().value();
 599
 600                                 result.put(key, value);
 601                         }
 602                 }
 603                 return result;
 604         }
 605
 606         public boolean exportUsers(String[] args) {
 607
 608                 TitanGraph graph = null;
 609                 try {
 610                         String titanFileLocation = args[1];
 611                         String outputDirectory = args[2];
 612                         graph = openGraph(titanFileLocation);
 613
 614                         String result = exportUsers(graph, outputDirectory);
 615
 616                         if (result == null) {
 617                                 return false;
 618                         }
 619
 620                         System.out.println("Exported file=" + result);
 621                 } catch (Exception e) {
 622                         log.info("export Users failed - {}" , e);
 623                         return false;
 624                 } finally {
 625                         if (graph != null) {
 626                                 // graph.shutdown();
 627                                 graph.close();
 628                         }
 629                 }
 630
 631                 return true;
 632         }
 633 }