asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/GraphMLConverter.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23 import com.google.gson.Gson;
  24 import org.janusgraph.core.JanusGraphEdge;
  25 import org.janusgraph.core.JanusGraphFactory;
  26 import org.janusgraph.core.JanusGraph;
  27 import org.janusgraph.core.JanusGraphQuery;
  28 import org.janusgraph.core.JanusGraphVertex;
  29 import org.apache.commons.configuration.BaseConfiguration;
  30 import org.apache.commons.lang3.tuple.ImmutablePair;
  31 import org.apache.tinkerpop.gremlin.structure.*;
  32 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
  33 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
  34 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
  35 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
  36 import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
  37 import org.openecomp.sdc.asdctool.Utils;
  38 import org.openecomp.sdc.be.dao.neo4j.GraphPropertiesDictionary;
  39 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
  40 import org.openecomp.sdc.common.log.wrappers.Logger;
  41
  42 import java.io.BufferedInputStream;
  43 import java.io.BufferedOutputStream;
  44 import java.io.File;
  45 import java.io.FileInputStream;
  46 import java.io.FileOutputStream;
  47 import java.io.FileWriter;
  48 import java.io.IOException;
  49 import java.io.InputStream;
  50 import java.io.OutputStream;
  51 import java.util.ArrayList;
  52 import java.util.HashMap;
  53 import java.util.Iterator;
  54 import java.util.List;
  55 import java.util.Map;
  56 import java.util.Map.Entry;
  57
  58 public class GraphMLConverter {
  59
  60         private static final String FROM_VERTEX = "fromVertex={}";
  61
  62     private static final String STORAGE_BACKEND = "storage.backend";
  63
  64     private static final String INMEMORY = "inmemory";
  65
  66     private static final String CLOSE_FILE_OUTPUT_STREAM_FAILED = "close FileOutputStream failed - {}";
  67
  68     private static final String EXPORT_GRAPH = "exportGraph.";
  69
  70     private static final String DOT_JSON = ".json";
  71
  72     private static final String EXPORTED_FILE = "Exported file=";
  73
  74     private static final String NODE_LABEL = "nodeLabel";
  75
  76     private static Logger log = Logger.getLogger(GraphMLConverter.class.getName());
  77
  78         private Gson gson = new Gson();
  79
  80         public boolean importGraph(String[] args) {
  81
  82                 JanusGraph graph = null;
  83                 try {
  84                         String janusGraphFileLocation = args[1];
  85                         String inputFile = args[2];
  86                         graph = openGraph(janusGraphFileLocation);
  87
  88                         List<ImmutablePair<String, String>> propertiesCriteriaToDelete = new ArrayList<>();
  89                         ImmutablePair<String, String> immutablePair1 = new ImmutablePair<>("healthcheckis", "GOOD");
  90                         ImmutablePair<String, String> immutablePair2 = new ImmutablePair<>(NODE_LABEL, "user");
  91                         ImmutablePair<String, String> immutablePair3 = new ImmutablePair<>(NODE_LABEL,
  92                                         "resourceCategory");
  93                         ImmutablePair<String, String> immutablePair4 = new ImmutablePair<>(NODE_LABEL,
  94                                         "serviceCategory");
  95
  96                         propertiesCriteriaToDelete.add(immutablePair1);
  97                         propertiesCriteriaToDelete.add(immutablePair2);
  98                         propertiesCriteriaToDelete.add(immutablePair3);
  99                         propertiesCriteriaToDelete.add(immutablePair4);
 100
 101                         return importJsonGraph(graph, inputFile, propertiesCriteriaToDelete);
 102
 103                 } catch (Exception e) {
 104                         log.info("import graph failed - {} " , e);
 105                         return false;
 106                 } finally {
 107                         if (graph != null) {
 108                                 graph.close();
 109                         }
 110                 }
 111
 112         }
 113
 114         public boolean exportGraph(String[] args) {
 115
 116                 JanusGraph graph = null;
 117                 try {
 118                         String janusGraphFileLocation = args[1];
 119                         String outputDirectory = args[2];
 120                         graph = openGraph(janusGraphFileLocation);
 121
 122                         String result = exportJsonGraph(graph, outputDirectory);
 123
 124                         if (result == null) {
 125                                 return false;
 126                         }
 127
 128                         System.out.println(EXPORTED_FILE + result);
 129                 } catch (Exception e) {
 130                         log.info("export graph failed -{}" , e);
 131                         return false;
 132                 } finally {
 133                         if (graph != null) {
 134                                 graph.close();
 135                         }
 136                 }
 137
 138                 return true;
 139         }
 140
 141         public String exportGraphMl(String[] args) {
 142
 143                 JanusGraph graph = null;
 144                 String result = null;
 145                 try {
 146                         String janusGraphFileLocation = args[1];
 147                         String outputDirectory = args[2];
 148                         graph = openGraph(janusGraphFileLocation);
 149
 150                         result = exportGraphMl(graph, outputDirectory);
 151
 152                         System.out.println(EXPORTED_FILE + result);
 153                 } catch (Exception e) {
 154                         log.info("export exportGraphMl failed - {}" , e);
 155                         return null;
 156                 } finally {
 157                         if (graph != null) {
 158                                 graph.close();
 159                         }
 160                 }
 161
 162                 return result;
 163         }
 164
 165         public boolean findErrorInJsonGraph(String[] args) {
 166
 167                 JanusGraph graph = null;
 168                 try {
 169                         String janusGraphFileLocation = args[1];
 170                         String outputDirectory = args[2];
 171                         graph = openGraph(janusGraphFileLocation);
 172
 173                         String result = findErrorInJsonGraph(graph, outputDirectory);
 174
 175                         if (result == null) {
 176                                 return false;
 177                         }
 178
 179                         System.out.println(EXPORTED_FILE + result);
 180                 } catch (Exception e) {
 181                         log.info("find Error In Json Graph failed - {}" , e);
 182                         return false;
 183                 } finally {
 184                         if (graph != null) {
 185                                 graph.close();
 186                         }
 187                 }
 188
 189                 return true;
 190         }
 191
 192         public JanusGraph openGraph(String janusGraphFileLocation) {
 193
 194                 return JanusGraphFactory.open(janusGraphFileLocation);
 195
 196         }
 197
 198         public String exportJsonGraph(JanusGraph graph, String outputDirectory) {
 199
 200                 String result = null;
 201
 202                 String outputFile = outputDirectory + File.separator + EXPORT_GRAPH + System.currentTimeMillis() + DOT_JSON;
 203
 204                 OutputStream out = null;
 205                 try {
 206                         out = new BufferedOutputStream(new FileOutputStream(outputFile));
 207
 208                         final GraphSONWriter.Builder builder = GraphSONWriter.build();
 209                         final GraphSONMapper mapper = newGraphSONMapper(graph);
 210                         builder.mapper(mapper);
 211                         final GraphSONWriter writer = builder.create();
 212                         writer.writeGraph(out, graph);
 213
 214                         graph.tx().commit();
 215
 216                         result = outputFile;
 217
 218                 } catch (Exception e) {
 219                         log.info("export Json Graph failed - {}" , e);
 220                         graph.tx().rollback();
 221                 } finally {
 222                         try {
 223                                 if (out != null) {
 224                                         out.close();
 225                                 }
 226                         } catch (IOException e) {
 227                                 log.info(CLOSE_FILE_OUTPUT_STREAM_FAILED , e);
 228                         }
 229                 }
 230                 return result;
 231
 232         }
 233
 234         public String exportGraphMl(JanusGraph graph, String outputDirectory) {
 235                 String result = null;
 236                 String outputFile = outputDirectory + File.separator + EXPORT_GRAPH + System.currentTimeMillis() + ".graphml";
 237                 try {
 238                         try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(outputFile))) {
 239                                 graph.io(IoCore.graphml()).writer().normalize(true).create().writeGraph(os, graph);
 240                         }
 241                         result = outputFile;
 242                         graph.tx().commit();
 243                 } catch (Exception e) {
 244                         graph.tx().rollback();
 245                         log.info("export Graph Ml failed - {}" , e);
 246                 }
 247                 return result;
 248
 249         }
 250
 251         private static GraphSONMapper newGraphSONMapper(final Graph graph) {
 252                 final GraphSONMapper.Builder builder = graph.io(IoCore.graphson()).mapper();
 253                 return builder.create();
 254         }
 255
 256         public boolean importJsonGraph(JanusGraph graph, String graphJsonFile,
 257                         List<ImmutablePair<String, String>> propertiesCriteriaToDelete) {
 258
 259                 boolean result = false;
 260
 261                 InputStream is = null;
 262
 263                 try {
 264
 265                         if (propertiesCriteriaToDelete != null) {
 266                                 for (Entry<String, String> entry : propertiesCriteriaToDelete
 267
 268                                 ) {
 269
 270                                         String key = entry.getKey();
 271                                         String value = entry.getValue();
 272                                         Iterator iterator = graph.query().has(key, value).vertices().iterator();
 273                                         while (iterator.hasNext()) {
 274                                                 Vertex vertex = (Vertex) iterator.next();
 275                                                 vertex.remove();
 276                                                 System.out.println("Remove vertex of type " + key + " and value " + value);
 277                                         }
 278
 279                                 }
 280                         }
 281                         File file = new File(graphJsonFile);
 282                         if (!file.isFile()) {
 283                                 System.out.println("File " + graphJsonFile + " cannot be found.");
 284                                 return result;
 285                         }
 286
 287                         is = new BufferedInputStream(new FileInputStream(graphJsonFile));
 288                         System.out.println("Before importing file " + graphJsonFile);
 289
 290                         GraphSONReader create = GraphSONReader.build().create();
 291                         create.readGraph(is, graph);
 292
 293                         graph.tx().commit();
 294
 295                         result = true;
 296
 297                 } catch (Exception e) {
 298                         System.out.println("Failed to import graph " + e.getMessage());
 299                         log.info("Failed to import graph - {}" , e);
 300                         graph.tx().rollback();
 301                 } finally {
 302                         try {
 303                                 if (is != null) {
 304                                         is.close();
 305                                 }
 306                         } catch (IOException e) {
 307                                 log.info(CLOSE_FILE_OUTPUT_STREAM_FAILED , e);
 308                         }
 309                 }
 310
 311                 return result;
 312
 313         }
 314
 315         public String findErrorInJsonGraph(JanusGraph graph, String outputDirectory) {
 316
 317                 boolean runVertexScan = false;
 318                 boolean runEdgeScan = false;
 319
 320                 String result = null;
 321
 322                 String outputFile = outputDirectory + File.separator + EXPORT_GRAPH + System.currentTimeMillis() + DOT_JSON;
 323
 324                 OutputStream out = null;
 325                 try {
 326                         out = new BufferedOutputStream(new FileOutputStream(outputFile));
 327
 328                         if (runEdgeScan) {
 329
 330                                 Vertex vertexFrom = null;
 331                                 Vertex vertexTo = null;
 332                                 Edge edge = null;
 333
 334                                 Iterable<JanusGraphEdge> edges = graph.query().edges();
 335                                 Iterator<JanusGraphEdge> iterator = edges.iterator();
 336                                 while (iterator.hasNext()) {
 337
 338                                         try {
 339
 340                                                 edge = iterator.next();
 341
 342                                                 vertexFrom = edge.outVertex();
 343                                                 vertexTo = edge.inVertex();
 344
 345                                                 BaseConfiguration conf = new BaseConfiguration();
 346                                                 conf.setProperty(STORAGE_BACKEND, INMEMORY);
 347                                                 JanusGraph openGraph = Utils.openGraph(conf);
 348
 349                                                 JanusGraphVertex addVertexFrom = openGraph.addVertex();
 350                                                 Utils.setProperties(addVertexFrom, Utils.getProperties(vertexFrom));
 351
 352                                                 JanusGraphVertex addVertexTo = openGraph.addVertex();
 353                                                 Utils.setProperties(addVertexTo, Utils.getProperties(vertexTo));
 354
 355                                                 Edge addEdge = addVertexFrom.addEdge(edge.label(), addVertexTo);
 356                                                 Utils.setProperties(addEdge, Utils.getProperties(edge));
 357
 358                                                 log.info(FROM_VERTEX, Utils.getProperties(vertexFrom));
 359                                                 log.info("toVertex={}", Utils.getProperties(vertexTo));
 360                                                 log.info("edge={} {} ",edge.label(),Utils.getProperties(edge));
 361
 362                                                 GraphSONWriter create = GraphSONWriter.build().create();
 363                                                 create.writeGraph(out, openGraph);
 364
 365                                                 openGraph.tx().rollback();
 366
 367                                         } catch (Exception e) {
 368                                                 log.info("run Edge Scan failed - {}" , e);
 369
 370                                                 log.error(FROM_VERTEX, Utils.getProperties(vertexFrom));
 371                                                 log.error("toVertex={}", Utils.getProperties(vertexTo));
 372                                                 log.error("edge={} {} ",edge.label(),Utils.getProperties(edge));
 373
 374                                                 break;
 375
 376                                         }
 377                                 }
 378
 379                                 graph.tx().rollback();
 380
 381                         }
 382
 383                         if (runVertexScan) {
 384
 385                                 Vertex vertex = null;
 386                                 Iterator<Vertex> iteratorVertex = graph.vertices();
 387                                 while (iteratorVertex.hasNext()) {
 388
 389                                         try {
 390
 391                                                 vertex = iteratorVertex.next();
 392                                                 Iterator<Edge> iterator2 = vertex.edges(Direction.BOTH);
 393                                                 if (!iterator2.hasNext()) {
 394
 395                                                         BaseConfiguration conf = new BaseConfiguration();
 396                                                         conf.setProperty(STORAGE_BACKEND, INMEMORY);
 397                                                         JanusGraph openGraph = Utils.openGraph(conf);
 398
 399                                                         JanusGraphVertex addVertexFrom = openGraph.addVertex();
 400                                                         Utils.setProperties(addVertexFrom, Utils.getProperties(vertex));
 401
 402                                                         log.info(FROM_VERTEX, Utils.getProperties(addVertexFrom));
 403
 404                                                         GraphSONWriter create = GraphSONWriter.build().create();
 405                                                         create.writeGraph(out, openGraph);
 406
 407                                                         openGraph.tx().rollback();
 408
 409                                                 }
 410
 411                                         } catch (Exception e) {
 412                                                 log.info("run Vertex Scan failed - {}" , e);
 413
 414                                                 Object property1 = vertex.value(GraphPropertiesDictionary.HEALTH_CHECK.getProperty());
 415                                                 System.out.println(property1);
 416
 417                                                 Object property2 = vertex.value("healthcheck");
 418                                                 System.out.println(property2);
 419
 420                                                 break;
 421
 422                                         }
 423                                 }
 424
 425                                 graph.tx().rollback();
 426
 427                         }
 428
 429                         Iterable<JanusGraphVertex> vertices2 = graph.query()
 430                                         .has(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(), "GOOD").vertices();
 431                         ;
 432
 433                         BaseConfiguration conf = new BaseConfiguration();
 434                         conf.setProperty(STORAGE_BACKEND, INMEMORY);
 435                         for (NodeTypeEnum nodeTypeEnum : NodeTypeEnum.values()) {
 436                                 removeNodesByLabel(graph, nodeTypeEnum.getName());
 437                         }
 438
 439
 440                         GraphSONWriter create = GraphSONWriter.build().create();
 441                         create.writeGraph(out, graph);
 442
 443                         graph.tx().rollback();
 444
 445                 } catch (Exception e) {
 446                         log.info("find Error In Json Graph failed - {}" , e);
 447                         graph.tx().rollback();
 448                 } finally {
 449                         try {
 450                                 if (out != null) {
 451                                         out.close();
 452                                 }
 453                         } catch (IOException e) {
 454                                 log.info(CLOSE_FILE_OUTPUT_STREAM_FAILED , e);
 455                         }
 456                 }
 457                 return result;
 458
 459         }
 460
 461         private void removeNodesByLabel(JanusGraph graph, String label) {
 462                 Iterable<JanusGraphVertex> vertices = graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(), label)
 463                                 .vertices();
 464                 Iterator<JanusGraphVertex> iterator = vertices.iterator();
 465                 while (iterator.hasNext()) {
 466                         Vertex next2 = iterator.next();
 467                         next2.remove();
 468                 }
 469         }
 470
 471         public String exportUsers(JanusGraph graph, String outputDirectory) {
 472
 473                 List<Map<String, Object>> users = new ArrayList<>();
 474                 String result = null;
 475
 476                 String outputFile = outputDirectory + File.separator + "users." + System.currentTimeMillis() + DOT_JSON;
 477
 478                 FileWriter fileWriter = null;
 479                 try {
 480
 481                         JanusGraphQuery graphQuery = graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(),
 482                                         NodeTypeEnum.User.getName());
 483
 484                         @SuppressWarnings("unchecked")
 485                         Iterable<JanusGraphVertex> vertices = graphQuery.vertices();
 486
 487                         if (vertices != null) {
 488                                 for (Vertex v : vertices) {
 489                                         Map<String, Object> properties = getProperties(v);
 490                                         properties.remove(GraphPropertiesDictionary.LABEL.getProperty());
 491                                         users.add(properties);
 492                                 }
 493                         }
 494
 495                         graph.tx().commit();
 496
 497                         String jsonUsers = gson.toJson(users);
 498
 499                         fileWriter = new FileWriter(outputFile);
 500                         fileWriter.write(jsonUsers);
 501
 502                         result = outputFile;
 503
 504                 } catch (Exception e) {
 505                         log.info("export Users failed - {}" , e);
 506                         graph.tx().rollback();
 507                 } finally {
 508                         try {
 509                                 if (fileWriter != null) {
 510                                         fileWriter.close();
 511                                 }
 512                         } catch (IOException e) {
 513                                 log.info(CLOSE_FILE_OUTPUT_STREAM_FAILED , e);
 514                         }
 515                 }
 516                 return result;
 517
 518         }
 519
 520         public Map<String, Object> getProperties(Element element) {
 521
 522                 Map<String, Object> result = new HashMap<>();
 523                 ;
 524
 525                 if (element.keys() != null && !element.keys().isEmpty()) {
 526                         Map<String, Property> propertyMap = ElementHelper.propertyMap(element,
 527                                         element.keys().toArray(new String[element.keys().size()]));
 528
 529                         for (Entry<String, Property> entry : propertyMap.entrySet()) {
 530                                 String key = entry.getKey();
 531                                 Object value = entry.getValue().value();
 532
 533                                 result.put(key, value);
 534                         }
 535                 }
 536                 return result;
 537         }
 538
 539         public boolean exportUsers(String[] args) {
 540
 541                 JanusGraph graph = null;
 542                 try {
 543                         String janusGraphFileLocation = args[1];
 544                         String outputDirectory = args[2];
 545                         graph = openGraph(janusGraphFileLocation);
 546
 547                         String result = exportUsers(graph, outputDirectory);
 548
 549                         if (result == null) {
 550                                 return false;
 551                         }
 552
 553                         System.out.println(EXPORTED_FILE + result);
 554                 } catch (Exception e) {
 555                         log.info("export Users failed - {}" , e);
 556                         return false;
 557                 } finally {
 558                         if (graph != null) {
 559                                 graph.close();
 560                         }
 561                 }
 562
 563                 return true;
 564         }
 565 }