asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/GraphMLConverter.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23 import com.google.gson.Gson;
  24 import org.apache.commons.configuration.BaseConfiguration;
  25 import org.apache.commons.lang3.tuple.ImmutablePair;
  26 import org.apache.tinkerpop.gremlin.structure.Element;
  27 import org.apache.tinkerpop.gremlin.structure.Graph;
  28 import org.apache.tinkerpop.gremlin.structure.Property;
  29 import org.apache.tinkerpop.gremlin.structure.Vertex;
  30 import org.apache.tinkerpop.gremlin.structure.io.IoCore;
  31 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
  32 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
  33 import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
  34 import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
  35 import org.janusgraph.core.JanusGraph;
  36 import org.janusgraph.core.JanusGraphFactory;
  37 import org.janusgraph.core.JanusGraphQuery;
  38 import org.janusgraph.core.JanusGraphVertex;
  39 import org.openecomp.sdc.be.dao.neo4j.GraphPropertiesDictionary;
  40 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
  41 import org.openecomp.sdc.common.log.wrappers.Logger;
  42
  43 import java.io.BufferedInputStream;
  44 import java.io.BufferedOutputStream;
  45 import java.io.File;
  46 import java.io.FileInputStream;
  47 import java.io.FileOutputStream;
  48 import java.io.FileWriter;
  49 import java.io.InputStream;
  50 import java.io.OutputStream;
  51 import java.util.ArrayList;
  52 import java.util.HashMap;
  53 import java.util.Iterator;
  54 import java.util.List;
  55 import java.util.Map;
  56 import java.util.Map.Entry;
  57
  58 public class GraphMLConverter {
  59
  60     private static final String STORAGE_BACKEND = "storage.backend";
  61
  62     private static final String INMEMORY = "inmemory";
  63
  64     private static final String EXPORT_GRAPH = "exportGraph.";
  65
  66     private static final String DOT_JSON = ".json";
  67
  68     private static final String EXPORTED_FILE = "Exported file=";
  69
  70     private static final String NODE_LABEL = "nodeLabel";
  71
  72     private static Logger log = Logger.getLogger(GraphMLConverter.class.getName());
  73
  74     private Gson gson = new Gson();
  75     private static final String LOG_FORMATTER = "{} {}";
  76
  77     public boolean importGraph(String[] args) {
  78
  79         JanusGraph graph = null;
  80         try {
  81             String janusGraphFileLocation = args[1];
  82             String inputFile = args[2];
  83             graph = openGraph(janusGraphFileLocation);
  84
  85             List<ImmutablePair<String, String>> propertiesCriteriaToDelete = new ArrayList<>();
  86             ImmutablePair<String, String> immutablePair1 = new ImmutablePair<>("healthcheckis", "GOOD");
  87             ImmutablePair<String, String> immutablePair2 = new ImmutablePair<>(NODE_LABEL, "user");
  88             ImmutablePair<String, String> immutablePair3 = new ImmutablePair<>(NODE_LABEL, "resourceCategory");
  89             ImmutablePair<String, String> immutablePair4 = new ImmutablePair<>(NODE_LABEL, "serviceCategory");
  90
  91             propertiesCriteriaToDelete.add(immutablePair1);
  92             propertiesCriteriaToDelete.add(immutablePair2);
  93             propertiesCriteriaToDelete.add(immutablePair3);
  94             propertiesCriteriaToDelete.add(immutablePair4);
  95
  96             return importJsonGraph(graph, inputFile, propertiesCriteriaToDelete);
  97
  98         } catch (Exception e) {
  99                         e.printStackTrace();
 100             log.info("import graph failed ", e);
 101             return false;
 102         } finally {
 103             if (graph != null) {
 104                 graph.close();
 105             }
 106         }
 107
 108     }
 109
 110     public boolean exportGraph(String[] args) {
 111
 112         JanusGraph graph = null;
 113         try {
 114             String janusGraphFileLocation = args[1];
 115             String outputDirectory = args[2];
 116             graph = openGraph(janusGraphFileLocation);
 117
 118             String result = exportJsonGraph(graph, outputDirectory);
 119
 120             if (result == null) {
 121                 return false;
 122             }
 123
 124             log.info(LOG_FORMATTER, EXPORTED_FILE, result);
 125         } catch (Exception e) {
 126                         e.printStackTrace();
 127             log.info("export graph failed ", e);
 128             return false;
 129         } finally {
 130             if (graph != null) {
 131                 graph.close();
 132             }
 133         }
 134
 135         return true;
 136     }
 137
 138     public String exportGraphMl(String[] args) {
 139
 140         JanusGraph graph = null;
 141         String result = null;
 142         try {
 143             String janusGraphFileLocation = args[1];
 144             String outputDirectory = args[2];
 145             graph = openGraph(janusGraphFileLocation);
 146
 147             result = exportGraphMl(graph, outputDirectory);
 148
 149             log.info(LOG_FORMATTER, EXPORTED_FILE, result);
 150         } catch (Exception e) {
 151                         e.printStackTrace();
 152             log.info("export exportGraphMl failed ", e);
 153             return null;
 154         } finally {
 155             if (graph != null) {
 156                 graph.close();
 157             }
 158         }
 159
 160         return result;
 161     }
 162
 163     public boolean findErrorInJsonGraph(String[] args) {
 164
 165         JanusGraph graph = null;
 166         try {
 167             String janusGraphFileLocation = args[1];
 168             String outputDirectory = args[2];
 169             graph = openGraph(janusGraphFileLocation);
 170
 171             String result = findErrorInJsonGraph(graph, outputDirectory);
 172
 173             if (result == null) {
 174                 return false;
 175             }
 176
 177             log.info(LOG_FORMATTER, EXPORTED_FILE, result);
 178         } catch (Exception e) {
 179                         e.printStackTrace();
 180             log.info("find Error In Json Graph failed ", e);
 181             return false;
 182         } finally {
 183             if (graph != null) {
 184                 graph.close();
 185             }
 186         }
 187
 188         return true;
 189     }
 190
 191     public JanusGraph openGraph(String janusGraphFileLocation) {
 192
 193         return JanusGraphFactory.open(janusGraphFileLocation);
 194
 195     }
 196
 197     public String exportJsonGraph(JanusGraph graph, String outputDirectory) {
 198
 199         String result = null;
 200
 201         String outputFile = outputDirectory + File.separator + EXPORT_GRAPH + System.currentTimeMillis() + DOT_JSON;
 202
 203         try (final OutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) {
 204
 205             final GraphSONWriter.Builder builder = GraphSONWriter.build();
 206             final GraphSONMapper mapper = newGraphSONMapper(graph);
 207             builder.mapper(mapper);
 208             final GraphSONWriter writer = builder.create();
 209             writer.writeGraph(out, graph);
 210
 211             graph.tx().commit();
 212
 213             result = outputFile;
 214
 215         } catch (Exception e) {
 216                         e.printStackTrace();
 217             log.info("export Json Graph failed ", e);
 218             graph.tx().rollback();
 219                                 e.printStackTrace();
 220         }
 221         return result;
 222
 223     }
 224
 225     public String exportGraphMl(JanusGraph graph, String outputDirectory) {
 226         String result = null;
 227         String outputFile =
 228             outputDirectory + File.separator + EXPORT_GRAPH + System.currentTimeMillis() + ".graphml";
 229         try {
 230             try (final OutputStream os = new BufferedOutputStream(new FileOutputStream(outputFile))) {
 231                 graph.io(IoCore.graphml()).writer().normalize(true).create().writeGraph(os, graph);
 232             }
 233             result = outputFile;
 234             graph.tx().commit();
 235         } catch (Exception e) {
 236             graph.tx().rollback();
 237                         e.printStackTrace();
 238             log.info("export Graph Ml failed ", e);
 239         }
 240         return result;
 241
 242     }
 243
 244     private static GraphSONMapper newGraphSONMapper(final Graph graph) {
 245         final GraphSONMapper.Builder builder = graph.io(IoCore.graphson()).mapper();
 246         return builder.create();
 247     }
 248
 249     public boolean importJsonGraph(JanusGraph graph, String graphJsonFile,
 250                                    List<ImmutablePair<String, String>> propertiesCriteriaToDelete) {
 251
 252         boolean result = false;
 253
 254         if (propertiesCriteriaToDelete != null) {
 255             for (Entry<String, String> entry : propertiesCriteriaToDelete
 256
 257             ) {
 258
 259                 String key = entry.getKey();
 260                 String value = entry.getValue();
 261                 Iterator iterator = graph.query().has(key, value).vertices().iterator();
 262                 while (iterator.hasNext()) {
 263                     Vertex vertex = (Vertex) iterator.next();
 264                     vertex.remove();
 265                     log.info("Remove vertex of type{} ", key, " and value {}", value);
 266                 }
 267
 268             }
 269         }
 270         File file = new File(graphJsonFile);
 271         if (!file.isFile()) {
 272             log.info("File ", graphJsonFile, " cannot be found.");
 273             return result;
 274         }
 275
 276         try (final InputStream is = new BufferedInputStream(new FileInputStream(graphJsonFile))) {
 277
 278             log.info("Before importing file ", graphJsonFile);
 279
 280             GraphSONReader create = GraphSONReader.build().create();
 281             create.readGraph(is, graph);
 282
 283             graph.tx().commit();
 284
 285             result = true;
 286
 287         } catch (Exception e) {
 288             log.info("Failed to import graph ", e);
 289                         e.printStackTrace();
 290             graph.tx().rollback();
 291                                 e.printStackTrace();
 292         }
 293         return result;
 294
 295     }
 296
 297     public String findErrorInJsonGraph(JanusGraph graph, String outputDirectory) {
 298
 299         String result = null;
 300
 301         String outputFile = outputDirectory + File.separator + EXPORT_GRAPH + System.currentTimeMillis() + DOT_JSON;
 302
 303         try (final OutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) {
 304
 305             graph.query().has(GraphPropertiesDictionary.HEALTH_CHECK.getProperty(), "GOOD").vertices();
 306
 307             BaseConfiguration conf = new BaseConfiguration();
 308             conf.setProperty(STORAGE_BACKEND, INMEMORY);
 309             for (NodeTypeEnum nodeTypeEnum : NodeTypeEnum.values()) {
 310                 removeNodesByLabel(graph, nodeTypeEnum.getName());
 311             }
 312
 313             GraphSONWriter create = GraphSONWriter.build().create();
 314             create.writeGraph(out, graph);
 315
 316             graph.tx().rollback();
 317
 318         } catch (Exception e) {
 319                         e.printStackTrace();
 320             log.info("find Error In Json Graph failed ", e);
 321             graph.tx().rollback();
 322                                 e.printStackTrace();
 323         }
 324         return result;
 325
 326     }
 327
 328     private void removeNodesByLabel(JanusGraph graph, String label) {
 329         Iterable<JanusGraphVertex> vertices =
 330             graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(), label).vertices();
 331         Iterator<JanusGraphVertex> iterator = vertices.iterator();
 332         while (iterator.hasNext()) {
 333             Vertex next2 = iterator.next();
 334             next2.remove();
 335         }
 336     }
 337
 338     public String exportUsers(JanusGraph graph, String outputDirectory) {
 339
 340         List<Map<String, Object>> users = new ArrayList<>();
 341         String result = null;
 342
 343         String outputFile = outputDirectory + File.separator + "users." + System.currentTimeMillis() + DOT_JSON;
 344
 345         JanusGraphQuery graphQuery =
 346             graph.query().has(GraphPropertiesDictionary.LABEL.getProperty(), NodeTypeEnum.User.getName());
 347
 348         @SuppressWarnings("unchecked")
 349         Iterable<JanusGraphVertex> vertices = graphQuery.vertices();
 350
 351         if (vertices != null) {
 352             for (Vertex v : vertices) {
 353                 Map<String, Object> properties = getProperties(v);
 354                 properties.remove(GraphPropertiesDictionary.LABEL.getProperty());
 355                 users.add(properties);
 356             }
 357         }
 358
 359         graph.tx().commit();
 360
 361         String jsonUsers = gson.toJson(users);
 362
 363         try (final FileWriter fileWriter = new FileWriter(outputFile)) {
 364
 365             fileWriter.write(jsonUsers);
 366
 367             result = outputFile;
 368
 369         } catch (Exception e) {
 370                         e.printStackTrace();
 371             log.info("export Users failed ", e);
 372             graph.tx().rollback();
 373                                 e.printStackTrace();
 374         }
 375         return result;
 376
 377     }
 378
 379     public Map<String, Object> getProperties(Element element) {
 380
 381         Map<String, Object> result = new HashMap<>();
 382         ;
 383
 384         if (element.keys() != null && !element.keys().isEmpty()) {
 385             Map<String, Property> propertyMap =
 386                 ElementHelper.propertyMap(element, element.keys().toArray(new String[element.keys().size()]));
 387
 388             for (Entry<String, Property> entry : propertyMap.entrySet()) {
 389                 String key = entry.getKey();
 390                 Object value = entry.getValue().value();
 391
 392                 result.put(key, value);
 393             }
 394         }
 395         return result;
 396     }
 397
 398     public boolean exportUsers(String[] args) {
 399
 400         JanusGraph graph = null;
 401         try {
 402             String janusGraphFileLocation = args[1];
 403             String outputDirectory = args[2];
 404             graph = openGraph(janusGraphFileLocation);
 405
 406             String result = exportUsers(graph, outputDirectory);
 407
 408             if (result == null) {
 409                 return false;
 410             }
 411
 412             log.info(EXPORTED_FILE, result);
 413         } catch (Exception e) {
 414                         e.printStackTrace();
 415             log.info("export Users failed ", e);
 416             return false;
 417         } finally {
 418             if (graph != null) {
 419                 graph.close();
 420             }
 421         }
 422
 423         return true;
 424     }
 425 }