aai-core/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.onap.aai.dbgen;
  21
  22 import java.io.BufferedWriter;
  23 import java.io.File;
  24 import java.io.FileInputStream;
  25 import java.io.FileWriter;
  26 import java.io.IOException;
  27 import java.io.InputStream;
  28 import java.io.UnsupportedEncodingException;
  29 import java.net.URI;
  30 import java.nio.file.Files;
  31 import java.nio.file.Path;
  32 import java.nio.file.Paths;
  33 import java.util.ArrayList;
  34 import java.util.HashMap;
  35 import java.util.HashSet;
  36 import java.util.Iterator;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Map.Entry;
  40 import java.util.Scanner;
  41 import java.util.Set;
  42 import java.util.UUID;
  43
  44 import org.apache.commons.lang.exception.ExceptionUtils;
  45 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
  46 import org.apache.tinkerpop.gremlin.structure.Element;
  47 import org.apache.tinkerpop.gremlin.structure.Vertex;
  48 import org.onap.aai.db.props.AAIProperties;
  49 import org.onap.aai.dbmap.DBConnectionType;
  50 import org.onap.aai.dbmap.InMemoryGraph;
  51 import org.onap.aai.exceptions.AAIException;
  52 import org.onap.aai.introspection.Introspector;
  53 import org.onap.aai.introspection.Loader;
  54 import org.onap.aai.introspection.LoaderFactory;
  55 import org.onap.aai.introspection.ModelType;
  56 import org.onap.aai.introspection.Version;
  57 import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
  58 import org.onap.aai.logging.LogFormatTools;
  59 import org.onap.aai.parsers.uri.URIToObject;
  60 import org.onap.aai.serialization.engines.InMemoryDBEngine;
  61 import org.onap.aai.serialization.engines.QueryStyle;
  62 import org.onap.aai.serialization.db.DBSerializer;
  63
  64 import com.att.eelf.configuration.EELFLogger;
  65 import com.att.eelf.configuration.EELFManager;
  66
  67 import org.slf4j.MDC;
  68
  69 import org.codehaus.jackson.JsonNode;
  70 import org.codehaus.jackson.map.ObjectMapper;
  71 import org.codehaus.jackson.node.ObjectNode;
  72 import org.codehaus.jackson.type.TypeReference;
  73
  74 import java.util.Date;
  75 import java.text.DateFormat;
  76 import java.text.SimpleDateFormat;
  77 import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
  78 import org.onap.aai.util.AAIConfig;
  79 import org.onap.aai.util.AAIConstants;
  80
  81 import com.beust.jcommander.JCommander;
  82 import com.beust.jcommander.Parameter;
  83
  84 /**
  85  * The Class ListEndpoints.
  86  */
  87 public class DynamicPayloadGenerator {
  88
  89         /*
  90          * Create a Dynamic memory graph instance which should not affect the
  91          * AAIGraph
  92          */
  93         private InMemoryGraph inMemGraph = null;
  94         private InMemoryDBEngine dbEngine;
  95
  96         /*
  97          * Loader, QueryStyle, ConnectionType for the Serializer
  98          */
  99         private Loader loader;
 100         private String urlBase;
 101         private BufferedWriter bw = null;
 102
 103         private CommandLineArgs cArgs;
 104
 105         private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
 106
 107         private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
 108         private static final DBConnectionType type = DBConnectionType.CACHED;
 109         private static final ModelType introspectorFactoryType = ModelType.MOXY;
 110
 111         /*
 112          * Version
 113          */
 114         private static final Version version = Version.getLatest();
 115
 116         /**
 117          * The main method.
 118          *
 119          * @param args
 120          *            the arguments
 121          * @throws AAIException
 122          * @throws Exception
 123          */
 124         public static void main(String[] args) {
 125
 126                 MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
 127                 DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator();
 128                 try {
 129                         payloadgen.init(args);
 130
 131                         payloadgen.generatePayloads();
 132                 } catch (AAIException e) {
 133                         LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
 134                 } catch (IOException e) {
 135                         LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
 136                 }
 137                 System.exit(0);
 138         }
 139
 140         public void init(String[] args) throws AAIException {
 141                 cArgs = new CommandLineArgs();
 142                 JCommander jCommander = new JCommander(cArgs, args);
 143                 jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
 144                 LOGGER.info("Snapshot file" + cArgs.dataSnapshot);
 145                 //TODO- How to add dynamic.properties
 146
 147
 148                 LOGGER.info("output file" + cArgs.output);
 149
 150
 151                 LOGGER.info("format file" + cArgs.format);
 152                 LOGGER.info("format file" + cArgs.schemaEnabled);
 153                 if(cArgs.config.isEmpty())
 154                         cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
 155
 156                 LOGGER.info("config file" + cArgs.config);
 157                 if(cArgs.nodePropertyFile.isEmpty())
 158                         cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
 159                 LOGGER.info("nodePropertyFile file" + cArgs.nodePropertyFile);
 160                 AAIConfig.init();
 161
 162                 urlBase = AAIConfig.get("aai.server.url.base", "");
 163
 164         }
 165
 166         public void generatePayloads() throws AAIException, IOException{
 167
 168                         List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
 169                         LOGGER.info("Load the Graph");
 170
 171                         this.loadGraph();
 172                         LOGGER.info("Generate payload");
 173                         this.generatePayload(nodeFilters);
 174                         LOGGER.info("Close graph");
 175                         this.closeGraph();
 176
 177         }
 178
 179         private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
 180
 181                 InputStream is = new FileInputStream(inputFile);
 182                 Scanner scanner = new Scanner(is);
 183                 String jsonFile = scanner.useDelimiter("\\Z").next();
 184                 scanner.close();
 185
 186                 List<Map<String, List<String>>> allNodes = new ArrayList<>();
 187                 Map<String, List<String>> filterCousins = new HashMap<>();
 188                 Map<String, List<String>> filterParents = new HashMap<>();
 189
 190                 ObjectMapper mapper = new ObjectMapper();
 191
 192                 JsonNode rootNode = mapper.readTree(jsonFile);
 193
 194                 Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
 195
 196                 while (nodeFields.hasNext()) {
 197                         Entry<String, JsonNode> entry = nodeFields.next();
 198                         String nodeType = entry.getKey();
 199                         JsonNode nodeProperty = entry.getValue();
 200
 201                         JsonNode cousinFilter = nodeProperty.path("cousins");
 202                         JsonNode parentFilter = nodeProperty.path("parents");
 203                         List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
 204                                         new TypeReference<ArrayList<String>>() {
 205                                         });
 206
 207                         List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
 208                                         new TypeReference<ArrayList<String>>() {
 209                                         });
 210                         for (String cousin : cousins) {
 211                                 LOGGER.info("Cousins-Filtered" + cousin);
 212                         }
 213                         for (String parent : parents) {
 214                                 LOGGER.info("Parents-Filtered" + parent);
 215                         }
 216                         filterCousins.put(nodeType, cousins);
 217                         filterParents.put(nodeType, parents);
 218
 219                 }
 220
 221                 allNodes.add(filterCousins);
 222                 allNodes.add(filterParents);
 223                 return allNodes;
 224
 225         }
 226
 227         private void loadGraph() throws IOException {
 228
 229                 loadGraphIntoMemory();
 230                 buildDbEngine();
 231
 232         }
 233
 234         private void loadGraphIntoMemory() throws  IOException {
 235
 236                 inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled);
 237
 238         }
 239
 240         private void buildDbEngine() {
 241                 // TODO : parametrise version
 242                 loader = LoaderFactory.createLoaderForVersion(introspectorFactoryType, version);
 243
 244                 dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
 245                 dbEngine.startTransaction();
 246         }
 247
 248         private void generatePayload(List<Map<String, List<String>>> nodeFilters) throws AAIException, IOException {
 249
 250                 Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
 251                 Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
 252
 253                 Set<String> nodeTypes = filterCousinsMap.keySet();
 254
 255                 for (String nodeType : nodeTypes) {
 256                         if ("DMAAP-MR".equals(cArgs.format)) {
 257                                 bw = createFile(nodeType + ".json");
 258                         }
 259                         List<String> filterCousins = filterCousinsMap.get(nodeType);
 260                         List<String> filterParents = filterParentsMap.get(nodeType);
 261                         readVertices(nodeType, filterCousins, filterParents);
 262                         bw.close();
 263                         LOGGER.info("All Done-" + nodeType);
 264                 }
 265
 266         }
 267
 268         private BufferedWriter createFile(String outfileName) throws IOException {
 269                 // FileLocation
 270                 String fileName = outfileName;
 271
 272                 File outFile = new File(fileName);
 273                 LOGGER.info(" Will write to " + outFile);
 274                 FileWriter fw = new FileWriter(outFile.getAbsoluteFile());
 275                 return new BufferedWriter(fw);
 276
 277         }
 278
 279         private void createDirectory(String dirName) throws IOException {
 280                 // FileLocation
 281                 Path pathDir = Paths.get(dirName);
 282                 Files.createDirectories(pathDir);
 283
 284         }
 285
 286         public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents)
 287                         throws AAIException, IOException {
 288
 289                 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
 290                 List<Vertex> nodes = inMemGraph.getGraph().traversal().V().has("aai-node-type", nodeType).toList();
 291
 292                 LOGGER.info("Number of nodes" + nodes.size());
 293                 String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
 294                 createDirectory( dirName);
 295                 // TODO: Formatter
 296                 if ("DMAAP-MR".equals(cArgs.format)) {
 297                         for (Vertex node : nodes) {
 298
 299                                 Introspector nodeObj = serializer.getLatestVersionView(node);
 300                                 createPayloadForDmaap(node, nodeObj);
 301
 302                         }
 303                 }
 304                 int counter = 0;
 305                 if ("PAYLOAD".equals(cArgs.format)) {
 306                         for (Vertex node : nodes) {
 307
 308                                 counter++;
 309                                 String filename = dirName + counter + "-" + nodeType + ".json";
 310                                 bw = createFile(filename);
 311                                 Introspector obj = loader.introspectorFromName(nodeType);
 312                                 Set<Vertex> seen = new HashSet<>();
 313                                 int depth = AAIProperties.MAXIMUM_DEPTH;
 314                                 boolean nodeOnly = false;
 315
 316                                 Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
 317                                 TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
 318                                 serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins, filterParents);
 319                                 createPayloadForPut(obj);
 320                                 bw.close();
 321
 322                                 URI uri = serializer.getURIForVertex(node);
 323                                 String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
 324                                 bw = createFile(filenameWithUri);
 325                                 bw.write(uri.toString());
 326                                 bw.newLine();
 327                                 bw.close();
 328                         }
 329                 }
 330
 331         }
 332
 333         public void createPayloadForPut(Introspector nodeObj) throws IOException {
 334
 335                 String entityJson = nodeObj.marshal(false);
 336                 ObjectMapper mapper = new ObjectMapper();
 337
 338                 ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
 339                 rootNode.remove("resource-version");
 340
 341                 bw.newLine();
 342                 bw.write(rootNode.toString());
 343                 bw.newLine();
 344         }
 345
 346         public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
 347                         throws AAIException, UnsupportedEncodingException {
 348
 349                 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
 350
 351                 URI uri = serializer.getURIForVertex(node);
 352
 353                 String sourceOfTruth = "";
 354                 HashMap<String, Introspector> relatedVertices = new HashMap<>();
 355                 List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
 356
 357                 for (Vertex vertex : vertexChain) {
 358                         try {
 359
 360                                 Introspector vertexObj = serializer.getVertexProperties(vertex);
 361
 362                                 relatedVertices.put(vertexObj.getObjectId(), vertexObj);
 363                         } catch (AAIUnknownObjectException e) {
 364                                 LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
 365                         }
 366
 367                 }
 368
 369                 String transactionId = "TXID";
 370                 createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
 371
 372         }
 373
 374         public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
 375                         Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
 376
 377                 String action = "CREATE";
 378                 final Introspector notificationEvent = loader.introspectorFromName("notification-event");
 379
 380                 try {
 381                         Introspector eventHeader = loader.introspectorFromName("notification-event-header");
 382                         URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
 383
 384                         String entityLink = urlBase + version + uri;
 385
 386                         notificationEvent.setValue("cambria-partition", "AAI");
 387
 388                         eventHeader.setValue("entity-link", entityLink);
 389                         eventHeader.setValue("action", action);
 390                         eventHeader.setValue("entity-type", obj.getDbName());
 391                         eventHeader.setValue("top-entity-type", parser.getTopEntityName());
 392                         eventHeader.setValue("source-name", sourceOfTruth);
 393                         eventHeader.setValue("version", version.toString());
 394                         eventHeader.setValue("id", transactionId);
 395                         eventHeader.setValue("event-type", "AAI-BASELINE");
 396                         if (eventHeader.getValue("domain") == null) {
 397                                 eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
 398                         }
 399
 400                         if (eventHeader.getValue("sequence-number") == null) {
 401                                 eventHeader.setValue("sequence-number",
 402                                                 AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
 403                         }
 404
 405                         if (eventHeader.getValue("severity") == null) {
 406                                 eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
 407                         }
 408
 409                         if (eventHeader.getValue("id") == null) {
 410                                 eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
 411
 412                         }
 413
 414                         if (eventHeader.getValue("timestamp") == null) {
 415                                 eventHeader.setValue("timestamp", genDate());
 416                         }
 417
 418                         List<Object> parentList = parser.getParentList();
 419                         parentList.clear();
 420
 421                         if (!parser.getTopEntity().equals(parser.getEntity())) {
 422                                 Introspector child;
 423                                 String json = obj.marshal(false);
 424                                 child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
 425                                 parentList.add(child.getUnderlyingObject());
 426                         }
 427
 428                         final Introspector eventObject;
 429
 430                         String json = "";
 431                         if (parser.getTopEntity().equals(parser.getEntity())) {
 432                                 json = obj.marshal(false);
 433                                 eventObject = loader.unmarshal(obj.getName(), json);
 434                         } else {
 435                                 json = parser.getTopEntity().marshal(false);
 436
 437                                 eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
 438                         }
 439                         notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
 440                         notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
 441
 442                         String entityJson = notificationEvent.marshal(false);
 443
 444                         bw.newLine();
 445                         bw.write(entityJson);
 446
 447                 } catch (AAIUnknownObjectException e) {
 448                         LOGGER.error("Fatal error - notification-event-header object not found!");
 449                 } catch (Exception e) {
 450                         LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
 451                 }
 452         }
 453
 454         private void closeGraph() {
 455                 inMemGraph.getGraph().tx().rollback();
 456                 inMemGraph.getGraph().close();
 457         }
 458
 459         public static String genDate() {
 460                 Date date = new Date();
 461                 DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
 462                 return formatter.format(date);
 463         }
 464
 465         public static String genDate2() {
 466                 Date date = new Date();
 467                 DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
 468                 return formatter.format(date);
 469         }
 470
 471 }
 472
 473 class CommandLineArgs {
 474
 475         @Parameter(names = "--help", help = true)
 476         public boolean help;
 477
 478         @Parameter(names = "-d", description = "snapshot file to be loaded")
 479         public String dataSnapshot;
 480
 481         @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
 482         public boolean schemaEnabled = true;
 483
 484         @Parameter(names = "-c", description = "location of configuration file")
 485         public String config = "";
 486
 487         @Parameter(names = "-o", description = "output location")
 488         public String output = "";
 489
 490         @Parameter(names = "-f", description = "format of output")
 491         public String format = "PAYLOAD";
 492
 493         @Parameter(names = "-n", description = "Node input file")
 494         public String nodePropertyFile = "";
 495
 496 }