aai-core/src/main/java/org/onap/aai/dbgen/DynamicPayloadGenerator.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  *
  20  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  */
  22 package org.onap.aai.dbgen;
  23
  24 import java.io.BufferedWriter;
  25 import java.io.File;
  26 import java.io.FileInputStream;
  27 import java.io.FileWriter;
  28 import java.io.IOException;
  29 import java.io.InputStream;
  30 import java.io.UnsupportedEncodingException;
  31 import java.net.URI;
  32 import java.nio.file.Files;
  33 import java.nio.file.Path;
  34 import java.nio.file.Paths;
  35 import java.util.ArrayList;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.Iterator;
  39 import java.util.List;
  40 import java.util.Map;
  41 import java.util.Map.Entry;
  42 import java.util.Scanner;
  43 import java.util.Set;
  44 import java.util.UUID;
  45
  46 import org.apache.commons.lang.exception.ExceptionUtils;
  47 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
  48 import org.apache.tinkerpop.gremlin.structure.Element;
  49 import org.apache.tinkerpop.gremlin.structure.Vertex;
  50 import org.onap.aai.db.props.AAIProperties;
  51 import org.onap.aai.dbmap.DBConnectionType;
  52 import org.onap.aai.dbmap.InMemoryGraph;
  53 import org.onap.aai.exceptions.AAIException;
  54 import org.onap.aai.introspection.Introspector;
  55 import org.onap.aai.introspection.Loader;
  56 import org.onap.aai.introspection.LoaderFactory;
  57 import org.onap.aai.introspection.ModelType;
  58 import org.onap.aai.introspection.Version;
  59 import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
  60 import org.onap.aai.logging.LogFormatTools;
  61 import org.onap.aai.parsers.uri.URIToObject;
  62 import org.onap.aai.serialization.engines.InMemoryDBEngine;
  63 import org.onap.aai.serialization.engines.QueryStyle;
  64 import org.onap.aai.serialization.db.DBSerializer;
  65
  66 import com.att.eelf.configuration.EELFLogger;
  67 import com.att.eelf.configuration.EELFManager;
  68
  69 import org.slf4j.MDC;
  70
  71 import org.codehaus.jackson.JsonNode;
  72 import org.codehaus.jackson.map.ObjectMapper;
  73 import org.codehaus.jackson.node.ObjectNode;
  74 import org.codehaus.jackson.type.TypeReference;
  75
  76 import java.util.Date;
  77 import java.text.DateFormat;
  78 import java.text.SimpleDateFormat;
  79 import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
  80 import org.onap.aai.util.AAIConfig;
  81 import org.onap.aai.util.AAIConstants;
  82
  83 import com.beust.jcommander.JCommander;
  84 import com.beust.jcommander.Parameter;
  85
  86 /**
  87  * The Class ListEndpoints.
  88  */
  89 public class DynamicPayloadGenerator {
  90
  91         /*
  92          * Create a Dynamic memory graph instance which should not affect the
  93          * AAIGraph
  94          */
  95         private InMemoryGraph inMemGraph = null;
  96         private InMemoryDBEngine dbEngine;
  97
  98         /*
  99          * Loader, QueryStyle, ConnectionType for the Serializer
 100          */
 101         private Loader loader;
 102         private String urlBase;
 103         private BufferedWriter bw = null;
 104
 105         private CommandLineArgs cArgs;
 106
 107         private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
 108
 109         private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
 110         private static final DBConnectionType type = DBConnectionType.CACHED;
 111         private static final ModelType introspectorFactoryType = ModelType.MOXY;
 112
 113         /*
 114          * Version
 115          */
 116         private static final Version version = Version.getLatest();
 117
 118         /**
 119          * The main method.
 120          *
 121          * @param args
 122          *            the arguments
 123          * @throws AAIException
 124          * @throws Exception
 125          */
 126         public static void main(String[] args) {
 127
 128                 MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
 129                 DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator();
 130                 try {
 131                         payloadgen.init(args);
 132
 133                         payloadgen.generatePayloads();
 134                 } catch (AAIException e) {
 135                         LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
 136                 } catch (IOException e) {
 137                         LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
 138                 }
 139                 System.exit(0);
 140         }
 141
 142         public void init(String[] args) throws AAIException {
 143                 cArgs = new CommandLineArgs();
 144                 JCommander jCommander = new JCommander(cArgs, args);
 145                 jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
 146                 LOGGER.info("Snapshot file" + cArgs.dataSnapshot);
 147                 //TODO- How to add dynamic.properties
 148
 149
 150                 LOGGER.info("output file" + cArgs.output);
 151
 152
 153                 LOGGER.info("format file" + cArgs.format);
 154                 LOGGER.info("format file" + cArgs.schemaEnabled);
 155                 if(cArgs.config.isEmpty())
 156                         cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
 157
 158                 LOGGER.info("config file" + cArgs.config);
 159                 if(cArgs.nodePropertyFile.isEmpty())
 160                         cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
 161                 LOGGER.info("nodePropertyFile file" + cArgs.nodePropertyFile);
 162                 AAIConfig.init();
 163
 164                 urlBase = AAIConfig.get("aai.server.url.base", "");
 165
 166         }
 167
 168         public void generatePayloads() throws AAIException, IOException{
 169
 170                         List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
 171                         LOGGER.info("Load the Graph");
 172
 173                         this.loadGraph();
 174                         LOGGER.info("Generate payload");
 175                         this.generatePayload(nodeFilters);
 176                         LOGGER.info("Close graph");
 177                         this.closeGraph();
 178
 179         }
 180
 181         private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
 182
 183                 InputStream is = new FileInputStream(inputFile);
 184                 Scanner scanner = new Scanner(is);
 185                 String jsonFile = scanner.useDelimiter("\\Z").next();
 186                 scanner.close();
 187
 188                 List<Map<String, List<String>>> allNodes = new ArrayList<>();
 189                 Map<String, List<String>> filterCousins = new HashMap<>();
 190                 Map<String, List<String>> filterParents = new HashMap<>();
 191
 192                 ObjectMapper mapper = new ObjectMapper();
 193
 194                 JsonNode rootNode = mapper.readTree(jsonFile);
 195
 196                 Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
 197
 198                 while (nodeFields.hasNext()) {
 199                         Entry<String, JsonNode> entry = nodeFields.next();
 200                         String nodeType = entry.getKey();
 201                         JsonNode nodeProperty = entry.getValue();
 202
 203                         JsonNode cousinFilter = nodeProperty.path("cousins");
 204                         JsonNode parentFilter = nodeProperty.path("parents");
 205                         List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
 206                                         new TypeReference<ArrayList<String>>() {
 207                                         });
 208
 209                         List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
 210                                         new TypeReference<ArrayList<String>>() {
 211                                         });
 212                         for (String cousin : cousins) {
 213                                 LOGGER.info("Cousins-Filtered" + cousin);
 214                         }
 215                         for (String parent : parents) {
 216                                 LOGGER.info("Parents-Filtered" + parent);
 217                         }
 218                         filterCousins.put(nodeType, cousins);
 219                         filterParents.put(nodeType, parents);
 220
 221                 }
 222
 223                 allNodes.add(filterCousins);
 224                 allNodes.add(filterParents);
 225                 return allNodes;
 226
 227         }
 228
 229         private void loadGraph() throws IOException {
 230
 231                 loadGraphIntoMemory();
 232                 buildDbEngine();
 233
 234         }
 235
 236         private void loadGraphIntoMemory() throws  IOException {
 237
 238                 inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled);
 239
 240         }
 241
 242         private void buildDbEngine() {
 243                 // TODO : parametrise version
 244                 loader = LoaderFactory.createLoaderForVersion(introspectorFactoryType, version);
 245
 246                 dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
 247                 dbEngine.startTransaction();
 248         }
 249
 250         private void generatePayload(List<Map<String, List<String>>> nodeFilters) throws AAIException, IOException {
 251
 252                 Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
 253                 Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
 254
 255                 Set<String> nodeTypes = filterCousinsMap.keySet();
 256
 257                 for (String nodeType : nodeTypes) {
 258                         if ("DMAAP-MR".equals(cArgs.format)) {
 259                                 bw = createFile(nodeType + ".json");
 260                         }
 261                         List<String> filterCousins = filterCousinsMap.get(nodeType);
 262                         List<String> filterParents = filterParentsMap.get(nodeType);
 263                         readVertices(nodeType, filterCousins, filterParents);
 264                         bw.close();
 265                         LOGGER.info("All Done-" + nodeType);
 266                 }
 267
 268         }
 269
 270         private BufferedWriter createFile(String outfileName) throws IOException {
 271                 // FileLocation
 272                 String fileName = outfileName;
 273
 274                 File outFile = new File(fileName);
 275                 LOGGER.info(" Will write to " + outFile);
 276                 FileWriter fw = new FileWriter(outFile.getAbsoluteFile());
 277                 return new BufferedWriter(fw);
 278
 279         }
 280
 281         private void createDirectory(String dirName) throws IOException {
 282                 // FileLocation
 283                 Path pathDir = Paths.get(dirName);
 284                 Files.createDirectories(pathDir);
 285
 286         }
 287
 288         public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents)
 289                         throws AAIException, IOException {
 290
 291                 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
 292                 List<Vertex> nodes = inMemGraph.getGraph().traversal().V().has("aai-node-type", nodeType).toList();
 293
 294                 LOGGER.info("Number of nodes" + nodes.size());
 295                 String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
 296                 createDirectory( dirName);
 297                 // TODO: Formatter
 298                 if ("DMAAP-MR".equals(cArgs.format)) {
 299                         for (Vertex node : nodes) {
 300
 301                                 Introspector nodeObj = serializer.getLatestVersionView(node);
 302                                 createPayloadForDmaap(node, nodeObj);
 303
 304                         }
 305                 }
 306                 int counter = 0;
 307                 if ("PAYLOAD".equals(cArgs.format)) {
 308                         for (Vertex node : nodes) {
 309
 310                                 counter++;
 311                                 String filename = dirName + counter + "-" + nodeType + ".json";
 312                                 bw = createFile(filename);
 313                                 Introspector obj = loader.introspectorFromName(nodeType);
 314                                 Set<Vertex> seen = new HashSet<>();
 315                                 int depth = AAIProperties.MAXIMUM_DEPTH;
 316                                 boolean nodeOnly = false;
 317
 318                                 Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
 319                                 TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
 320                                 serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins, filterParents);
 321                                 createPayloadForPut(obj);
 322                                 bw.close();
 323
 324                                 URI uri = serializer.getURIForVertex(node);
 325                                 String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
 326                                 bw = createFile(filenameWithUri);
 327                                 bw.write(uri.toString());
 328                                 bw.newLine();
 329                                 bw.close();
 330                         }
 331                 }
 332
 333         }
 334
 335         public void createPayloadForPut(Introspector nodeObj) throws IOException {
 336
 337                 String entityJson = nodeObj.marshal(false);
 338                 ObjectMapper mapper = new ObjectMapper();
 339
 340                 ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
 341                 rootNode.remove("resource-version");
 342
 343                 bw.newLine();
 344                 bw.write(rootNode.toString());
 345                 bw.newLine();
 346         }
 347
 348         public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
 349                         throws AAIException, UnsupportedEncodingException {
 350
 351                 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
 352
 353                 URI uri = serializer.getURIForVertex(node);
 354
 355                 String sourceOfTruth = "";
 356                 HashMap<String, Introspector> relatedVertices = new HashMap<>();
 357                 List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
 358
 359                 for (Vertex vertex : vertexChain) {
 360                         try {
 361
 362                                 Introspector vertexObj = serializer.getVertexProperties(vertex);
 363
 364                                 relatedVertices.put(vertexObj.getObjectId(), vertexObj);
 365                         } catch (AAIUnknownObjectException e) {
 366                                 LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
 367                         }
 368
 369                 }
 370
 371                 String transactionId = "TXID";
 372                 createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
 373
 374         }
 375
 376         public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
 377                         Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
 378
 379                 String action = "CREATE";
 380                 final Introspector notificationEvent = loader.introspectorFromName("notification-event");
 381
 382                 try {
 383                         Introspector eventHeader = loader.introspectorFromName("notification-event-header");
 384                         URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
 385
 386                         String entityLink = urlBase + version + uri;
 387
 388                         notificationEvent.setValue("cambria-partition", "AAI");
 389
 390                         eventHeader.setValue("entity-link", entityLink);
 391                         eventHeader.setValue("action", action);
 392                         eventHeader.setValue("entity-type", obj.getDbName());
 393                         eventHeader.setValue("top-entity-type", parser.getTopEntityName());
 394                         eventHeader.setValue("source-name", sourceOfTruth);
 395                         eventHeader.setValue("version", version.toString());
 396                         eventHeader.setValue("id", transactionId);
 397                         eventHeader.setValue("event-type", "AAI-BASELINE");
 398                         if (eventHeader.getValue("domain") == null) {
 399                                 eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
 400                         }
 401
 402                         if (eventHeader.getValue("sequence-number") == null) {
 403                                 eventHeader.setValue("sequence-number",
 404                                                 AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
 405                         }
 406
 407                         if (eventHeader.getValue("severity") == null) {
 408                                 eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
 409                         }
 410
 411                         if (eventHeader.getValue("id") == null) {
 412                                 eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
 413
 414                         }
 415
 416                         if (eventHeader.getValue("timestamp") == null) {
 417                                 eventHeader.setValue("timestamp", genDate());
 418                         }
 419
 420                         List<Object> parentList = parser.getParentList();
 421                         parentList.clear();
 422
 423                         if (!parser.getTopEntity().equals(parser.getEntity())) {
 424                                 Introspector child;
 425                                 String json = obj.marshal(false);
 426                                 child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
 427                                 parentList.add(child.getUnderlyingObject());
 428                         }
 429
 430                         final Introspector eventObject;
 431
 432                         String json = "";
 433                         if (parser.getTopEntity().equals(parser.getEntity())) {
 434                                 json = obj.marshal(false);
 435                                 eventObject = loader.unmarshal(obj.getName(), json);
 436                         } else {
 437                                 json = parser.getTopEntity().marshal(false);
 438
 439                                 eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
 440                         }
 441                         notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
 442                         notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
 443
 444                         String entityJson = notificationEvent.marshal(false);
 445
 446                         bw.newLine();
 447                         bw.write(entityJson);
 448
 449                 } catch (AAIUnknownObjectException e) {
 450                         LOGGER.error("Fatal error - notification-event-header object not found!");
 451                 } catch (Exception e) {
 452                         LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
 453                 }
 454         }
 455
 456         private void closeGraph() {
 457                 inMemGraph.getGraph().tx().rollback();
 458                 inMemGraph.getGraph().close();
 459         }
 460
 461         public static String genDate() {
 462                 Date date = new Date();
 463                 DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
 464                 return formatter.format(date);
 465         }
 466
 467         public static String genDate2() {
 468                 Date date = new Date();
 469                 DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
 470                 return formatter.format(date);
 471         }
 472
 473 }
 474
 475 class CommandLineArgs {
 476
 477         @Parameter(names = "--help", help = true)
 478         public boolean help;
 479
 480         @Parameter(names = "-d", description = "snapshot file to be loaded")
 481         public String dataSnapshot;
 482
 483         @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
 484         public boolean schemaEnabled = true;
 485
 486         @Parameter(names = "-c", description = "location of configuration file")
 487         public String config = "";
 488
 489         @Parameter(names = "-o", description = "output location")
 490         public String output = "";
 491
 492         @Parameter(names = "-f", description = "format of output")
 493         public String format = "PAYLOAD";
 494
 495         @Parameter(names = "-n", description = "Node input file")
 496         public String nodePropertyFile = "";
 497
 498 }