2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 package org.onap.aai.dbgen;
22 import java.io.BufferedWriter;
24 import java.io.FileInputStream;
25 import java.io.FileWriter;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.UnsupportedEncodingException;
30 import java.nio.file.Files;
31 import java.nio.file.Path;
32 import java.nio.file.Paths;
33 import java.util.ArrayList;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.Iterator;
37 import java.util.List;
39 import java.util.Map.Entry;
40 import java.util.Scanner;
42 import java.util.UUID;
44 import org.apache.commons.lang.exception.ExceptionUtils;
45 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
46 import org.apache.tinkerpop.gremlin.structure.Element;
47 import org.apache.tinkerpop.gremlin.structure.Vertex;
48 import org.onap.aai.db.props.AAIProperties;
49 import org.onap.aai.dbmap.DBConnectionType;
50 import org.onap.aai.dbmap.InMemoryGraph;
51 import org.onap.aai.exceptions.AAIException;
52 import org.onap.aai.introspection.Introspector;
53 import org.onap.aai.introspection.Loader;
54 import org.onap.aai.introspection.LoaderFactory;
55 import org.onap.aai.introspection.ModelType;
56 import org.onap.aai.introspection.Version;
57 import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
58 import org.onap.aai.logging.LogFormatTools;
59 import org.onap.aai.parsers.uri.URIToObject;
60 import org.onap.aai.serialization.engines.InMemoryDBEngine;
61 import org.onap.aai.serialization.engines.QueryStyle;
62 import org.onap.aai.serialization.db.DBSerializer;
64 import com.att.eelf.configuration.EELFLogger;
65 import com.att.eelf.configuration.EELFManager;
69 import org.codehaus.jackson.JsonNode;
70 import org.codehaus.jackson.map.ObjectMapper;
71 import org.codehaus.jackson.node.ObjectNode;
72 import org.codehaus.jackson.type.TypeReference;
74 import java.util.Date;
75 import java.text.DateFormat;
76 import java.text.SimpleDateFormat;
77 import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
78 import org.onap.aai.util.AAIConfig;
79 import org.onap.aai.util.AAIConstants;
81 import com.beust.jcommander.JCommander;
82 import com.beust.jcommander.Parameter;
85 * The Class ListEndpoints.
87 public class DynamicPayloadGenerator {
90 * Create a Dynamic memory graph instance which should not affect the
93 private InMemoryGraph inMemGraph = null;
94 private InMemoryDBEngine dbEngine;
97 * Loader, QueryStyle, ConnectionType for the Serializer
99 private Loader loader;
100 private String urlBase;
101 private BufferedWriter bw = null;
103 private CommandLineArgs cArgs;
105 private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
107 private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
108 private static final DBConnectionType type = DBConnectionType.CACHED;
109 private static final ModelType introspectorFactoryType = ModelType.MOXY;
114 private static final Version version = Version.getLatest();
121 * @throws AAIException
124 public static void main(String[] args) {
126 MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
127 DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator();
129 payloadgen.init(args);
131 payloadgen.generatePayloads();
132 } catch (AAIException e) {
133 LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
134 } catch (IOException e) {
135 LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
140 public void init(String[] args) throws AAIException {
141 cArgs = new CommandLineArgs();
142 JCommander jCommander = new JCommander(cArgs, args);
143 jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
144 LOGGER.info("Snapshot file" + cArgs.dataSnapshot);
145 //TODO- How to add dynamic.properties
148 LOGGER.info("output file" + cArgs.output);
151 LOGGER.info("format file" + cArgs.format);
152 LOGGER.info("format file" + cArgs.schemaEnabled);
153 if(cArgs.config.isEmpty())
154 cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
156 LOGGER.info("config file" + cArgs.config);
157 if(cArgs.nodePropertyFile.isEmpty())
158 cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
159 LOGGER.info("nodePropertyFile file" + cArgs.nodePropertyFile);
162 urlBase = AAIConfig.get("aai.server.url.base", "");
166 public void generatePayloads() throws AAIException, IOException{
168 List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
169 LOGGER.info("Load the Graph");
172 LOGGER.info("Generate payload");
173 this.generatePayload(nodeFilters);
174 LOGGER.info("Close graph");
179 private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
181 InputStream is = new FileInputStream(inputFile);
182 Scanner scanner = new Scanner(is);
183 String jsonFile = scanner.useDelimiter("\\Z").next();
186 List<Map<String, List<String>>> allNodes = new ArrayList<>();
187 Map<String, List<String>> filterCousins = new HashMap<>();
188 Map<String, List<String>> filterParents = new HashMap<>();
190 ObjectMapper mapper = new ObjectMapper();
192 JsonNode rootNode = mapper.readTree(jsonFile);
194 Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
196 while (nodeFields.hasNext()) {
197 Entry<String, JsonNode> entry = nodeFields.next();
198 String nodeType = entry.getKey();
199 JsonNode nodeProperty = entry.getValue();
201 JsonNode cousinFilter = nodeProperty.path("cousins");
202 JsonNode parentFilter = nodeProperty.path("parents");
203 List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
204 new TypeReference<ArrayList<String>>() {
207 List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
208 new TypeReference<ArrayList<String>>() {
210 for (String cousin : cousins) {
211 LOGGER.info("Cousins-Filtered" + cousin);
213 for (String parent : parents) {
214 LOGGER.info("Parents-Filtered" + parent);
216 filterCousins.put(nodeType, cousins);
217 filterParents.put(nodeType, parents);
221 allNodes.add(filterCousins);
222 allNodes.add(filterParents);
227 private void loadGraph() throws IOException {
229 loadGraphIntoMemory();
234 private void loadGraphIntoMemory() throws IOException {
236 inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled);
240 private void buildDbEngine() {
241 // TODO : parametrise version
242 loader = LoaderFactory.createLoaderForVersion(introspectorFactoryType, version);
244 dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
245 dbEngine.startTransaction();
248 private void generatePayload(List<Map<String, List<String>>> nodeFilters) throws AAIException, IOException {
250 Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
251 Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
253 Set<String> nodeTypes = filterCousinsMap.keySet();
255 for (String nodeType : nodeTypes) {
256 if ("DMAAP-MR".equals(cArgs.format)) {
257 bw = createFile(nodeType + ".json");
259 List<String> filterCousins = filterCousinsMap.get(nodeType);
260 List<String> filterParents = filterParentsMap.get(nodeType);
261 readVertices(nodeType, filterCousins, filterParents);
263 LOGGER.info("All Done-" + nodeType);
268 private BufferedWriter createFile(String outfileName) throws IOException {
270 String fileName = outfileName;
272 File outFile = new File(fileName);
273 LOGGER.info(" Will write to " + outFile);
274 FileWriter fw = new FileWriter(outFile.getAbsoluteFile());
275 return new BufferedWriter(fw);
279 private void createDirectory(String dirName) throws IOException {
281 Path pathDir = Paths.get(dirName);
282 Files.createDirectories(pathDir);
286 public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents)
287 throws AAIException, IOException {
289 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
290 List<Vertex> nodes = inMemGraph.getGraph().traversal().V().has("aai-node-type", nodeType).toList();
292 LOGGER.info("Number of nodes" + nodes.size());
293 String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
294 createDirectory( dirName);
296 if ("DMAAP-MR".equals(cArgs.format)) {
297 for (Vertex node : nodes) {
299 Introspector nodeObj = serializer.getLatestVersionView(node);
300 createPayloadForDmaap(node, nodeObj);
305 if ("PAYLOAD".equals(cArgs.format)) {
306 for (Vertex node : nodes) {
309 String filename = dirName + counter + "-" + nodeType + ".json";
310 bw = createFile(filename);
311 Introspector obj = loader.introspectorFromName(nodeType);
312 Set<Vertex> seen = new HashSet<>();
313 int depth = AAIProperties.MAXIMUM_DEPTH;
314 boolean nodeOnly = false;
316 Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
317 TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
318 serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins, filterParents);
319 createPayloadForPut(obj);
322 URI uri = serializer.getURIForVertex(node);
323 String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
324 bw = createFile(filenameWithUri);
325 bw.write(uri.toString());
333 public void createPayloadForPut(Introspector nodeObj) throws IOException {
335 String entityJson = nodeObj.marshal(false);
336 ObjectMapper mapper = new ObjectMapper();
338 ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
339 rootNode.remove("resource-version");
342 bw.write(rootNode.toString());
346 public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
347 throws AAIException, UnsupportedEncodingException {
349 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
351 URI uri = serializer.getURIForVertex(node);
353 String sourceOfTruth = "";
354 HashMap<String, Introspector> relatedVertices = new HashMap<>();
355 List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
357 for (Vertex vertex : vertexChain) {
360 Introspector vertexObj = serializer.getVertexProperties(vertex);
362 relatedVertices.put(vertexObj.getObjectId(), vertexObj);
363 } catch (AAIUnknownObjectException e) {
364 LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
369 String transactionId = "TXID";
370 createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
374 public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
375 Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
377 String action = "CREATE";
378 final Introspector notificationEvent = loader.introspectorFromName("notification-event");
381 Introspector eventHeader = loader.introspectorFromName("notification-event-header");
382 URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
384 String entityLink = urlBase + version + uri;
386 notificationEvent.setValue("cambria-partition", "AAI");
388 eventHeader.setValue("entity-link", entityLink);
389 eventHeader.setValue("action", action);
390 eventHeader.setValue("entity-type", obj.getDbName());
391 eventHeader.setValue("top-entity-type", parser.getTopEntityName());
392 eventHeader.setValue("source-name", sourceOfTruth);
393 eventHeader.setValue("version", version.toString());
394 eventHeader.setValue("id", transactionId);
395 eventHeader.setValue("event-type", "AAI-BASELINE");
396 if (eventHeader.getValue("domain") == null) {
397 eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
400 if (eventHeader.getValue("sequence-number") == null) {
401 eventHeader.setValue("sequence-number",
402 AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
405 if (eventHeader.getValue("severity") == null) {
406 eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
409 if (eventHeader.getValue("id") == null) {
410 eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
414 if (eventHeader.getValue("timestamp") == null) {
415 eventHeader.setValue("timestamp", genDate());
418 List<Object> parentList = parser.getParentList();
421 if (!parser.getTopEntity().equals(parser.getEntity())) {
423 String json = obj.marshal(false);
424 child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
425 parentList.add(child.getUnderlyingObject());
428 final Introspector eventObject;
431 if (parser.getTopEntity().equals(parser.getEntity())) {
432 json = obj.marshal(false);
433 eventObject = loader.unmarshal(obj.getName(), json);
435 json = parser.getTopEntity().marshal(false);
437 eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
439 notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
440 notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
442 String entityJson = notificationEvent.marshal(false);
445 bw.write(entityJson);
447 } catch (AAIUnknownObjectException e) {
448 LOGGER.error("Fatal error - notification-event-header object not found!");
449 } catch (Exception e) {
450 LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
454 private void closeGraph() {
455 inMemGraph.getGraph().tx().rollback();
456 inMemGraph.getGraph().close();
459 public static String genDate() {
460 Date date = new Date();
461 DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
462 return formatter.format(date);
465 public static String genDate2() {
466 Date date = new Date();
467 DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
468 return formatter.format(date);
473 class CommandLineArgs {
475 @Parameter(names = "--help", help = true)
478 @Parameter(names = "-d", description = "snapshot file to be loaded")
479 public String dataSnapshot;
481 @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
482 public boolean schemaEnabled = true;
484 @Parameter(names = "-c", description = "location of configuration file")
485 public String config = "";
487 @Parameter(names = "-o", description = "output location")
488 public String output = "";
490 @Parameter(names = "-f", description = "format of output")
491 public String format = "PAYLOAD";
493 @Parameter(names = "-n", description = "Node input file")
494 public String nodePropertyFile = "";