2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 * ECOMP is a trademark and service mark of AT&T Intellectual Property.
22 package org.onap.aai.dbgen;
24 import java.io.BufferedWriter;
26 import java.io.FileInputStream;
27 import java.io.FileWriter;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.UnsupportedEncodingException;
32 import java.nio.file.Files;
33 import java.nio.file.Path;
34 import java.nio.file.Paths;
35 import java.util.ArrayList;
36 import java.util.HashMap;
37 import java.util.HashSet;
38 import java.util.Iterator;
39 import java.util.List;
41 import java.util.Map.Entry;
42 import java.util.Scanner;
44 import java.util.UUID;
46 import org.apache.commons.lang.exception.ExceptionUtils;
47 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
48 import org.apache.tinkerpop.gremlin.structure.Element;
49 import org.apache.tinkerpop.gremlin.structure.Vertex;
50 import org.onap.aai.db.props.AAIProperties;
51 import org.onap.aai.dbmap.DBConnectionType;
52 import org.onap.aai.dbmap.InMemoryGraph;
53 import org.onap.aai.exceptions.AAIException;
54 import org.onap.aai.introspection.Introspector;
55 import org.onap.aai.introspection.Loader;
56 import org.onap.aai.introspection.LoaderFactory;
57 import org.onap.aai.introspection.ModelType;
58 import org.onap.aai.introspection.Version;
59 import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
60 import org.onap.aai.logging.LogFormatTools;
61 import org.onap.aai.parsers.uri.URIToObject;
62 import org.onap.aai.serialization.engines.InMemoryDBEngine;
63 import org.onap.aai.serialization.engines.QueryStyle;
64 import org.onap.aai.serialization.db.DBSerializer;
66 import com.att.eelf.configuration.EELFLogger;
67 import com.att.eelf.configuration.EELFManager;
71 import org.codehaus.jackson.JsonNode;
72 import org.codehaus.jackson.map.ObjectMapper;
73 import org.codehaus.jackson.node.ObjectNode;
74 import org.codehaus.jackson.type.TypeReference;
76 import java.util.Date;
77 import java.text.DateFormat;
78 import java.text.SimpleDateFormat;
79 import org.onap.aai.serialization.tinkerpop.TreeBackedVertex;
80 import org.onap.aai.util.AAIConfig;
81 import org.onap.aai.util.AAIConstants;
83 import com.beust.jcommander.JCommander;
84 import com.beust.jcommander.Parameter;
87 * The Class ListEndpoints.
89 public class DynamicPayloadGenerator {
92 * Create a Dynamic memory graph instance which should not affect the
95 private InMemoryGraph inMemGraph = null;
96 private InMemoryDBEngine dbEngine;
99 * Loader, QueryStyle, ConnectionType for the Serializer
101 private Loader loader;
102 private String urlBase;
103 private BufferedWriter bw = null;
105 private CommandLineArgs cArgs;
107 private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DynamicPayloadGenerator.class);
109 private static final QueryStyle queryStyle = QueryStyle.TRAVERSAL;
110 private static final DBConnectionType type = DBConnectionType.CACHED;
111 private static final ModelType introspectorFactoryType = ModelType.MOXY;
116 private static final Version version = Version.getLatest();
123 * @throws AAIException
126 public static void main(String[] args) {
128 MDC.put("logFilenameAppender", DynamicPayloadGenerator.class.getSimpleName());
129 DynamicPayloadGenerator payloadgen = new DynamicPayloadGenerator();
131 payloadgen.init(args);
133 payloadgen.generatePayloads();
134 } catch (AAIException e) {
135 LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
136 } catch (IOException e) {
137 LOGGER.error("Exception " + LogFormatTools.getStackTop(e));
142 public void init(String[] args) throws AAIException {
143 cArgs = new CommandLineArgs();
144 JCommander jCommander = new JCommander(cArgs, args);
145 jCommander.setProgramName(DynamicPayloadGenerator.class.getSimpleName());
146 LOGGER.info("Snapshot file" + cArgs.dataSnapshot);
147 //TODO- How to add dynamic.properties
150 LOGGER.info("output file" + cArgs.output);
153 LOGGER.info("format file" + cArgs.format);
154 LOGGER.info("format file" + cArgs.schemaEnabled);
155 if(cArgs.config.isEmpty())
156 cArgs.config = AAIConstants.AAI_HOME_ETC_APP_PROPERTIES + "dynamic.properties";
158 LOGGER.info("config file" + cArgs.config);
159 if(cArgs.nodePropertyFile.isEmpty())
160 cArgs.nodePropertyFile = AAIConstants.AAI_HOME_ETC_SCRIPT + "/tenant_isolation/nodes.json";
161 LOGGER.info("nodePropertyFile file" + cArgs.nodePropertyFile);
164 urlBase = AAIConfig.get("aai.server.url.base", "");
168 public void generatePayloads() throws AAIException, IOException{
170 List<Map<String, List<String>>> nodeFilters = readFile(cArgs.nodePropertyFile);
171 LOGGER.info("Load the Graph");
174 LOGGER.info("Generate payload");
175 this.generatePayload(nodeFilters);
176 LOGGER.info("Close graph");
181 private List<Map<String, List<String>>> readFile(String inputFile) throws IOException {
183 InputStream is = new FileInputStream(inputFile);
184 Scanner scanner = new Scanner(is);
185 String jsonFile = scanner.useDelimiter("\\Z").next();
188 List<Map<String, List<String>>> allNodes = new ArrayList<>();
189 Map<String, List<String>> filterCousins = new HashMap<>();
190 Map<String, List<String>> filterParents = new HashMap<>();
192 ObjectMapper mapper = new ObjectMapper();
194 JsonNode rootNode = mapper.readTree(jsonFile);
196 Iterator<Entry<String, JsonNode>> nodeFields = rootNode.getFields();
198 while (nodeFields.hasNext()) {
199 Entry<String, JsonNode> entry = nodeFields.next();
200 String nodeType = entry.getKey();
201 JsonNode nodeProperty = entry.getValue();
203 JsonNode cousinFilter = nodeProperty.path("cousins");
204 JsonNode parentFilter = nodeProperty.path("parents");
205 List<String> cousins = new ObjectMapper().readValue(cousinFilter.traverse(),
206 new TypeReference<ArrayList<String>>() {
209 List<String> parents = new ObjectMapper().readValue(parentFilter.traverse(),
210 new TypeReference<ArrayList<String>>() {
212 for (String cousin : cousins) {
213 LOGGER.info("Cousins-Filtered" + cousin);
215 for (String parent : parents) {
216 LOGGER.info("Parents-Filtered" + parent);
218 filterCousins.put(nodeType, cousins);
219 filterParents.put(nodeType, parents);
223 allNodes.add(filterCousins);
224 allNodes.add(filterParents);
229 private void loadGraph() throws IOException {
231 loadGraphIntoMemory();
236 private void loadGraphIntoMemory() throws IOException {
238 inMemGraph = new InMemoryGraph.Builder().build(cArgs.dataSnapshot, cArgs.config, cArgs.schemaEnabled);
242 private void buildDbEngine() {
243 // TODO : parametrise version
244 loader = LoaderFactory.createLoaderForVersion(introspectorFactoryType, version);
246 dbEngine = new InMemoryDBEngine(queryStyle, type, loader, inMemGraph.getGraph());
247 dbEngine.startTransaction();
250 private void generatePayload(List<Map<String, List<String>>> nodeFilters) throws AAIException, IOException {
252 Map<String, List<String>> filterCousinsMap = nodeFilters.get(0);
253 Map<String, List<String>> filterParentsMap = nodeFilters.get(1);
255 Set<String> nodeTypes = filterCousinsMap.keySet();
257 for (String nodeType : nodeTypes) {
258 if ("DMAAP-MR".equals(cArgs.format)) {
259 bw = createFile(nodeType + ".json");
261 List<String> filterCousins = filterCousinsMap.get(nodeType);
262 List<String> filterParents = filterParentsMap.get(nodeType);
263 readVertices(nodeType, filterCousins, filterParents);
265 LOGGER.info("All Done-" + nodeType);
270 private BufferedWriter createFile(String outfileName) throws IOException {
272 String fileName = outfileName;
274 File outFile = new File(fileName);
275 LOGGER.info(" Will write to " + outFile);
276 FileWriter fw = new FileWriter(outFile.getAbsoluteFile());
277 return new BufferedWriter(fw);
281 private void createDirectory(String dirName) throws IOException {
283 Path pathDir = Paths.get(dirName);
284 Files.createDirectories(pathDir);
288 public void readVertices(String nodeType, List<String> filterCousins, List<String> filterParents)
289 throws AAIException, IOException {
291 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
292 List<Vertex> nodes = inMemGraph.getGraph().traversal().V().has("aai-node-type", nodeType).toList();
294 LOGGER.info("Number of nodes" + nodes.size());
295 String dirName = cArgs.output + AAIConstants.AAI_FILESEP + nodeType + AAIConstants.AAI_FILESEP;
296 createDirectory( dirName);
298 if ("DMAAP-MR".equals(cArgs.format)) {
299 for (Vertex node : nodes) {
301 Introspector nodeObj = serializer.getLatestVersionView(node);
302 createPayloadForDmaap(node, nodeObj);
307 if ("PAYLOAD".equals(cArgs.format)) {
308 for (Vertex node : nodes) {
311 String filename = dirName + counter + "-" + nodeType + ".json";
312 bw = createFile(filename);
313 Introspector obj = loader.introspectorFromName(nodeType);
314 Set<Vertex> seen = new HashSet<>();
315 int depth = AAIProperties.MAXIMUM_DEPTH;
316 boolean nodeOnly = false;
318 Tree<Element> tree = dbEngine.getQueryEngine().findSubGraph(node, depth, nodeOnly);
319 TreeBackedVertex treeVertex = new TreeBackedVertex(node, tree);
320 serializer.dbToObjectWithFilters(obj, treeVertex, seen, depth, nodeOnly, filterCousins, filterParents);
321 createPayloadForPut(obj);
324 URI uri = serializer.getURIForVertex(node);
325 String filenameWithUri = dirName + counter + "-" + nodeType + ".txt";
326 bw = createFile(filenameWithUri);
327 bw.write(uri.toString());
335 public void createPayloadForPut(Introspector nodeObj) throws IOException {
337 String entityJson = nodeObj.marshal(false);
338 ObjectMapper mapper = new ObjectMapper();
340 ObjectNode rootNode = (ObjectNode) mapper.readTree(entityJson);
341 rootNode.remove("resource-version");
344 bw.write(rootNode.toString());
348 public void createPayloadForDmaap(Vertex node, Introspector nodeObj)
349 throws AAIException, UnsupportedEncodingException {
351 DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, "sourceOfTruth");
353 URI uri = serializer.getURIForVertex(node);
355 String sourceOfTruth = "";
356 HashMap<String, Introspector> relatedVertices = new HashMap<>();
357 List<Vertex> vertexChain = dbEngine.getQueryEngine().findParents(node);
359 for (Vertex vertex : vertexChain) {
362 Introspector vertexObj = serializer.getVertexProperties(vertex);
364 relatedVertices.put(vertexObj.getObjectId(), vertexObj);
365 } catch (AAIUnknownObjectException e) {
366 LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
371 String transactionId = "TXID";
372 createNotificationEvent(transactionId, sourceOfTruth, uri, nodeObj, relatedVertices);
376 public void createNotificationEvent(String transactionId, String sourceOfTruth, URI uri, Introspector obj,
377 Map<String, Introspector> relatedObjects) throws AAIException, UnsupportedEncodingException {
379 String action = "CREATE";
380 final Introspector notificationEvent = loader.introspectorFromName("notification-event");
383 Introspector eventHeader = loader.introspectorFromName("notification-event-header");
384 URIToObject parser = new URIToObject(loader, uri, (HashMap) relatedObjects);
386 String entityLink = urlBase + version + uri;
388 notificationEvent.setValue("cambria-partition", "AAI");
390 eventHeader.setValue("entity-link", entityLink);
391 eventHeader.setValue("action", action);
392 eventHeader.setValue("entity-type", obj.getDbName());
393 eventHeader.setValue("top-entity-type", parser.getTopEntityName());
394 eventHeader.setValue("source-name", sourceOfTruth);
395 eventHeader.setValue("version", version.toString());
396 eventHeader.setValue("id", transactionId);
397 eventHeader.setValue("event-type", "AAI-BASELINE");
398 if (eventHeader.getValue("domain") == null) {
399 eventHeader.setValue("domain", AAIConfig.get("aai.notificationEvent.default.domain", "UNK"));
402 if (eventHeader.getValue("sequence-number") == null) {
403 eventHeader.setValue("sequence-number",
404 AAIConfig.get("aai.notificationEvent.default.sequenceNumber", "UNK"));
407 if (eventHeader.getValue("severity") == null) {
408 eventHeader.setValue("severity", AAIConfig.get("aai.notificationEvent.default.severity", "UNK"));
411 if (eventHeader.getValue("id") == null) {
412 eventHeader.setValue("id", genDate2() + "-" + UUID.randomUUID().toString());
416 if (eventHeader.getValue("timestamp") == null) {
417 eventHeader.setValue("timestamp", genDate());
420 List<Object> parentList = parser.getParentList();
423 if (!parser.getTopEntity().equals(parser.getEntity())) {
425 String json = obj.marshal(false);
426 child = parser.getLoader().unmarshal(parser.getEntity().getName(), json);
427 parentList.add(child.getUnderlyingObject());
430 final Introspector eventObject;
433 if (parser.getTopEntity().equals(parser.getEntity())) {
434 json = obj.marshal(false);
435 eventObject = loader.unmarshal(obj.getName(), json);
437 json = parser.getTopEntity().marshal(false);
439 eventObject = loader.unmarshal(parser.getTopEntity().getName(), json);
441 notificationEvent.setValue("event-header", eventHeader.getUnderlyingObject());
442 notificationEvent.setValue("entity", eventObject.getUnderlyingObject());
444 String entityJson = notificationEvent.marshal(false);
447 bw.write(entityJson);
449 } catch (AAIUnknownObjectException e) {
450 LOGGER.error("Fatal error - notification-event-header object not found!");
451 } catch (Exception e) {
452 LOGGER.error("Unmarshalling error occurred while generating Notification " + LogFormatTools.getStackTop(e));
456 private void closeGraph() {
457 inMemGraph.getGraph().tx().rollback();
458 inMemGraph.getGraph().close();
461 public static String genDate() {
462 Date date = new Date();
463 DateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH:mm:ss:SSS");
464 return formatter.format(date);
467 public static String genDate2() {
468 Date date = new Date();
469 DateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
470 return formatter.format(date);
475 class CommandLineArgs {
477 @Parameter(names = "--help", help = true)
480 @Parameter(names = "-d", description = "snapshot file to be loaded")
481 public String dataSnapshot;
483 @Parameter(names = "-s", description = "is schema to be enabled ", arity = 1)
484 public boolean schemaEnabled = true;
486 @Parameter(names = "-c", description = "location of configuration file")
487 public String config = "";
489 @Parameter(names = "-o", description = "output location")
490 public String output = "";
492 @Parameter(names = "-f", description = "format of output")
493 public String format = "PAYLOAD";
495 @Parameter(names = "-n", description = "Node input file")
496 public String nodePropertyFile = "";