[dcaegen2/services.git] /

/*\r
* ============LICENSE_START=======================================================\r
* ONAP : DataLake\r
* ================================================================================\r
* Copyright 2019 China Mobile\r
*=================================================================================\r
* Licensed under the Apache License, Version 2.0 (the "License");\r
* you may not use this file except in compliance with the License.\r
* You may obtain a copy of the License at\r
*\r
*     http://www.apache.org/licenses/LICENSE-2.0\r
*\r
* Unless required by applicable law or agreed to in writing, software\r
* distributed under the License is distributed on an "AS IS" BASIS,\r
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
* See the License for the specific language governing permissions and\r
* limitations under the License.\r
* ============LICENSE_END=========================================================\r
*/\r
package org.onap.datalake.feeder.util;\r
\r
import java.io.BufferedWriter;\r
import java.io.FileWriter;\r
import java.io.IOException;\r
import java.net.MalformedURLException;\r
import java.util.ArrayList;\r
import java.util.Iterator;\r
import java.util.List;\r
import java.util.Map.Entry;\r
\r
import org.apache.velocity.Template;\r
import org.apache.velocity.VelocityContext;\r
import org.apache.velocity.app.Velocity;\r
import org.apache.velocity.runtime.RuntimeConstants;\r
import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;\r
import org.onap.datalake.feeder.enumeration.DataFormat;\r
\r
import com.fasterxml.jackson.databind.JsonNode;\r
import com.fasterxml.jackson.databind.ObjectMapper;\r
import com.fasterxml.jackson.databind.node.JsonNodeType;\r
\r
import lombok.Getter;\r
import lombok.Setter;\r
\r
\r
/*\r
 * read sample json and output supervisor to  resources\druid\generated\r
 * need manual edit to be production ready, final versions are in resources\druid\r
 * \r
 * http://druid.io/docs/latest/tutorials/tutorial-ingestion-spec.html\r
 * http://druid.io/docs/latest/ingestion/flatten-json\r
 * \r
 * \r
 * todo:\r
 * reduce the manual editing\r
 * path hard coded\r
 * auto get topics,\r
 * auto get sample, and for each topic, get multiple samples.\r
 * make supervisor file names consistent\r
 * dimension type default is string, in msgrtr.apinode.metrics.dmaap , many are long/double, so need to generate dimensionsSpec, this is done at the end of printFlattenSpec()\r
 */\r
\r
@Getter\r
public class DruidSupervisorGenerator {\r
\r
        Template template = null;\r
        VelocityContext context;\r
\r
        List<String[]> dimensions;\r
\r
        public DruidSupervisorGenerator() {\r
                dimensions = new ArrayList<>();\r
\r
                Velocity.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath");\r
                Velocity.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());\r
\r
                Velocity.init();\r
\r
                context = new VelocityContext();\r
\r
                context.put("host", "message-router-kafka:9092");//TODO get from config\r
\r
                template = Velocity.getTemplate("druid/kafka-supervisor-template.vm");\r
        }\r
\r
        private void printNode(String prefix, JsonNode node) {\r
\r
                // lets see what type the node is\r
                //              System.out.println("NodeType=" + node.getNodeType() + ", isContainerNode=" + node.isContainerNode() + ", " + node); // prints OBJECT\r
\r
                if (node.isContainerNode()) {\r
\r
                        Iterator<Entry<String, JsonNode>> fields = node.fields();\r
\r
                        while (fields.hasNext()) {\r
                                Entry<String, JsonNode> field = fields.next();\r
                                //                              System.out.println("--------"+field.getKey()+"--------");\r
                                printNode(prefix + "." + field.getKey(), field.getValue());\r
                        }\r
\r
                        if (node.isArray()) {\r
                                Iterator<JsonNode> elements = node.elements();\r
                                int i = 0;\r
                                while (elements.hasNext()) {\r
                                        JsonNode element = elements.next();\r
                                        printNode(prefix + "[" + i + "]", element);\r
                                        i++;\r
                                }\r
                        }\r
\r
                } else {\r
                        printFlattenSpec(node.getNodeType(), prefix);\r
                }\r
\r
        }\r
\r
        private void printFlattenSpec(JsonNodeType type, String path) {\r
                String name = path.substring(2).replace('.', ':');\r
                // lets see what type the node is\r
                System.out.println("{");\r
                System.out.println("\"type\": \"path\",");\r
                System.out.println("\"name\": \"" + name + "\",");\r
                System.out.println("\"expr\": \"" + path + "\"");\r
                System.out.println("},");\r
\r
                dimensions.add(new String[] { name, path });\r
                /*\r
                 //for  dimensionsSpec\r
                                if (JsonNodeType.NUMBER.equals(type)) {\r
                                        System.out.println("{");\r
                                        System.out.println("\"type\": \"long\",");\r
                                        System.out.println("\"name\": \"" + name + "\","); \r
                                        System.out.println("},");\r
                                } else {\r
                                        System.out.println("\"" + name + "\",");\r
                \r
                                }\r
                */\r
        }\r
\r
        public void doTopic(String topic) throws IOException {\r
                dimensions.clear();\r
\r
                String sampleFileName = "C:\\git\\onap\\datalake\\olap\\src\\main\\resources\\druid\\" + topic + "-sample-format.json";//FIXME hard coded path\r
                String outputFileName = "C:\\git\\onap\\datalake\\olap\\src\\main\\resources\\druid\\generated\\" + topic + "-kafka-supervisor.json";\r
\r
                // Get the contents of json as a string using commons IO IOUTils class.\r
                String sampleJson = Util.getTextFromFile(sampleFileName);\r
\r
                // create an ObjectMapper instance.\r
                ObjectMapper mapper = new ObjectMapper();\r
                // use the ObjectMapper to read the json string and create a tree\r
                JsonNode root = mapper.readTree(sampleJson);\r
                printNode("$", root);\r
\r
                context.put("topic", topic);\r
                context.put("timestamp", "event-header:timestamp");//FIXME hard coded, should be topic based\r
                context.put("timestampFormat", "yyyyMMdd-HH:mm:ss:SSS");//FIXME hard coded, should be topic based\r
                context.put("dimensions", dimensions);\r
\r
                BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));\r
\r
                template.merge(context, out);\r
                out.close();\r
        }\r
\r
        public static void main(String[] args) throws MalformedURLException, IOException {\r
                String[] topics = new String[] { "AAI-EVENT", "msgrtr.apinode.metrics.dmaap", "unauthenticated.DCAE_CL_OUTPUT", "unauthenticated.SEC_FAULT_OUTPUT" };//FIXME hard coded\r
\r
                DruidSupervisorGenerator p = new DruidSupervisorGenerator();\r
\r
                for (String topic : topics) {\r
                        p.doTopic(topic);\r
                }\r
        }\r
}\r