Druid Kafka Indexing Service Supervisor Spec for topic 64/89664/1
authorGuobiao Mo <guobiaomo@chinamobile.com>
Tue, 11 Jun 2019 00:08:54 +0000 (17:08 -0700)
committerGuobiao Mo <guobiaomo@chinamobile.com>
Tue, 11 Jun 2019 00:08:54 +0000 (17:08 -0700)
HW and EPC

Issue-ID: DCAEGEN2-1598

Change-Id: Iee4a3a3efc2aa04afb1ab234070cd393ca110055
Signed-off-by: Guobiao Mo <guobiaomo@chinamobile.com>
components/datalake-handler/feeder/src/assembly/scripts/init_db.sql
components/datalake-handler/feeder/src/main/resources/druid/AAI-EVENT-kafka-supervisor.json
components/datalake-handler/feeder/src/main/resources/druid/EPC-kafka-supervisor.json [new file with mode: 0644]
components/datalake-handler/feeder/src/main/resources/druid/EPC-sample-format.json [new file with mode: 0644]
components/datalake-handler/feeder/src/main/resources/druid/HW-aggs-kafka-supervisor.json [new file with mode: 0644]
components/datalake-handler/feeder/src/main/resources/druid/HW-aggs-sample-format.json [new file with mode: 0644]

index ad142dc..4fe9ad3 100644 (file)
@@ -88,30 +88,30 @@ insert into db (`name`,`host`,`login`) values ('HDFS','dlhdfs','dl');
 \r
 -- in production, default enabled should be off\r
 insert into `topic`(`name`,`enabled`,`save_raw`,`ttl`,`data_format`) values ('_DL_DEFAULT_',1,0,3650,'JSON');\r
+insert into `map_db_topic`(`db_name`,`topic_name`) select `name`, '_DL_DEFAULT_' from db;\r
+\r
+\r
 insert into `topic`(`name`,correlate_cleared_message,`enabled`, message_id_path,`data_format`) values ('unauthenticated.SEC_FAULT_OUTPUT',1,1,'/event/commonEventHeader/eventName,/event/commonEventHeader/reportingEntityName,/event/faultFields/specificProblem,/event/commonEventHeader/eventId','JSON');\r
+insert into `map_db_topic`(`db_name`,`topic_name`) select `name`, 'unauthenticated.SEC_FAULT_OUTPUT' from db;\r
+\r
 insert into `topic`(`name`,`enabled`, aggregate_array_path,flatten_array_path,`data_format`) \r
 values ('unauthenticated.VES_MEASUREMENT_OUTPUT',1,\r
-'/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray',\r
+'/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray',\r
 '/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface',\r
 'JSON');\r
+insert into `map_db_topic`(`db_name`,`topic_name`) select `name`, 'unauthenticated.VES_MEASUREMENT_OUTPUT' from db;\r
 \r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Couchbase','_DL_DEFAULT_');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Elasticsearch','_DL_DEFAULT_');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('MongoDB','_DL_DEFAULT_');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Druid','_DL_DEFAULT_');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('HDFS','_DL_DEFAULT_');\r
-\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Couchbase','unauthenticated.SEC_FAULT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Elasticsearch','unauthenticated.SEC_FAULT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('MongoDB','unauthenticated.SEC_FAULT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Druid','unauthenticated.SEC_FAULT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('HDFS','unauthenticated.SEC_FAULT_OUTPUT');\r
+insert into `topic`(`name`,`enabled`,  flatten_array_path,`data_format`) \r
+values ('EPC',1, \r
+'/event/measurementsForVfScalingFields/astriMeasurement/astriDPMeasurementArray/astriInterface',\r
+'JSON');\r
+insert into `map_db_topic`(`db_name`,`topic_name`) select `name`, 'EPC' from db;\r
 \r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Couchbase','unauthenticated.VES_MEASUREMENT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Elasticsearch','unauthenticated.VES_MEASUREMENT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('MongoDB','unauthenticated.VES_MEASUREMENT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('Druid','unauthenticated.VES_MEASUREMENT_OUTPUT');\r
-insert into `map_db_topic`(`db_name`,`topic_name`) values ('HDFS','unauthenticated.VES_MEASUREMENT_OUTPUT');\r
+insert into `topic`(`name`,`enabled`, aggregate_array_path,`data_format`) \r
+values ('HW',1,\r
+'/event/measurementsForVfScalingFields/memoryUsageArray,/event/measurementsForVfScalingFields/diskUsageArray,/event/measurementsForVfScalingFields/cpuUsageArray,/event/measurementsForVfScalingFields/vNicPerformanceArray',\r
+'JSON');\r
+insert into `map_db_topic`(`db_name`,`topic_name`) select `name`, 'HW' from db;\r
 \r
 insert into portal (`name`,`related_db`, host) values ('Kibana', 'Elasticsearch', 'dl_es');\r
 insert into portal (`name`,`related_db`) values ('Elasticsearch', 'Elasticsearch');\r
@@ -121,5 +121,5 @@ insert into design_type (`name`,`portal`) values ('Kibana Dashboard', 'Kibana');
 insert into design_type (`name`,`portal`) values ('Kibana Search', 'Kibana');\r
 insert into design_type (`name`,`portal`) values ('Kibana Visualization', 'Kibana');\r
 insert into design_type (`name`,`portal`) values ('Elasticsearch Field Mapping Template', 'Elasticsearch');\r
-insert into design_type (`name`,`portal`) values ('Druid Kafka Indexing Service Supervisor', 'Druid');\r
+insert into design_type (`name`,`portal`) values ('Druid Kafka Indexing Service Supervisor Spec', 'Druid');\r
 \r
index a20e5eb..cf63c45 100644 (file)
                                "flattenSpec": {
                                        "useFieldDiscovery": false,
                                        "fields": [
-{
-"type": "path",
-"name": "cambria:partition",
-"expr": "$.cambria.partition"
-},
-{
-"type": "path",
-"name": "event-header:severity",
-"expr": "$.event-header.severity"
-},
-{
-"type": "path",
-"name": "event-header:entity-type",
-"expr": "$.event-header.entity-type"
-},
-{
-"type": "path",
-"name": "event-header:top-entity-type",
-"expr": "$.event-header.top-entity-type"
-},
-{
-"type": "path",
-"name": "event-header:entity-link",
-"expr": "$.event-header.entity-link"
-},
-{
-"type": "path",
-"name": "event-header:event-type",
-"expr": "$.event-header.event-type"
-},
-{
-"type": "path",
-"name": "event-header:domain",
-"expr": "$.event-header.domain"
-},
-{
-"type": "path",
-"name": "event-header:action",
-"expr": "$.event-header.action"
-},
-{
-"type": "path",
-"name": "event-header:sequence-number",
-"expr": "$.event-header.sequence-number"
-},
-{
-"type": "path",
-"name": "event-header:id",
-"expr": "$.event-header.id"
-},
-{
-"type": "path",
-"name": "event-header:source-name",
-"expr": "$.event-header.source-name"
-},
-{
-"type": "path",
-"name": "event-header:version",
-"expr": "$.event-header.version"
-},
-{
-"type": "path",
-"name": "event-header:timestamp",
-"expr": "$.event-header.timestamp"
-},
-{
-"type": "path",
-"name": "entity:thirdparty-sdnc-id",
-"expr": "$.entity.thirdparty-sdnc-id"
-},
-{
-"type": "path",
-"name": "entity:resource-version",
-"expr": "$.entity.resource-version"
-},
-{
-"type": "path",
-"name": "entity:location",
-"expr": "$.entity.location"
-},
-{
-"type": "path",
-"name": "entity:product-name",
-"expr": "$.entity.product-name"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:esr-system-info-id",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].esr-system-info-id"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:system-type",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].system-type"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:service-url",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].service-url"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:ssl-cacert",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].ssl-cacert"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:type",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].type"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:ssl-insecure",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].ssl-insecure"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:system-status",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].system-status"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:version",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].version"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:passive",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].passive"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:password",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].password"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:protocol",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].protocol"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:ip-address",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].ip-address"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:cloud-domain",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].cloud-domain"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:user-name",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].user-name"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:system-name",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].system-name"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:port",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].port"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:vendor",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].vendor"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:resource-version",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].resource-version"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:remote-path",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].remote-path"
-},
-{
-"type": "path",
-"name": "entity:esr-system-info-list:esr-system-info[0]:default-tenant",
-"expr": "$.entity.esr-system-info-list.esr-system-info[0].default-tenant"
-}
+                                               {
+                                                       "type": "path",
+                                                       "name": "cambria:partition",
+                                                       "expr": "$.cambria.partition"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:severity",
+                                                       "expr": "$.event-header.severity"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:entity-type",
+                                                       "expr": "$.event-header.entity-type"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:top-entity-type",
+                                                       "expr": "$.event-header.top-entity-type"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:entity-link",
+                                                       "expr": "$.event-header.entity-link"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:event-type",
+                                                       "expr": "$.event-header.event-type"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:domain",
+                                                       "expr": "$.event-header.domain"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:action",
+                                                       "expr": "$.event-header.action"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:sequence-number",
+                                                       "expr": "$.event-header.sequence-number"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:id",
+                                                       "expr": "$.event-header.id"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:source-name",
+                                                       "expr": "$.event-header.source-name"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:version",
+                                                       "expr": "$.event-header.version"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "event-header:timestamp",
+                                                       "expr": "$.event-header.timestamp"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:thirdparty-sdnc-id",
+                                                       "expr": "$.entity.thirdparty-sdnc-id"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:resource-version",
+                                                       "expr": "$.entity.resource-version"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:location",
+                                                       "expr": "$.entity.location"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:product-name",
+                                                       "expr": "$.entity.product-name"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:esr-system-info-id",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].esr-system-info-id"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:system-type",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].system-type"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:service-url",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].service-url"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:ssl-cacert",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].ssl-cacert"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:type",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].type"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:ssl-insecure",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].ssl-insecure"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:system-status",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].system-status"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:version",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].version"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:passive",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].passive"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:password",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].password"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:protocol",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].protocol"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:ip-address",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].ip-address"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:cloud-domain",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].cloud-domain"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:user-name",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].user-name"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:system-name",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].system-name"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:port",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].port"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:vendor",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].vendor"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:resource-version",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].resource-version"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:remote-path",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].remote-path"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "entity:esr-system-info-list:esr-system-info[0]:default-tenant",
+                                                       "expr": "$.entity.esr-system-info-list.esr-system-info[0].default-tenant"
+                                               }
                                        ]
                                },
                                "timestampSpec": {
                                "dimensionsSpec": {
                                        "dimensions": [],
                                        "dimensionsExclusions": [
-                                               "cambria.partition","password"
+                                               "cambria.partition",
+                                               "password"
                                        ]
                                }
                        }
diff --git a/components/datalake-handler/feeder/src/main/resources/druid/EPC-kafka-supervisor.json b/components/datalake-handler/feeder/src/main/resources/druid/EPC-kafka-supervisor.json
new file mode 100644 (file)
index 0000000..c6562a2
--- /dev/null
@@ -0,0 +1,247 @@
+{
+       "type": "kafka",
+       "dataSchema": {
+               "dataSource": "EPC",
+               "parser": {
+                       "type": "string",
+                       "parseSpec": {
+                               "format": "json",
+                               "flattenSpec": {
+                                       "useFieldDiscovery": false,
+                                       "fields": [
+                                               {
+                                                       "type": "path",
+                                                       "name": "astriDPHost",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPHost"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriDPtx_pps",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriDPtx_pps"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriDPrx_error_packets",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriDPrx_error_packets"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriDPrx_throughput",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriDPrx_throughput"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriInterface",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriInterface"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriDPtx_throughput",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriDPtx_throughput"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriDPrx_pps",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriDPrx_pps"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port1-astriDPrx_missed_packets",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[0].astriDPrx_missed_packets"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriDPtx_pps",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriDPtx_pps"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriDPrx_error_packets",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriDPrx_error_packets"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriDPrx_throughput",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriDPrx_throughput"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriInterface",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriInterface"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriDPtx_throughput",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriDPtx_throughput"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriDPrx_pps",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriDPrx_pps"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "port0-astriDPrx_missed_packets",
+                                                       "expr": "$.event.measurementsForVfScalingFields.astriMeasurement.astriDPMeasurementArray[1].astriDPrx_missed_packets"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "eventId",
+                                                       "expr": "$.event.commonEventHeader.eventId"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "reportingEntityId",
+                                                       "expr": "$.event.commonEventHeader.reportingEntityId"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "lastEpochMicrosec",
+                                                       "expr": "$.event.commonEventHeader.lastEpochMicrosec"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "reportingEntityName",
+                                                       "expr": "$.event.commonEventHeader.reportingEntityName"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "sourceId",
+                                                       "expr": "$.event.commonEventHeader.sourceId"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "eventType",
+                                                       "expr": "$.event.commonEventHeader.eventType"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "sequence",
+                                                       "expr": "$.event.commonEventHeader.sequence"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "collectorTimeStamp",
+                                                       "expr": "$.event.commonEventHeader.internalHeaderFields.collectorTimeStamp"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "priority",
+                                                       "expr": "$.event.commonEventHeader.priority"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "sourceName",
+                                                       "expr": "$.event.commonEventHeader.sourceName"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "eventName",
+                                                       "expr": "$.event.commonEventHeader.eventName"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "startEpochMicrosec",
+                                                       "expr": "$.event.commonEventHeader.startEpochMicrosec"
+                                               }
+                                       ]
+                               },
+                               "timestampSpec": {
+                                       "column": "collectorTimeStamp",
+                                       "format": "EEE, MM dd yyyy HH:mm:ss z"
+                               },
+                               "dimensionsSpec": {
+                                       "dimensions": [],
+                                       "dimensionsExclusions": []
+                               }
+                       }
+               },
+               "metricsSpec": [
+                       {
+                               "type": "count",
+                               "name": "count"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port0-astriDPtx_pps",
+                               "fieldName": "port0-astriDPtx_pps"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port0-astriDPrx_error_packets",
+                               "fieldName": "port0-astriDPrx_error_packets"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port0-astriDPrx_throughput",
+                               "fieldName": "port0-astriDPrx_throughput"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port0-astriDPtx_throughput",
+                               "fieldName": "port0-astriDPtx_throughput"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port0-astriDPrx_pps",
+                               "fieldName": "port0-astriDPrx_pps"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port0-astriDPrx_missed_packets",
+                               "fieldName": "port0-astriDPrx_missed_packets"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port1-astriDPtx_pps",
+                               "fieldName": "port1-astriDPtx_pps"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port1-astriDPrx_error_packets",
+                               "fieldName": "port1-astriDPrx_error_packets"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port1-astriDPrx_throughput",
+                               "fieldName": "port1-astriDPrx_throughput"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port1-astriDPtx_throughput",
+                               "fieldName": "port1-astriDPtx_throughput"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port1-astriDPrx_pps",
+                               "fieldName": "port1-astriDPrx_pps"
+                       },
+                       {
+                               "type": "longSum",
+                               "name": "port1-astriDPrx_missed_packets",
+                               "fieldName": "port1-astriDPrx_missed_packets"
+                       }
+               ],
+               "granularitySpec": {
+                       "type": "uniform",
+                       "segmentGranularity": "HOUR",
+                       "queryGranularity": "MINUTE",
+                       "rollup": true
+               }
+       },
+       "tuningConfig": {
+               "type": "kafka",
+               "reportParseExceptions": true
+       },
+       "ioConfig": {
+               "topic": "EPC",
+               "replicas": 1,
+               "startDelay": "PT1S",
+               "taskDuration": "PT1H",
+               "completionTimeout": "PT30M",
+               "consumerProperties": {
+                       "bootstrap.servers": "message-router-kafka:9092"
+               },
+               "useEarliestOffset": true
+       }
+}
diff --git a/components/datalake-handler/feeder/src/main/resources/druid/EPC-sample-format.json b/components/datalake-handler/feeder/src/main/resources/druid/EPC-sample-format.json
new file mode 100644 (file)
index 0000000..2d15837
--- /dev/null
@@ -0,0 +1,51 @@
+{\r
+       "event": {\r
+               "measurementsForVfScalingFields": {\r
+                       "additionalMeasurements": [],\r
+                       "measurementInterval": 10,\r
+                       "astriMeasurement": {\r
+                               "valuesAreSuspect": "true",\r
+                               "astriDPHost": "DP5-10.213.1.143",\r
+                               "astriDPMeasurementArray": [\r
+                                       {\r
+                                               "astriDPtx_pps": 0,\r
+                                               "astriDPrx_error_packets": 0,\r
+                                               "astriDPrx_throughput": 16619551392,\r
+                                               "astriInterface": "port1",\r
+                                               "astriDPtx_throughput": 0,\r
+                                               "astriDPrx_pps": 1675357,\r
+                                               "astriDPrx_missed_packets": 0\r
+                                       },\r
+                                       {\r
+                                               "astriDPtx_pps": 586325,\r
+                                               "astriDPrx_error_packets": 0,\r
+                                               "astriDPrx_throughput": 5984991232,\r
+                                               "astriInterface": "port0",\r
+                                               "astriDPtx_throughput": 309579600,\r
+                                               "astriDPrx_pps": 586304,\r
+                                               "astriDPrx_missed_packets": 0\r
+                                       }\r
+                               ]\r
+                       },\r
+                       "measurementsForVfScalingVersion": 2\r
+               },\r
+               "commonEventHeader": {\r
+                       "eventId": "84891",\r
+                       "reportingEntityId": "localhost",\r
+                       "domain": "measurementsForVfScaling",\r
+                       "lastEpochMicrosec": 1558757056,\r
+                       "reportingEntityName": "localhost",\r
+                       "sourceId": "DP5-10.213.1.143",\r
+                       "eventType": "platform",\r
+                       "sequence": 0,\r
+                       "internalHeaderFields": {\r
+                               "collectorTimeStamp": "Sat, 05 25 2019 04:04:16 UTC"\r
+                       },\r
+                       "priority": "Normal",\r
+                       "sourceName": "DP5-10.213.1.143",\r
+                       "eventName": "",\r
+                       "version": 2,\r
+                       "startEpochMicrosec": 1558757046\r
+               }\r
+       }\r
+}\r
diff --git a/components/datalake-handler/feeder/src/main/resources/druid/HW-aggs-kafka-supervisor.json b/components/datalake-handler/feeder/src/main/resources/druid/HW-aggs-kafka-supervisor.json
new file mode 100644 (file)
index 0000000..3a3967e
--- /dev/null
@@ -0,0 +1,1131 @@
+{
+       "type": "kafka",
+       "dataSchema": {
+               "dataSource": "HW-aggs",
+               "parser": {
+                       "type": "string",
+                       "parseSpec": {
+                               "format": "json",
+                               "flattenSpec": {
+                                       "useFieldDiscovery": false,
+                                       "fields": [ 
+                                               {
+                                                       "type": "path",
+                                                       "name": "memoryUsageArray.memorySlabUnrecl",
+                                                       "expr": "$.event.measurementsForVfScalingFields.memoryUsageArray_average.memorySlabUnrecl"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "memoryUsageArray.memorySlabRecl",
+                                                       "expr": "$.event.measurementsForVfScalingFields.memoryUsageArray_average.memorySlabRecl"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "memoryUsageArray.memoryCached",
+                                                       "expr": "$.event.measurementsForVfScalingFields.memoryUsageArray_average.memoryCached"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "memoryUsageArray.memoryBuffered",
+                                                       "expr": "$.event.measurementsForVfScalingFields.memoryUsageArray_average.memoryBuffered"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "memoryUsageArray.memoryUsed",
+                                                       "expr": "$.event.measurementsForVfScalingFields.memoryUsageArray_average.memoryUsed"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "memoryUsageArray.memoryFree",
+                                                       "expr": "$.event.measurementsForVfScalingFields.memoryUsageArray_average.memoryFree"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "measurementsForVfScalingVersion",
+                                                       "expr": "$.event.measurementsForVfScalingFields.measurementsForVfScalingVersion"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_count",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_count"
+                                               }, 
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_min.cpuUsageSystem",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_min.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_min.percentUsage",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_min.percentUsage"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_min.cpuWait",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_min.cpuWait"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_min.cpuIdentifier",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_min.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_min.cpuUsageUser",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_min.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_min.cpuIdle",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_min.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_count",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_count"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskIoTimeLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskOctetsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskMergedReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskOpsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskTimeWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskTimeReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskOctetsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskOpsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_average.diskMergedWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_average.diskMergedWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_sum.cpuUsageSystem",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_sum.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_sum.percentUsage",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_sum.percentUsage"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_sum.cpuWait",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_sum.cpuWait"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_sum.cpuIdentifier",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_sum.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_sum.cpuUsageUser",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_sum.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_sum.cpuIdle",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_sum.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.receivedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.transmittedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.transmittedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.receivedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.transmittedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.receivedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.receivedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_max.transmittedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_max.transmittedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_count",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_count"
+                                               }, 
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskIoTimeLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskOctetsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskMergedReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskOpsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskTimeWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskTimeReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskOctetsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskOpsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_max.diskMergedWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_max.diskMergedWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "measurementInterval",
+                                                       "expr": "$.event.measurementsForVfScalingFields.measurementInterval"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_average.cpuUsageSystem",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_average.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_average.percentUsage",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_average.percentUsage"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_average.cpuWait",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_average.cpuWait"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_average.cpuIdentifier",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_average.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_average.cpuUsageUser",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_average.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_average.cpuIdle",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_average.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.receivedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.transmittedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.transmittedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.receivedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.transmittedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.receivedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.receivedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_sum.transmittedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_sum.transmittedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.receivedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.transmittedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.transmittedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.receivedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.transmittedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.receivedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.receivedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_average.transmittedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_average.transmittedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskIoTimeLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskOctetsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskMergedReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskOpsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskTimeWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskTimeReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskOctetsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskOpsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_min.diskMergedWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_min.diskMergedWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.receivedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.transmittedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.transmittedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.receivedErrorPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.transmittedOctetsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.receivedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.receivedTotalPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "vNicPerformanceArray_min.transmittedDiscardedPacketsAccumulated",
+                                                       "expr": "$.event.measurementsForVfScalingFields.vNicPerformanceArray_min.transmittedDiscardedPacketsAccumulated"
+                                               }, 
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_max.cpuUsageSystem",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_max.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_max.percentUsage",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_max.percentUsage"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_max.cpuWait",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_max.cpuWait"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_max.cpuIdentifier",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_max.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_max.cpuUsageUser",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_max.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "cpuUsageArray_max.cpuIdle",
+                                                       "expr": "$.event.measurementsForVfScalingFields.cpuUsageArray_max.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskIoTimeLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskOctetsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskMergedReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskOpsWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskTimeWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskTimeReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskOctetsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskOpsReadLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "diskUsageArray_sum.diskMergedWriteLast",
+                                                       "expr": "$.event.measurementsForVfScalingFields.diskUsageArray_sum.diskMergedWriteLast"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "datalake_ts_",
+                                                       "expr": "$.datalake_ts_"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "eventId",
+                                                       "expr": "$.event.commonEventHeader.eventId"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "reportingEntityId",
+                                                       "expr": "$.event.commonEventHeader.reportingEntityId"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "lastEpochMicrosec",
+                                                       "expr": "$.event.commonEventHeader.lastEpochMicrosec"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "reportingEntityName",
+                                                       "expr": "$.event.commonEventHeader.reportingEntityName"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "sourceId",
+                                                       "expr": "$.event.commonEventHeader.sourceId"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "eventType",
+                                                       "expr": "$.event.commonEventHeader.eventType"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "sequence",
+                                                       "expr": "$.event.commonEventHeader.sequence"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "collectorTimeStamp",
+                                                       "expr": "$.event.commonEventHeader.internalHeaderFields.collectorTimeStamp"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "priority",
+                                                       "expr": "$.event.commonEventHeader.priority"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "sourceName",
+                                                       "expr": "$.event.commonEventHeader.sourceName"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "eventName",
+                                                       "expr": "$.event.commonEventHeader.eventName"
+                                               },
+                                               {
+                                                       "type": "path",
+                                                       "name": "startEpochMicrosec",
+                                                       "expr": "$.event.commonEventHeader.startEpochMicrosec"
+                                               }
+                                       ]
+                               },
+                               "timestampSpec": {
+                                       "column": "collectorTimeStamp",
+                                       "format": "EEE, MM dd yyyy HH:mm:ss z"
+                               },
+                               "dimensionsSpec": {
+                                       "dimensions": [],
+                                       "dimensionsExclusions": []
+                               }
+                       }
+               },
+               "metricsSpec": [{
+                                                       "type": "doubleSum",
+                                                       "name": "memoryUsageArray.memorySlabUnrecl",
+                                                       "fieldName": "memoryUsageArray.memorySlabUnrecl"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "memoryUsageArray.memorySlabRecl",
+                                                       "fieldName": "memoryUsageArray.memorySlabRecl"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "memoryUsageArray.memoryCached",
+                                                       "fieldName": "memoryUsageArray.memoryCached"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "memoryUsageArray.memoryBuffered",
+                                                       "fieldName": "memoryUsageArray.memoryBuffered"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "memoryUsageArray.memoryUsed",
+                                                       "fieldName": "memoryUsageArray.memoryUsed"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "memoryUsageArray.memoryFree",
+                                                       "fieldName": "memoryUsageArray.memoryFree"
+                                               }, 
+                                               {
+                                                       "type": "longSum",
+                                                       "name": "cpuUsageArray_count",
+                                                       "fieldName": "cpuUsageArray_count"
+                                               }, 
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "cpuUsageArray_min.cpuUsageSystem",
+                                                       "fieldName": "cpuUsageArray_min.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "cpuUsageArray_min.percentUsage",
+                                                       "fieldName": "cpuUsageArray_min.percentUsage"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "cpuUsageArray_min.cpuWait",
+                                                       "fieldName": "cpuUsageArray_min.cpuWait"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "cpuUsageArray_min.cpuIdentifier",
+                                                       "fieldName": "cpuUsageArray_min.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "cpuUsageArray_min.cpuUsageUser",
+                                                       "fieldName": "cpuUsageArray_min.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "cpuUsageArray_min.cpuIdle",
+                                                       "fieldName": "cpuUsageArray_min.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "longSum",
+                                                       "name": "vNicPerformanceArray_count",
+                                                       "fieldName": "vNicPerformanceArray_count"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskIoTimeLast",
+                                                       "fieldName": "diskUsageArray_average.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskOctetsWriteLast",
+                                                       "fieldName": "diskUsageArray_average.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskMergedReadLast",
+                                                       "fieldName": "diskUsageArray_average.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskOpsWriteLast",
+                                                       "fieldName": "diskUsageArray_average.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskTimeWriteLast",
+                                                       "fieldName": "diskUsageArray_average.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskTimeReadLast",
+                                                       "fieldName": "diskUsageArray_average.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskOctetsReadLast",
+                                                       "fieldName": "diskUsageArray_average.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskOpsReadLast",
+                                                       "fieldName": "diskUsageArray_average.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_average.diskMergedWriteLast",
+                                                       "fieldName": "diskUsageArray_average.diskMergedWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_sum.cpuUsageSystem",
+                                                       "fieldName": "cpuUsageArray_sum.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_sum.percentUsage",
+                                                       "fieldName": "cpuUsageArray_sum.percentUsage"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_sum.cpuWait",
+                                                       "fieldName": "cpuUsageArray_sum.cpuWait"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_sum.cpuIdentifier",
+                                                       "fieldName": "cpuUsageArray_sum.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_sum.cpuUsageUser",
+                                                       "fieldName": "cpuUsageArray_sum.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_sum.cpuIdle",
+                                                       "fieldName": "cpuUsageArray_sum.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.receivedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.transmittedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.transmittedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.receivedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.transmittedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.receivedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.receivedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "vNicPerformanceArray_max.transmittedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_max.transmittedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "longSum",
+                                                       "name": "diskUsageArray_count",
+                                                       "fieldName": "diskUsageArray_count"
+                                               }, 
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskIoTimeLast",
+                                                       "fieldName": "diskUsageArray_max.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskOctetsWriteLast",
+                                                       "fieldName": "diskUsageArray_max.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskMergedReadLast",
+                                                       "fieldName": "diskUsageArray_max.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskOpsWriteLast",
+                                                       "fieldName": "diskUsageArray_max.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskTimeWriteLast",
+                                                       "fieldName": "diskUsageArray_max.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskTimeReadLast",
+                                                       "fieldName": "diskUsageArray_max.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskOctetsReadLast",
+                                                       "fieldName": "diskUsageArray_max.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskOpsReadLast",
+                                                       "fieldName": "diskUsageArray_max.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "diskUsageArray_max.diskMergedWriteLast",
+                                                       "fieldName": "diskUsageArray_max.diskMergedWriteLast"
+                                               }, 
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_average.cpuUsageSystem",
+                                                       "fieldName": "cpuUsageArray_average.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_average.percentUsage",
+                                                       "fieldName": "cpuUsageArray_average.percentUsage"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_average.cpuWait",
+                                                       "fieldName": "cpuUsageArray_average.cpuWait"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_average.cpuIdentifier",
+                                                       "fieldName": "cpuUsageArray_average.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_average.cpuUsageUser",
+                                                       "fieldName": "cpuUsageArray_average.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "cpuUsageArray_average.cpuIdle",
+                                                       "fieldName": "cpuUsageArray_average.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.receivedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.transmittedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.transmittedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.receivedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.transmittedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.receivedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.receivedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_sum.transmittedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_sum.transmittedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.receivedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.transmittedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.transmittedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.receivedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.transmittedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.receivedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.receivedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_average.transmittedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_average.transmittedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskIoTimeLast",
+                                                       "fieldName": "diskUsageArray_min.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskOctetsWriteLast",
+                                                       "fieldName": "diskUsageArray_min.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskMergedReadLast",
+                                                       "fieldName": "diskUsageArray_min.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskOpsWriteLast",
+                                                       "fieldName": "diskUsageArray_min.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskTimeWriteLast",
+                                                       "fieldName": "diskUsageArray_min.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskTimeReadLast",
+                                                       "fieldName": "diskUsageArray_min.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskOctetsReadLast",
+                                                       "fieldName": "diskUsageArray_min.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskOpsReadLast",
+                                                       "fieldName": "diskUsageArray_min.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "diskUsageArray_min.diskMergedWriteLast",
+                                                       "fieldName": "diskUsageArray_min.diskMergedWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.receivedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.receivedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.transmittedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.transmittedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.transmittedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.transmittedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.receivedErrorPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.receivedErrorPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "vNicPerformanceArray_min.transmittedOctetsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.transmittedOctetsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.receivedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.receivedDiscardedPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.receivedTotalPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.receivedTotalPacketsAccumulated"
+                                               },
+                                               {
+                                                       "type": "doubleMin",
+                                                       "name": "vNicPerformanceArray_min.transmittedDiscardedPacketsAccumulated",
+                                                       "fieldName": "vNicPerformanceArray_min.transmittedDiscardedPacketsAccumulated"
+                                               }, 
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "cpuUsageArray_max.cpuUsageSystem",
+                                                       "fieldName": "cpuUsageArray_max.cpuUsageSystem"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "cpuUsageArray_max.percentUsage",
+                                                       "fieldName": "cpuUsageArray_max.percentUsage"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "cpuUsageArray_max.cpuWait",
+                                                       "fieldName": "cpuUsageArray_max.cpuWait"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "cpuUsageArray_max.cpuIdentifier",
+                                                       "fieldName": "cpuUsageArray_max.cpuIdentifier"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "cpuUsageArray_max.cpuUsageUser",
+                                                       "fieldName": "cpuUsageArray_max.cpuUsageUser"
+                                               },
+                                               {
+                                                       "type": "doubleMax",
+                                                       "name": "cpuUsageArray_max.cpuIdle",
+                                                       "fieldName": "cpuUsageArray_max.cpuIdle"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskIoTimeLast",
+                                                       "fieldName": "diskUsageArray_sum.diskIoTimeLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskOctetsWriteLast",
+                                                       "fieldName": "diskUsageArray_sum.diskOctetsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskMergedReadLast",
+                                                       "fieldName": "diskUsageArray_sum.diskMergedReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskOpsWriteLast",
+                                                       "fieldName": "diskUsageArray_sum.diskOpsWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskTimeWriteLast",
+                                                       "fieldName": "diskUsageArray_sum.diskTimeWriteLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskTimeReadLast",
+                                                       "fieldName": "diskUsageArray_sum.diskTimeReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskOctetsReadLast",
+                                                       "fieldName": "diskUsageArray_sum.diskOctetsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskOpsReadLast",
+                                                       "fieldName": "diskUsageArray_sum.diskOpsReadLast"
+                                               },
+                                               {
+                                                       "type": "doubleSum",
+                                                       "name": "diskUsageArray_sum.diskMergedWriteLast",
+                                                       "fieldName": "diskUsageArray_sum.diskMergedWriteLast"
+                                               }],
+               "granularitySpec": {
+                       "type": "uniform",
+                       "segmentGranularity": "HOUR",
+                       "queryGranularity": "MINUTE",
+                       "rollup": true
+               }
+       },
+       "tuningConfig": {
+               "type": "kafka",
+               "reportParseExceptions": true
+       },
+       "ioConfig": {
+               "topic": "HW-aggs",
+               "replicas": 1,
+               "startDelay": "PT1S",
+               "taskDuration": "PT1H",
+               "completionTimeout": "PT30M",
+               "consumerProperties": {
+                       "bootstrap.servers": "message-router-kafka:9092"
+               },
+               "useEarliestOffset": true
+       }
+}
diff --git a/components/datalake-handler/feeder/src/main/resources/druid/HW-aggs-sample-format.json b/components/datalake-handler/feeder/src/main/resources/druid/HW-aggs-sample-format.json
new file mode 100644 (file)
index 0000000..638f721
--- /dev/null
@@ -0,0 +1,179 @@
+{\r
+       "event": {\r
+               "commonEventHeader": {\r
+                       "sourceId": "compute-2",\r
+                       "startEpochMicrosec": 1558850006,\r
+                       "eventId": "17252",\r
+                       "reportingEntityId": "localhost",\r
+                       "internalHeaderFields": {\r
+                               "collectorTimeStamp": "Sun, 05 26 2019 05:53:36 UTC"\r
+                       },\r
+                       "eventType": "platform",\r
+                       "priority": "Normal",\r
+                       "version": 2,\r
+                       "reportingEntityName": "localhost",\r
+                       "sequence": 0,\r
+                       "domain": "measurementsForVfScaling",\r
+                       "lastEpochMicrosec": 1558850016,\r
+                       "eventName": "",\r
+                       "sourceName": "compute-2"\r
+               },\r
+               "measurementsForVfScalingFields": {\r
+                       "memoryUsageArray_sum": {\r
+                               "memorySlabUnrecl": 634188,\r
+                               "memorySlabRecl": 486928,\r
+                               "memoryCached": 7635224,\r
+                               "memoryBuffered": 2100,\r
+                               "memoryUsed": 4.6876648E7,\r
+                               "memoryFree": 3.47144859648E11\r
+                       },\r
+                       "memoryUsageArray_average": {\r
+                               "memorySlabUnrecl": 634188,\r
+                               "memorySlabRecl": 486928,\r
+                               "memoryCached": 7635224,\r
+                               "memoryBuffered": 2100,\r
+                               "memoryUsed": 4.6876648E7,\r
+                               "memoryFree": 3.47144859648E11\r
+                       },\r
+                       "measurementsForVfScalingVersion": 2,\r
+                       "cpuUsageArray_count": 88,\r
+                       "memoryUsageArray_min": {\r
+                               "memorySlabUnrecl": 634188,\r
+                               "memorySlabRecl": 486928,\r
+                               "memoryCached": 7635224,\r
+                               "memoryBuffered": 2100,\r
+                               "memoryUsed": 4.6876648E7,\r
+                               "memoryFree": 3.47144859648E11\r
+                       },\r
+                       "cpuUsageArray_min": {\r
+                               "cpuUsageSystem": 0,\r
+                               "percentUsage": 0,\r
+                               "cpuWait": 0,\r
+                               "cpuIdentifier": 0,\r
+                               "cpuUsageUser": 0,\r
+                               "cpuIdle": 0\r
+                       },\r
+                       "vNicPerformanceArray_count": 51, \r
+                       "diskUsageArray_average": {\r
+                               "diskIoTimeLast": 6.633306574873333,\r
+                               "diskOctetsWriteLast": 1801687.4324733336,\r
+                               "diskMergedReadLast": 0,\r
+                               "diskOpsWriteLast": 336.9987602213334,\r
+                               "diskTimeWriteLast": 1.4666613132266668,\r
+                               "diskTimeReadLast": 0,\r
+                               "diskOctetsReadLast": 0,\r
+                               "diskOpsReadLast": 0,\r
+                               "diskMergedWriteLast": 15.266605225866664\r
+                       },\r
+                       "cpuUsageArray_sum": {\r
+                               "cpuUsageSystem": 31.499984875965808,\r
+                               "percentUsage": 0,\r
+                               "cpuWait": 0.0999987854677,\r
+                               "cpuIdentifier": 3828,\r
+                               "cpuUsageUser": 464.402795424227,\r
+                               "cpuIdle": 31.499984875965808\r
+                       },\r
+                       "vNicPerformanceArray_max": {\r
+                               "receivedOctetsAccumulated": 7420055.60296,\r
+                               "transmittedErrorPacketsAccumulated": 4.9E-324,\r
+                               "transmittedTotalPacketsAccumulated": 3891.02709722,\r
+                               "receivedErrorPacketsAccumulated": 4.9E-324,\r
+                               "transmittedOctetsAccumulated": 7030576.37061,\r
+                               "receivedDiscardedPacketsAccumulated": 0.200002362849,\r
+                               "receivedTotalPacketsAccumulated": 5127.16747474,\r
+                               "transmittedDiscardedPacketsAccumulated": 0.200000846185\r
+                       },\r
+                       "diskUsageArray_count": 3,\r
+                       "memoryUsageArray_max": {\r
+                               "memorySlabUnrecl": 634188,\r
+                               "memorySlabRecl": 486928,\r
+                               "memoryCached": 7635224,\r
+                               "memoryBuffered": 2100,\r
+                               "memoryUsed": 4.6876648E7,\r
+                               "memoryFree": 3.47144859648E11\r
+                       }, \r
+                       "diskUsageArray_max": {\r
+                               "diskIoTimeLast": 9.99995970463,\r
+                               "diskOctetsWriteLast": 2702532.45109,\r
+                               "diskMergedReadLast": 4.9E-324,\r
+                               "diskOpsWriteLast": 506.098328814,\r
+                               "diskTimeWriteLast": 2.19999284748,\r
+                               "diskTimeReadLast": 4.9E-324,\r
+                               "diskOctetsReadLast": 4.9E-324,\r
+                               "diskOpsReadLast": 4.9E-324,\r
+                               "diskMergedWriteLast": 22.8999080073\r
+                       },\r
+                       "measurementInterval": 10, \r
+                       "cpuUsageArray_average": {\r
+                               "cpuUsageSystem": 0.35795437359052057,\r
+                               "percentUsage": 0,\r
+                               "cpuWait": 0.0011363498348602272,\r
+                               "cpuIdentifier": 43.5,\r
+                               "cpuUsageUser": 5.277304493457125,\r
+                               "cpuIdle": 0.35795437359052057\r
+                       },\r
+                       "vNicPerformanceArray_sum": {\r
+                               "receivedOctetsAccumulated": 2.497791331284713E7,\r
+                               "transmittedErrorPacketsAccumulated": 0,\r
+                               "transmittedTotalPacketsAccumulated": 5657.180629766652,\r
+                               "receivedErrorPacketsAccumulated": 0,\r
+                               "transmittedOctetsAccumulated": 2.497472905660059E7,\r
+                               "receivedDiscardedPacketsAccumulated": 0.200002362849,\r
+                               "receivedTotalPacketsAccumulated": 6834.521263097156,\r
+                               "transmittedDiscardedPacketsAccumulated": 0.200000846185\r
+                       },\r
+                       "vNicPerformanceArray_average": {\r
+                               "receivedOctetsAccumulated": 489763.0061342575,\r
+                               "transmittedErrorPacketsAccumulated": 0,\r
+                               "transmittedTotalPacketsAccumulated": 110.92511038758141,\r
+                               "receivedErrorPacketsAccumulated": 0,\r
+                               "transmittedOctetsAccumulated": 489700.56973726646,\r
+                               "receivedDiscardedPacketsAccumulated": 0.003921614957823529,\r
+                               "receivedTotalPacketsAccumulated": 134.01022084504228,\r
+                               "transmittedDiscardedPacketsAccumulated": 0.003921585219313725\r
+                       },\r
+                       "diskUsageArray_min": {\r
+                               "diskIoTimeLast": 0,\r
+                               "diskOctetsWriteLast": 0,\r
+                               "diskMergedReadLast": 0,\r
+                               "diskOpsWriteLast": 0,\r
+                               "diskTimeWriteLast": 0,\r
+                               "diskTimeReadLast": 0,\r
+                               "diskOctetsReadLast": 0,\r
+                               "diskOpsReadLast": 0,\r
+                               "diskMergedWriteLast": 0\r
+                       },\r
+                       "vNicPerformanceArray_min": {\r
+                               "receivedOctetsAccumulated": 0,\r
+                               "transmittedErrorPacketsAccumulated": 0,\r
+                               "transmittedTotalPacketsAccumulated": 0,\r
+                               "receivedErrorPacketsAccumulated": 0,\r
+                               "transmittedOctetsAccumulated": 0,\r
+                               "receivedDiscardedPacketsAccumulated": 0,\r
+                               "receivedTotalPacketsAccumulated": 0,\r
+                               "transmittedDiscardedPacketsAccumulated": 0\r
+                       },\r
+                       "memoryUsageArray_count": 1,\r
+                       "cpuUsageArray_max": {\r
+                               "cpuUsageSystem": 1.90000067932,\r
+                               "percentUsage": 4.9E-324,\r
+                               "cpuWait": 0.0999987854677,\r
+                               "cpuIdentifier": 87,\r
+                               "cpuUsageUser": 98.1005644149,\r
+                               "cpuIdle": 1.90000067932\r
+                       },\r
+                       "diskUsageArray_sum": {\r
+                               "diskIoTimeLast": 19.89991972462,\r
+                               "diskOctetsWriteLast": 5405062.297420001,\r
+                               "diskMergedReadLast": 0,\r
+                               "diskOpsWriteLast": 1010.9962806640001,\r
+                               "diskTimeWriteLast": 4.39998393968,\r
+                               "diskTimeReadLast": 0,\r
+                               "diskOctetsReadLast": 0,\r
+                               "diskOpsReadLast": 0,\r
+                               "diskMergedWriteLast": 45.799815677599995\r
+                       }\r
+               }\r
+       },\r
+       "datalake_ts_": 1560149458487\r
+}\r