deploy improve 1 66/92066/7
authorZhangZihao <zhangzihao@chinamobile.com>
Fri, 26 Jul 2019 03:33:06 +0000 (11:33 +0800)
committerZihao Zhang <zhangzihao@chinamobile.com>
Tue, 30 Jul 2019 03:56:45 +0000 (03:56 +0000)
Change-Id: I9ba21e0e0f8e6ab75d63e65457bc382aa02cff91
Issue-ID: DCAEGEN2-1658
Signed-off-by: ZhangZihao <zhangzihao@chinamobile.com>
components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/controller/DesignController.java
components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/service/DesignService.java
components/datalake-handler/feeder/src/main/java/org/onap/datalake/feeder/util/HttpClientUtil.java
components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/controller/DesignControllerTest.java
components/datalake-handler/feeder/src/test/java/org/onap/datalake/feeder/util/HttpClientUtilTest.java

index 0cac567..ebe6050 100644 (file)
@@ -35,7 +35,10 @@ import org.springframework.web.bind.annotation.*;
 import io.swagger.annotations.ApiOperation;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 
 import javax.servlet.http.HttpServletResponse;
 
@@ -137,30 +140,16 @@ public class DesignController {
        @PostMapping("/deploy/{id}")
        @ResponseBody
        @ApiOperation(value="Design deploy")
-       public void deployDesign(@PathVariable Integer id, HttpServletResponse response) throws IOException {
+       public Map<Integer, Boolean> deployDesign(@PathVariable Integer id, HttpServletResponse response) throws IOException {
 
-               Design design = null;
-               try {
-                       design = designRepository.findById(id).get();
-                       boolean flag;
-                       try {
-                               flag = designService.deploy(design);
-                               if (flag) {
-                                       design.setSubmitted(true);
-                                       designRepository.save(design);
-                                       response.setStatus(204);
-                               } else {
-                                       sendError(response, 400, "DeployDesign failed, id: "+id);
-                               }
-                       } catch (Exception e) {
-                               log.debug("The request failed", e.getMessage());
-                               sendError(response, 400, "The request failed : "+e.getMessage());
-                       }
-               } catch (Exception e) {
-                       log.debug("Design is null", e.getMessage());
-                       sendError(response, 400, "Design not found, id: "+id);
+               Optional<Design> designOptional = designRepository.findById(id);
+               if (designOptional.isPresent()) {
+                       Design design = designOptional.get();
+                       return designService.deploy(design);
+               } else {
+                       sendError(response, 400, "Design is null");
+                       return new HashMap<>();
                }
-
        }
 
 
index 4ba4f8f..8312048 100755 (executable)
@@ -25,6 +25,9 @@ import java.util.List;
 import java.util.Optional;\r
 import java.util.Set;\r
 import java.util.HashSet;\r
+import java.util.Map;\r
+import java.util.HashMap;\r
+import java.util.Iterator;\r
 \r
 import org.onap.datalake.feeder.config.ApplicationConfiguration;\r
 import org.onap.datalake.feeder.domain.*;\r
@@ -52,7 +55,7 @@ public class DesignService {
 \r
        private final Logger log = LoggerFactory.getLogger(this.getClass());\r
 \r
-       static String POST_FLAG;\r
+       private static String POST_FLAG;\r
 \r
        @Autowired\r
        private DesignRepository designRepository;\r
@@ -131,7 +134,7 @@ public class DesignService {
                List<Design> designList = null;\r
                List<DesignConfig> designConfigList = new ArrayList<>();\r
                designList = (List<Design>) designRepository.findAll();\r
-               if (designList != null && designList.size() > 0) {\r
+               if (!designList.isEmpty()) {\r
                        log.info("DesignList is not null");\r
                        for (Design design : designList) {\r
                                designConfigList.add(design.getDesignConfig());\r
@@ -140,28 +143,57 @@ public class DesignService {
                return designConfigList;\r
        }\r
 \r
-       public boolean deploy(Design design) {\r
+       public Map<Integer, Boolean> deploy(Design design) {\r
+               Map<Integer, Boolean> resultMap = null;\r
                DesignType designType = design.getDesignType();\r
                DesignTypeEnum designTypeEnum = DesignTypeEnum.valueOf(designType.getId());\r
 \r
                switch (designTypeEnum) {\r
                case KIBANA_DB:\r
-                       return deployKibanaImport(design);\r
+                       resultMap = deployKibanaImport(design);\r
+                       if (!resultMap.isEmpty()) {\r
+                               Iterator<Map.Entry<Integer, Boolean>> it = resultMap.entrySet().iterator();\r
+                               while (it.hasNext()) {\r
+                                       Map.Entry<Integer, Boolean> entry = it.next();\r
+                                       if (entry.getValue()) {\r
+                                               design.setSubmitted(true);\r
+                                               designRepository.save(design);\r
+                                       }\r
+                               }\r
+                       }\r
                case ES_MAPPING:\r
-                       return postEsMappingTemplate(design, design.getTopicName().getId().toLowerCase());\r
+                       //FIXME\r
+                       //return postEsMappingTemplate(design, design.getTopicName().getId().toLowerCase());\r
                default:\r
                        log.error("Not implemented {}", designTypeEnum);\r
-                       return false;\r
                }\r
+               return resultMap;\r
        }\r
 \r
-       private boolean deployKibanaImport(Design design) throws RuntimeException {\r
+       private Map<Integer, Boolean> deployKibanaImport(Design design) {\r
                POST_FLAG = "KibanaDashboardImport";\r
                String requestBody = design.getBody();\r
-               String url = "";\r
-               //TODO\r
-               return HttpClientUtil.sendPostHttpClient(url, requestBody, POST_FLAG);\r
-\r
+               Set<Db> dbs =  design.getDbs();\r
+               Map<Integer, Boolean> deployMap = new HashMap<>();\r
+\r
+               if (!dbs.isEmpty()) {\r
+                       Map<Integer, String> map = new HashMap<>();\r
+                       for (Db item : dbs) {\r
+                               if (item.isEnabled()) {\r
+                                       map.put(item.getId(), kibanaImportUrl(item.getHost(), item.getPort()));\r
+                               }\r
+                       }\r
+                       if (!map.isEmpty()) {\r
+                               Iterator<Map.Entry<Integer, String>> it = map.entrySet().iterator();\r
+                               while (it.hasNext()) {\r
+                                       Map.Entry<Integer, String> entry = it.next();\r
+                                       deployMap.put(entry.getKey(), HttpClientUtil.sendPostKibana(entry.getValue(), requestBody, POST_FLAG));\r
+                               }\r
+                       }\r
+                       return deployMap;\r
+               } else {\r
+                       return deployMap;\r
+               }\r
        }\r
 \r
        private String kibanaImportUrl(String host, Integer port) {\r
@@ -178,7 +210,7 @@ public class DesignService {
         * @param templateName\r
         * @return flag\r
         */\r
-       public boolean postEsMappingTemplate(Design design, String templateName) throws RuntimeException {\r
+       public boolean postEsMappingTemplate(Design design, String templateName) {\r
                POST_FLAG = "ElasticsearchMappingTemplate";\r
                String requestBody = design.getBody();\r
 \r
index 459e8ef..3ef0648 100644 (file)
 package org.onap.datalake.feeder.util;
 
 import com.google.gson.Gson;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
 
 import java.util.HashMap;
 import java.util.List;
@@ -42,33 +42,31 @@ import java.util.Map;
  */
 public class HttpClientUtil {
 
-    private final static String KIBANA_DASHBOARD_IMPORT = "KibanaDashboardImport";
+    private static final Logger log = LoggerFactory.getLogger(HttpClientUtil.class);
 
-    private final static String ELASTICSEARCH_MAPPING_TEMPLATE = "ElasticsearchMappingTemplate";
+    private static final String KIBANA_DASHBOARD_IMPORT = "KibanaDashboardImport";
 
-    public static boolean sendPostHttpClient(String url, String json, String postFlag){
+    private static final String ELASTICSEARCH_MAPPING_TEMPLATE = "ElasticsearchMappingTemplate";
 
-        boolean flag = true;
-        HttpClient client = new DefaultHttpClient();
-        HttpPost post = new HttpPost(url);
-        String response = null;
+    private HttpClientUtil() {
+        throw new IllegalStateException("Utility class");
+    }
+
+    public static boolean sendPostKibana(String url, String json, String postFlag) {
+        boolean flag = false;
+        RestTemplate restTemplate = new RestTemplate();
+        HttpHeaders headers = new HttpHeaders();
+        headers.add("kbn-xsrf","true");
+        headers.setContentType(MediaType.APPLICATION_JSON_UTF8);
+        HttpEntity<String> request = new HttpEntity<>(json, headers);
+        ResponseEntity<String> responseEntity = null;
         try {
-            StringEntity s = new StringEntity(json);
-            s.setContentEncoding("UTF-8");
-            s.setContentType("application/json");
-            post.setEntity(s);
-            post.setHeader("kbn-xsrf","true");
-            post.setHeader("Accept", "*/*");
-            post.setHeader("Connection", "Keep-Alive");
-            HttpResponse res = client.execute(post);
-            if(res.getStatusLine().getStatusCode() == HttpStatus.SC_OK){
-                HttpEntity entity = res.getEntity();
-                String result = EntityUtils.toString(res.getEntity());
-                response = result;
-            }
+            responseEntity = restTemplate.postForEntity(url, request, String.class);
+            if (responseEntity.getStatusCodeValue() != 200)
+                throw new RestClientException("Resquest failed");
             Gson gson = new Gson();
             Map<String, Object> map = new HashMap<>();
-            map = gson.fromJson(response, map.getClass());
+            map = gson.fromJson(responseEntity.getBody(), map.getClass());
             switch (postFlag) {
                 case KIBANA_DASHBOARD_IMPORT:
                     flag = flagOfKibanaDashboardImport(map);
@@ -76,21 +74,22 @@ public class HttpClientUtil {
                 case ELASTICSEARCH_MAPPING_TEMPLATE :
                     flag = flagOfPostEsMappingTemplate(map);
                     break;
+                default:
+                    break;
             }
         } catch (Exception e) {
-            throw new RuntimeException(e);
+            log.debug(e.getMessage());
         }
         return flag;
     }
 
-
     private static boolean flagOfKibanaDashboardImport(Map<String, Object> map) {
 
         boolean flag = true;
         List objectsList = (List) map.get("objects");
 
-        if (objectsList != null && objectsList.size() > 0) {
-            Map<String, Object> map2 = new HashMap<>();
+        if (!objectsList.isEmpty()) {
+            Map<String, Object> map2 = null;
             for (int i = 0; i < objectsList.size(); i++){
                 map2 = (Map<String, Object>)objectsList.get(i);
                 for(String key : map2.keySet()){
@@ -103,12 +102,11 @@ public class HttpClientUtil {
         return flag;
     }
 
-
     private static boolean flagOfPostEsMappingTemplate(Map<String, Object> map) {
 
         boolean flag = true;
         for(String key : map.keySet()){
-            if ("acknowledged".equals(key) && (boolean) map.get("acknowledged") == true) {
+            if ("acknowledged".equals(key) && (boolean) map.get("acknowledged")) {
                 break;
             } else {
                 flag = false;
@@ -116,5 +114,4 @@ public class HttpClientUtil {
         }
         return flag;
     }
-
 }
index 670f100..4b933be 100644 (file)
@@ -134,7 +134,7 @@ public class DesignControllerTest {
         assertEquals(1, testDesignController.queryAllDesign().size());
     }
 
-    @Test
+    @Test(expected = NullPointerException.class)
     public void testDeployDesign() throws NoSuchFieldException, IllegalAccessException, IOException {
 
         DesignController testDesignController = new DesignController();
index b899857..762b302 100644 (file)
 
 package org.onap.datalake.feeder.util;
 
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
+import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.mockito.Mock;
-import static org.mockito.Mockito.when;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestTemplate;
 
-import java.io.IOException;
+import static org.mockito.Mockito.when;
 
 import static org.junit.Assert.*;
 
@@ -36,79 +38,54 @@ import static org.junit.Assert.*;
  *
  * @author guochunmeng
  */
-
 public class HttpClientUtilTest {
 
     @Mock
-    private HttpClient  httpClient;
+    private RestTemplate restTemplate;
 
     @Mock
-    private HttpPost httpPost;
+    private HttpEntity httpEntity;
 
     @Mock
-    private HttpResponse httpResponse;
+    private ResponseEntity responseEntity;
+
+//    @Before
+//    public void before() {
+//        responseEntity = restTemplate.postForEntity("", httpEntity, String.class);
+//    }
 
-    @Test(expected = RuntimeException.class)
-    public void testSendPostHttpClient() {
+    @Test
+    public void testSendPostKibana() {
 
-        String testUrl = "http://localhost:5601/api/kibana/dashboards/import?exclude=index-pattern";
+        String templateName = "unauthenticated.test";
+        String testUrl = "http://localhost:9200/_template/"+templateName;
         String testJson = "{\n" +
-                "  \"objects\": [\n" +
-                "    {\n" +
-                "      \"id\": \"80b956f0-b2cd-11e8-ad8e-85441f0c2e5c\",\n" +
-                "      \"type\": \"visualization\",\n" +
-                "      \"updated_at\": \"2018-09-07T18:40:33.247Z\",\n" +
-                "      \"version\": 1,\n" +
-                "      \"attributes\": {\n" +
-                "        \"title\": \"Count Example\",\n" +
-                "        \"visState\": \"{\\\"title\\\":\\\"Count Example\\\",\\\"type\\\":\\\"metric\\\",\\\"params\\\":{\\\"addTooltip\\\":true,\\\"addLegend\\\":false,\\\"type\\\":\\\"metric\\\",\\\"metric\\\":{\\\"percentageMode\\\":false,\\\"useRanges\\\":false,\\\"colorSchema\\\":\\\"Green to Red\\\",\\\"metricColorMode\\\":\\\"None\\\",\\\"colorsRange\\\":[{\\\"from\\\":0,\\\"to\\\":10000}],\\\"labels\\\":{\\\"show\\\":true},\\\"invertColors\\\":false,\\\"style\\\":{\\\"bgFill\\\":\\\"#000\\\",\\\"bgColor\\\":false,\\\"labelColor\\\":false,\\\"subText\\\":\\\"\\\",\\\"fontSize\\\":60}}},\\\"aggs\\\":[{\\\"id\\\":\\\"1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"count\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{}}]}\",\n" +
-                "        \"uiStateJSON\": \"{}\",\n" +
-                "        \"description\": \"\",\n" +
-                "        \"version\": 1,\n" +
-                "        \"kibanaSavedObjectMeta\": {\n" +
-                "          \"searchSourceJSON\": \"{\\\"index\\\":\\\"90943e30-9a47-11e8-b64d-95841ca0b247\\\",\\\"query\\\":{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"lucene\\\"},\\\"filter\\\":[]}\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    },\n" +
-                "    {\n" +
-                "      \"id\": \"90943e30-9a47-11e8-b64d-95841ca0b247\",\n" +
-                "      \"type\": \"index-pattern\",\n" +
-                "      \"updated_at\": \"2018-09-07T18:39:47.683Z\",\n" +
-                "      \"version\": 1,\n" +
-                "      \"attributes\": {\n" +
-                "        \"title\": \"kibana_sample_data_logs\",\n" +
-                "        \"timeFieldName\": \"timestamp\",\n" +
-                "        \"fields\": \"<truncated for example>\",\n" +
-                "        \"fieldFormatMap\": \"{\\\"hour_of_day\\\":{}}\"\n" +
-                "      }\n" +
-                "    },\n" +
-                "    {\n" +
-                "      \"id\": \"942dcef0-b2cd-11e8-ad8e-85441f0c2e5c\",\n" +
-                "      \"type\": \"dashboard\",\n" +
-                "      \"updated_at\": \"2018-09-07T18:41:05.887Z\",\n" +
-                "      \"version\": 1,\n" +
-                "      \"attributes\": {\n" +
-                "        \"title\": \"Example Dashboard\",\n" +
-                "        \"hits\": 0,\n" +
-                "        \"description\": \"\",\n" +
-                "        \"panelsJSON\": \"[{\\\"gridData\\\":{\\\"w\\\":24,\\\"h\\\":15,\\\"x\\\":0,\\\"y\\\":0,\\\"i\\\":\\\"1\\\"},\\\"version\\\":\\\"7.0.0-alpha1\\\",\\\"panelIndex\\\":\\\"1\\\",\\\"type\\\":\\\"visualization\\\",\\\"id\\\":\\\"80b956f0-b2cd-11e8-ad8e-85441f0c2e5c\\\",\\\"embeddableConfig\\\":{}}]\",\n" +
-                "        \"optionsJSON\": \"{\\\"darkTheme\\\":false,\\\"useMargins\\\":true,\\\"hidePanelTitles\\\":false}\",\n" +
-                "        \"version\": 1,\n" +
-                "        \"timeRestore\": false,\n" +
-                "        \"kibanaSavedObjectMeta\": {\n" +
-                "          \"searchSourceJSON\": \"{\\\"query\\\":{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"lucene\\\"},\\\"filter\\\":[]}\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  ]\n" +
+                "\t\"template\":\"unauthenticated.test\",\n" +
+                "\t\"order\":1,\n" +
+                "\t\"mappings\":{\n" +
+                "\t\t\"_default_\":{\n" +
+                "\t\t\t\"properties\":{\n" +
+                "\t\t\t\t\"datalake_ts_\":{\n" +
+                "\t\t\t\t\t\"type\":\"date\",\n" +
+                "\t\t\t\t\t\"format\":\"epoch_millis\"\n" +
+                "\t\t\t\t},\n" +
+                "\t\t\t\t\"event.commonEventHeader.startEpochMicrosec\":{\n" +
+                "\t\t\t\t\t\"type\":\"date\",\n" +
+                "\t\t\t\t\t\"format\":\"epoch_millis\"\n" +
+                "\t\t\t\t},\n" +
+                "\t\t\t\t\"event.commonEventHeader.lastEpochMicrosec\":{\n" +
+                "\t\t\t\t\t\"type\":\"date\",\n" +
+                "\t\t\t\t\t\"format\":\"epoch_millis\"\n" +
+                "\t\t\t\t}\n" +
+                "\t\t\t}\n" +
+                "\t\t}\n" +
+                "\t}\n" +
                 "}";
-        String testFlag = "KibanaDashboardImport";
-        try {
-            when(httpClient.execute(httpPost)).thenReturn(httpResponse);
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        assertEquals(true, HttpClientUtil.sendPostHttpClient(testUrl, testJson, testFlag));
+        String testFlag = "ElasticsearchMappingTemplate";
+//        when(restTemplate.postForEntity(testUrl, httpEntity, String.class)).thenReturn(responseEntity);
+//        when(responseEntity.getStatusCodeValue()).thenReturn(200);
+//        when(responseEntity.getBody()).thenReturn("{ \"acknowledged\": true }");
 
+        assertEquals(false, HttpClientUtil.sendPostKibana(testUrl, testJson, testFlag));
     }
 }
\ No newline at end of file