1 /*******************************************************************************
\r
2 * ============LICENSE_START==================================================
\r
4 * * ===========================================================================
\r
5 * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
\r
6 * * ===========================================================================
\r
7 * * Licensed under the Apache License, Version 2.0 (the "License");
\r
8 * * you may not use this file except in compliance with the License.
\r
9 * * You may obtain a copy of the License at
\r
11 * * http://www.apache.org/licenses/LICENSE-2.0
\r
13 * * Unless required by applicable law or agreed to in writing, software
\r
14 * * distributed under the License is distributed on an "AS IS" BASIS,
\r
15 * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\r
16 * * See the License for the specific language governing permissions and
\r
17 * * limitations under the License.
\r
18 * * ============LICENSE_END====================================================
\r
20 * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
\r
22 ******************************************************************************/
\r
24 package org.onap.dmaap.datarouter.provisioning;
\r
26 import static org.onap.dmaap.datarouter.provisioning.utils.HttpServletUtils.sendResponseError;
\r
28 import java.io.IOException;
\r
29 import java.sql.Connection;
\r
30 import java.sql.PreparedStatement;
\r
31 import java.sql.ResultSet;
\r
32 import java.sql.SQLException;
\r
33 import java.text.ParseException;
\r
34 import java.text.SimpleDateFormat;
\r
35 import java.time.LocalDateTime;
\r
36 import java.time.format.DateTimeFormatter;
\r
37 import java.util.Calendar;
\r
38 import java.util.Date;
\r
39 import java.util.HashMap;
\r
40 import java.util.Map;
\r
41 import java.util.TimeZone;
\r
42 import javax.servlet.ServletOutputStream;
\r
43 import javax.servlet.http.HttpServletRequest;
\r
44 import javax.servlet.http.HttpServletResponse;
\r
45 import org.json.JSONException;
\r
46 import org.onap.dmaap.datarouter.provisioning.beans.EventLogRecord;
\r
47 import org.onap.dmaap.datarouter.provisioning.utils.LOGJSONObject;
\r
48 import org.onap.dmaap.datarouter.provisioning.utils.ProvDbUtils;
\r
52 * This Servlet handles requests to the <Statistics API> and <Statistics consilidated
\r
55 * @author Manish Singh
\r
56 * @version $Id: StatisticsServlet.java,v 1.11 2016/08/10 17:27:02 Manish Exp $
\r
58 @SuppressWarnings("serial")
\r
61 public class StatisticsServlet extends BaseServlet {
\r
63 private static final long TWENTYFOUR_HOURS = (24 * 60 * 60 * 1000L);
\r
64 private static final String FMT1 = "yyyy-MM-dd'T'HH:mm:ss'Z'";
\r
65 private static final String FMT2 = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
\r
66 public static final String FEEDID = "FEEDID";
\r
69 private static final String SQL_SELECT_NAME = "SELECT (SELECT NAME FROM FEEDS AS f WHERE f.FEEDID in(";
\r
70 private static final String SQL_FEED_ID = ") and f.FEEDID=e.FEEDID) AS FEEDNAME, e.FEEDID as FEEDID, ";
\r
71 private static final String SQL_SELECT_COUNT = "(SELECT COUNT(*) FROM LOG_RECORDS AS c WHERE c.FEEDID in(";
\r
72 private static final String SQL_TYPE_PUB = ") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS FILES_PUBLISHED,";
\r
73 private static final String SQL_SELECT_SUM = "(SELECT SUM(content_length) FROM LOG_RECORDS AS c WHERE c.FEEDID in(";
\r
74 private static final String SQL_PUBLISH_LENGTH = ") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS PUBLISH_LENGTH, COUNT(e.EVENT_TIME) as FILES_DELIVERED,";
\r
75 private static final String SQL_SUBSCRIBER_URL = " sum(m.content_length) as DELIVERED_LENGTH, SUBSTRING_INDEX(e.REQURI,'/',+3) as SUBSCRIBER_URL,";
\r
76 private static final String SQL_SUB_ID = " e.DELIVERY_SUBID as SUBID, ";
\r
77 private static final String SQL_DELIVERY_TIME = " e.EVENT_TIME AS PUBLISH_TIME, m.EVENT_TIME AS DELIVERY_TIME, ";
\r
78 private static final String SQL_AVERAGE_DELAY = " AVG(e.EVENT_TIME - m.EVENT_TIME)/1000 as AverageDelay FROM LOG_RECORDS";
\r
79 private static final String SQL_JOIN_RECORDS = " e JOIN LOG_RECORDS m ON m.PUBLISH_ID = e.PUBLISH_ID AND e.FEEDID IN (";
\r
80 private static final String SQL_STATUS_204 = " AND m.STATUS=204 AND e.RESULT=204 ";
\r
81 private static final String SQL_GROUP_SUB_ID = " group by SUBID";
\r
82 private static final String JSON_OUTPUT_TYPE = "json";
\r
83 private static final String CSV_OUTPUT_TYPE = "csv";
\r
87 * DELETE a logging URL -- not supported.
\r
90 public void doDelete(HttpServletRequest req, HttpServletResponse resp) {
\r
91 String message = "DELETE not allowed for the logURL.";
\r
92 EventLogRecord elr = new EventLogRecord(req);
\r
93 elr.setMessage(message);
\r
94 elr.setResult(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
\r
95 eventlogger.error(elr.toString());
\r
96 sendResponseError(resp, HttpServletResponse.SC_METHOD_NOT_ALLOWED, message, eventlogger);
\r
100 * GET a Statistics URL -- retrieve Statistics data for a feed or subscription. See the
\r
101 * <b>Statistics API</b> document for details on how this method should be invoked.
\r
104 public void doGet(HttpServletRequest req, HttpServletResponse resp) {
\r
105 Map<String, String> map = buildMapFromRequest(req);
\r
106 if (map.get("err") != null) {
\r
107 sendResponseError(resp, HttpServletResponse.SC_BAD_REQUEST,
\r
108 "Invalid arguments: " + map.get("err"), eventlogger);
\r
111 // check Accept: header??
\r
112 resp.setStatus(HttpServletResponse.SC_OK);
\r
113 resp.setContentType(LOGLIST_CONTENT_TYPE);
\r
114 String outputType = JSON_OUTPUT_TYPE;
\r
115 if (req.getParameter(FEEDID) == null && req.getParameter(GROUPID) == null) {
\r
117 resp.getOutputStream().print("Invalid request, Feedid or Group ID is required.");
\r
118 } catch (IOException ioe) {
\r
119 eventlogger.error("PROV0171 StatisticsServlet.doGet: " + ioe.getMessage(), ioe);
\r
122 if (req.getParameter(FEEDID) != null && req.getParameter(GROUPID) == null) {
\r
123 map.put(FEEDIDS, req.getParameter(FEEDID).replace("|", ","));
\r
125 if (req.getParameter(GROUPID) != null && req.getParameter(FEEDID) == null) {
\r
126 StringBuilder groupid1;
\r
128 groupid1 = this.getFeedIdsByGroupId(Integer.parseInt(req.getParameter(GROUPID)));
\r
129 map.put(FEEDIDS, groupid1.toString());
\r
130 } catch (NumberFormatException e) {
\r
131 eventlogger.error("PROV0172 StatisticsServlet.doGet: " + e.getMessage(), e);
\r
134 if (req.getParameter(GROUPID) != null && req.getParameter(FEEDID) != null) {
\r
135 StringBuilder groupid1;
\r
137 groupid1 = this.getFeedIdsByGroupId(Integer.parseInt(req.getParameter(GROUPID)));
\r
138 groupid1.append(",");
\r
139 groupid1.append(req.getParameter(FEEDID).replace("|", ","));
\r
140 map.put(FEEDIDS, groupid1.toString());
\r
141 } catch (NumberFormatException e) {
\r
142 eventlogger.error("PROV0173 StatisticsServlet.doGet: " + e.getMessage(), e);
\r
145 if (req.getParameter(SUBID) != null && req.getParameter(FEEDID) != null) {
\r
146 String subidstr = "and e.DELIVERY_SUBID in("
\r
147 + req.getParameter(SUBID).replace("|", ",") + ")";
\r
148 map.put(SUBID, subidstr);
\r
150 if (req.getParameter(SUBID) != null && req.getParameter(GROUPID) != null) {
\r
151 String subidstr = "and e.DELIVERY_SUBID in("
\r
152 + req.getParameter(SUBID).replace("|", ",") + ")";
\r
153 map.put(SUBID, subidstr);
\r
155 if (req.getParameter("type") != null) {
\r
156 map.put(EVENT_TYPE, req.getParameter("type").replace("|", ","));
\r
158 if (req.getParameter(OUTPUT_TYPE) != null) {
\r
159 map.put(OUTPUT_TYPE, req.getParameter(OUTPUT_TYPE));
\r
160 outputType = req.getParameter(OUTPUT_TYPE);
\r
162 if (req.getParameter(OUTPUT_TYPE) != null && req.getParameter(OUTPUT_TYPE).equals(JSON_OUTPUT_TYPE)) {
\r
163 map.put(OUTPUT_TYPE, JSON_OUTPUT_TYPE);
\r
165 if (req.getParameter(START_TIME) != null) {
\r
166 String start_time = req.getParameter(START_TIME);
\r
168 Long.parseLong(start_time);
\r
169 map.put(START_TIME, start_time);
\r
171 catch (NumberFormatException e){
\r
172 eventlogger.error("Invalid start time StatisticsServlet.doGet: " + e.getMessage(), e);
\r
175 if (req.getParameter(END_TIME) != null) {
\r
176 String end_time = req.getParameter(END_TIME);
\r
178 Long.parseLong(end_time);
\r
179 map.put(END_TIME, end_time);
\r
181 catch (NumberFormatException e){
\r
182 eventlogger.error("Invalid end time StatisticsServlet.doGet: " + e.getMessage(), e);
\r
185 if (req.getParameter("time") != null) {
\r
186 String time = req.getParameter("time");
\r
188 Long.parseLong(time);
\r
189 map.put(START_TIME, time);
\r
190 map.put(END_TIME, null);
\r
192 catch (NumberFormatException e){
\r
193 eventlogger.error("Invalid end time StatisticsServlet.doGet: " + e.getMessage(), e);
\r
197 this.getRecordsForSQL(map, outputType, resp.getOutputStream(), resp);
\r
198 } catch (IOException ioe) {
\r
199 eventlogger.error("PROV0174 StatisticsServlet.doGet: " + ioe.getMessage(), ioe);
\r
204 private boolean validateDateInput(String date){
\r
210 * rsToJson - Converting RS to JSON object.
\r
212 * @param out ServletOutputStream
\r
213 * @param rs as ResultSet
\r
214 * @throws IOException input/output exception
\r
215 * @throws SQLException SQL exception
\r
217 private void rsToCSV(ResultSet rs, ServletOutputStream out) throws IOException, SQLException {
\r
218 String header = "FEEDNAME,FEEDID,FILES_PUBLISHED,PUBLISH_LENGTH, FILES_DELIVERED, "
\r
219 + "DELIVERED_LENGTH, SUBSCRIBER_URL, SUBID, PUBLISH_TIME,DELIVERY_TIME, AverageDelay\n";
\r
220 out.write(header.getBytes());
\r
222 while (rs.next()) {
\r
223 String line = rs.getString("FEEDNAME")
\r
225 + rs.getString(FEEDID)
\r
227 + rs.getString("FILES_PUBLISHED")
\r
229 + rs.getString("PUBLISH_LENGTH")
\r
231 + rs.getString("FILES_DELIVERED")
\r
233 + rs.getString("DELIVERED_LENGTH")
\r
235 + rs.getString("SUBSCRIBER_URL")
\r
237 + rs.getString("SUBID")
\r
239 + rs.getString("PUBLISH_TIME")
\r
241 + rs.getString("DELIVERY_TIME")
\r
243 + rs.getString("AverageDelay")
\r
246 out.write(line.getBytes());
\r
252 * rsToJson - Converting RS to JSON object.
\r
254 * @param out ServletOutputStream
\r
255 * @param rs as ResultSet
\r
256 * @throws IOException input/output exception
\r
257 * @throws SQLException SQL exception
\r
259 private void rsToJson(ResultSet rs, ServletOutputStream out) throws IOException, SQLException {
\r
260 String[] fields = {"FEEDNAME", FEEDID, "FILES_PUBLISHED", "PUBLISH_LENGTH", "FILES_DELIVERED",
\r
261 "DELIVERED_LENGTH", "SUBSCRIBER_URL", "SUBID", "PUBLISH_TIME", "DELIVERY_TIME",
\r
263 StringBuilder line = new StringBuilder();
\r
264 line.append("[\n");
\r
265 while (rs.next()) {
\r
266 LOGJSONObject j2 = new LOGJSONObject();
\r
267 for (String key : fields) {
\r
268 Object val = rs.getString(key);
\r
270 j2.put(key.toLowerCase(), val);
\r
272 j2.put(key.toLowerCase(), "");
\r
275 line.append(j2.toString());
\r
276 line.append(",\n");
\r
279 out.print(line.toString());
\r
283 * getFeedIdsByGroupId - Getting FEEDID's by GROUP ID.
\r
285 * @param groupIds Integer ref of Group
\r
287 private StringBuilder getFeedIdsByGroupId(int groupIds) {
\r
288 StringBuilder feedIds = new StringBuilder();
\r
289 try (Connection conn = ProvDbUtils.getInstance().getConnection();
\r
290 PreparedStatement prepareStatement = conn.prepareStatement(
\r
291 " SELECT FEEDID from FEEDS WHERE GROUPID = ?")) {
\r
292 prepareStatement.setInt(1, groupIds);
\r
293 try (ResultSet resultSet = prepareStatement.executeQuery()) {
\r
294 while (resultSet.next()) {
\r
295 feedIds.append(resultSet.getInt(FEEDID));
\r
296 feedIds.append(",");
\r
299 feedIds.deleteCharAt(feedIds.length() - 1);
\r
300 eventlogger.info("PROV0177 StatisticsServlet.getFeedIdsByGroupId: feedIds = " + feedIds.toString());
\r
301 } catch (SQLException e) {
\r
302 eventlogger.error("PROV0175 StatisticsServlet.getFeedIdsByGroupId: " + e.getMessage(), e);
\r
309 * queryGeneretor - Generating sql query.
\r
311 * @param map as key value pare of all user input fields
\r
313 private String queryGeneretor(Map<String, String> map) throws ParseException {
\r
316 String feedids = null;
\r
317 String startTime = null;
\r
318 String endTime = null;
\r
319 String subid = " ";
\r
321 if (map.get(FEEDIDS) != null) {
\r
322 feedids = map.get(FEEDIDS);
\r
324 if (map.get(START_TIME) != null) {
\r
325 startTime = map.get(START_TIME);
\r
327 if (map.get(END_TIME) != null) {
\r
328 endTime = map.get(END_TIME);
\r
330 if (map.get(SUBID) != null) {
\r
331 subid = map.get(SUBID);
\r
334 eventlogger.info("Generating sql query to get Statistics resultset. ");
\r
336 if (endTime == null && startTime == null) {
\r
338 sql = SQL_SELECT_NAME + feedids + SQL_FEED_ID + SQL_SELECT_COUNT + feedids + SQL_TYPE_PUB
\r
340 + feedids + SQL_PUBLISH_LENGTH
\r
341 + SQL_SUBSCRIBER_URL + SQL_SUB_ID + SQL_DELIVERY_TIME + SQL_AVERAGE_DELAY + SQL_JOIN_RECORDS
\r
342 + feedids + ") " + subid
\r
343 + SQL_STATUS_204 + SQL_GROUP_SUB_ID;
\r
346 } else if (startTime != null && endTime == null) {
\r
348 long inputTimeInMilli = 60000 * Long.parseLong(startTime);
\r
349 Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
\r
350 long currentTimeInMilli = cal.getTimeInMillis();
\r
351 long compareTime = currentTimeInMilli - inputTimeInMilli;
\r
353 sql = SQL_SELECT_NAME + feedids + SQL_FEED_ID + SQL_SELECT_COUNT + feedids + SQL_TYPE_PUB
\r
355 + feedids + SQL_PUBLISH_LENGTH
\r
356 + SQL_SUBSCRIBER_URL + SQL_SUB_ID + SQL_DELIVERY_TIME + SQL_AVERAGE_DELAY + SQL_JOIN_RECORDS
\r
357 + feedids + ") " + subid
\r
358 + SQL_STATUS_204 + " and e.event_time>=" + compareTime + SQL_GROUP_SUB_ID;
\r
362 SimpleDateFormat inFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
\r
363 Date startDate = inFormat.parse(startTime);
\r
364 Date endDate = inFormat.parse(endTime);
\r
366 long startInMillis = startDate.getTime();
\r
367 long endInMillis = endDate.getTime();
\r
369 sql = SQL_SELECT_NAME + feedids + SQL_FEED_ID + SQL_SELECT_COUNT + feedids + SQL_TYPE_PUB
\r
371 + feedids + SQL_PUBLISH_LENGTH + SQL_SUBSCRIBER_URL
\r
372 + SQL_SUB_ID + SQL_DELIVERY_TIME + SQL_AVERAGE_DELAY + SQL_JOIN_RECORDS + feedids + ")" + subid + SQL_STATUS_204
\r
373 +" and e.event_time between " + startInMillis + " and " + endInMillis + SQL_GROUP_SUB_ID;
\r
381 * PUT a Statistics URL -- not supported.
\r
384 public void doPut(HttpServletRequest req, HttpServletResponse resp) {
\r
385 String message = "PUT not allowed for the StatisticsURL.";
\r
386 EventLogRecord elr = new EventLogRecord(req);
\r
387 elr.setMessage(message);
\r
388 elr.setResult(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
\r
389 eventlogger.error(elr.toString());
\r
390 sendResponseError(resp, HttpServletResponse.SC_METHOD_NOT_ALLOWED, message, eventlogger);
\r
394 * POST a Statistics URL -- not supported.
\r
397 public void doPost(HttpServletRequest req, HttpServletResponse resp) {
\r
398 String message = "POST not allowed for the StatisticsURL.";
\r
399 EventLogRecord elr = new EventLogRecord(req);
\r
400 elr.setMessage(message);
\r
401 elr.setResult(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
\r
402 eventlogger.error(elr.toString());
\r
403 sendResponseError(resp, HttpServletResponse.SC_METHOD_NOT_ALLOWED, message, eventlogger);
\r
406 private Map<String, String> buildMapFromRequest(HttpServletRequest req) {
\r
407 Map<String, String> map = new HashMap<>();
\r
408 String str = req.getParameter("type");
\r
410 if ("pub".equals(str) || "del".equals(str) || "exp".equals(str)) {
\r
411 map.put("type", str);
\r
413 map.put("err", "bad type");
\r
417 map.put("type", "all");
\r
419 map.put("publishSQL", "");
\r
420 map.put("statusSQL", "");
\r
421 map.put("resultSQL", "");
\r
422 map.put(REASON_SQL, "");
\r
424 str = req.getParameter("publishId");
\r
426 if (str.indexOf("'") >= 0) {
\r
427 map.put("err", "bad publishId");
\r
430 map.put("publishSQL", " AND PUBLISH_ID = '" + str + "'");
\r
433 str = req.getParameter("statusCode");
\r
438 sql = " AND STATUS >= 200 AND STATUS < 300";
\r
441 sql = " AND STATUS >= 300 AND STATUS < 400";
\r
444 sql = " AND STATUS >= 400";
\r
448 int statusCode = Integer.parseInt(str);
\r
449 if ((statusCode >= 100 && statusCode < 600) || (statusCode == -1)) {
\r
450 sql = " AND STATUS = " + statusCode;
\r
452 } catch (NumberFormatException e) {
\r
453 eventlogger.error("Failed to parse input", e);
\r
458 map.put("err", "bad statusCode");
\r
461 map.put("statusSQL", sql);
\r
462 map.put("resultSQL", sql.replaceAll("STATUS", "RESULT"));
\r
465 str = req.getParameter("expiryReason");
\r
467 map.put("type", "exp");
\r
469 case "notRetryable":
\r
470 map.put(REASON_SQL, " AND REASON = 'notRetryable'");
\r
472 case "retriesExhausted":
\r
473 map.put(REASON_SQL, " AND REASON = 'retriesExhausted'");
\r
476 map.put(REASON_SQL, " AND REASON = 'diskFull'");
\r
479 map.put(REASON_SQL, " AND REASON = 'other'");
\r
482 map.put("err", "bad expiryReason");
\r
487 long stime = getTimeFromParam(req.getParameter("start"));
\r
489 map.put("err", "bad start");
\r
492 long etime = getTimeFromParam(req.getParameter("end"));
\r
494 map.put("err", "bad end");
\r
497 if (stime == 0 && etime == 0) {
\r
498 etime = System.currentTimeMillis();
\r
499 stime = etime - TWENTYFOUR_HOURS;
\r
500 } else if (stime == 0) {
\r
501 stime = etime - TWENTYFOUR_HOURS;
\r
502 } else if (etime == 0) {
\r
503 etime = stime + TWENTYFOUR_HOURS;
\r
505 map.put("timeSQL", String.format(" AND EVENT_TIME >= %d AND EVENT_TIME <= %d", stime, etime));
\r
509 private long getTimeFromParam(final String str) {
\r
514 // First, look for an RFC 3339 date
\r
515 String fmt = (str.indexOf('.') > 0) ? FMT2 : FMT1;
\r
516 SimpleDateFormat sdf = new SimpleDateFormat(fmt);
\r
517 Date date = sdf.parse(str);
\r
518 return date.getTime();
\r
519 } catch (ParseException e) {
\r
520 intlogger.error("Exception in getting Time :- " + e.getMessage(), e);
\r
523 // Also allow a long (in ms); useful for testing
\r
524 return Long.parseLong(str);
\r
525 } catch (NumberFormatException e) {
\r
526 intlogger.error("Exception in getting Time :- " + e.getMessage(), e);
\r
528 intlogger.info("Error parsing time=" + str);
\r
532 private void getRecordsForSQL(Map<String, String> map, String outputType, ServletOutputStream out,
\r
533 HttpServletResponse resp) {
\r
535 String filterQuery = this.queryGeneretor(map);
\r
536 eventlogger.debug("SQL Query for Statistics resultset. " + filterQuery);
\r
537 intlogger.debug(filterQuery);
\r
538 long start = System.currentTimeMillis();
\r
539 try (Connection conn = ProvDbUtils.getInstance().getConnection();
\r
540 PreparedStatement ps = conn.prepareStatement(filterQuery);
\r
541 ResultSet rs = ps.executeQuery()) {
\r
542 if (CSV_OUTPUT_TYPE.equals(outputType)) {
\r
543 resp.setContentType("application/octet-stream");
\r
544 DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd-MM-yyyy HH:mm:ss");
\r
545 resp.setHeader("Content-Disposition",
\r
546 "attachment; filename=\"result:" + LocalDateTime.now().format(formatter) + ".csv\"");
\r
547 eventlogger.info("Generating CSV file from Statistics resultset");
\r
550 eventlogger.info("Generating JSON for Statistics resultset");
\r
551 this.rsToJson(rs, out);
\r
553 } catch (SQLException e) {
\r
554 eventlogger.error("SQLException:" + e);
\r
556 intlogger.debug("Time: " + (System.currentTimeMillis() - start) + " ms");
\r
557 } catch (IOException e) {
\r
558 eventlogger.error("IOException - Generating JSON/CSV:" + e);
\r
559 } catch (JSONException e) {
\r
560 eventlogger.error("JSONException - executing SQL query:" + e);
\r
561 } catch (ParseException e) {
\r
562 eventlogger.error("ParseException - executing SQL query:" + e);
\r