1 /*******************************************************************************
\r
2 * ============LICENSE_START==================================================
\r
4 * * ===========================================================================
\r
5 * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
\r
6 * * ===========================================================================
\r
7 * * Licensed under the Apache License, Version 2.0 (the "License");
\r
8 * * you may not use this file except in compliance with the License.
\r
9 * * You may obtain a copy of the License at
\r
11 * * http://www.apache.org/licenses/LICENSE-2.0
\r
13 * * Unless required by applicable law or agreed to in writing, software
\r
14 * * distributed under the License is distributed on an "AS IS" BASIS,
\r
15 * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
\r
16 * * See the License for the specific language governing permissions and
\r
17 * * limitations under the License.
\r
18 * * ============LICENSE_END====================================================
\r
20 * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
\r
22 ******************************************************************************/
\r
24 package com.att.research.datarouter.provisioning;
\r
26 import java.io.IOException;
\r
27 import java.sql.Connection;
\r
28 import java.sql.PreparedStatement;
\r
29 import java.sql.ResultSet;
\r
30 import java.sql.SQLException;
\r
31 import java.sql.Statement;
\r
32 import java.text.ParseException;
\r
33 import java.text.SimpleDateFormat;
\r
34 import java.util.Calendar;
\r
35 import java.util.Date;
\r
36 import java.util.HashMap;
\r
37 import java.util.Map;
\r
38 import java.util.TimeZone;
\r
39 import javax.servlet.ServletOutputStream;
\r
40 import javax.servlet.http.HttpServletRequest;
\r
41 import javax.servlet.http.HttpServletResponse;
\r
42 import org.json.JSONException;
\r
43 import org.json.LOGJSONObject;
\r
44 import com.att.research.datarouter.provisioning.beans.EventLogRecord;
\r
45 import com.att.research.datarouter.provisioning.utils.DB;
\r
48 * This Servlet handles requests to the <Statistics API> and <Statistics consilidated resultset>,
\r
49 * @author Manish Singh
\r
50 * @version $Id: StatisticsServlet.java,v 1.11 2016/08/10 17:27:02 Manish Exp $
\r
52 @SuppressWarnings("serial")
\r
54 public class StatisticsServlet extends BaseServlet {
\r
56 private static final long TWENTYFOUR_HOURS = (24 * 60 * 60 * 1000L);
\r
57 private static final String fmt1 = "yyyy-MM-dd'T'HH:mm:ss'Z'";
\r
58 private static final String fmt2 = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
\r
62 * DELETE a logging URL -- not supported.
\r
65 public void doDelete(HttpServletRequest req, HttpServletResponse resp) throws IOException {
\r
66 String message = "DELETE not allowed for the logURL.";
\r
67 EventLogRecord elr = new EventLogRecord(req);
\r
68 elr.setMessage(message);
\r
69 elr.setResult(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
\r
70 eventlogger.info(elr);
\r
71 resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, message);
\r
74 * GET a Statistics URL -- retrieve Statistics data for a feed or subscription.
\r
75 * See the <b>Statistics API</b> document for details on how this method should be invoked.
\r
78 public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
\r
80 Map<String, String> map = buildMapFromRequest(req);
\r
81 if (map.get("err") != null) {
\r
82 resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid arguments: "+map.get("err"));
\r
85 // check Accept: header??
\r
87 resp.setStatus(HttpServletResponse.SC_OK);
\r
88 resp.setContentType(LOGLIST_CONTENT_TYPE);
\r
89 ServletOutputStream out = resp.getOutputStream();
\r
92 String outputType = "json";
\r
93 String feedids = null;
\r
95 if(req.getParameter("feedid") ==null && req.getParameter("groupid") ==null)
\r
97 out.print("Invalid request, Feedid or Group ID is required.");
\r
100 if(req.getParameter("feedid")!=null && req.getParameter("groupid") == null) {
\r
101 map.put("feedids", req.getParameter("feedid").replace("|", ",").toString());
\r
104 if(req.getParameter("groupid") != null && req.getParameter("feedid") ==null) {
\r
105 // String groupid1 = null;
\r
106 StringBuffer groupid1 = new StringBuffer();
\r
109 System.out.println("feeedidsssssssss");
\r
110 groupid1 = this.getFeedIdsByGroupId(Integer.parseInt(req.getParameter("groupid")));
\r
111 System.out.println("feeedids"+req.getParameter("groupid"));
\r
113 map.put("feedids", groupid1.toString());
\r
114 System.out.println("groupid1" +groupid1.toString());
\r
117 } catch (NumberFormatException e) {
\r
118 e.printStackTrace();
\r
119 } catch (SQLException e) {
\r
120 e.printStackTrace();
\r
123 if(req.getParameter("groupid") != null && req.getParameter("feedid") !=null) {
\r
124 StringBuffer groupid1 = new StringBuffer();
\r
128 System.out.println("both r not null");
\r
129 groupid1 = this.getFeedIdsByGroupId(Integer.parseInt(req.getParameter("groupid")));
\r
130 System.out.println("feeedids"+req.getParameter("groupid"));
\r
131 groupid1.append(",");
\r
132 groupid1.append(req.getParameter("feedid").replace("|", ",").toString());
\r
134 map.put("feedids", groupid1.toString());
\r
137 System.out.println("groupid1" +groupid1.toString());
\r
140 } catch (NumberFormatException e) {
\r
141 e.printStackTrace();
\r
142 } catch (SQLException e) {
\r
143 e.printStackTrace();
\r
149 if(req.getParameter("subid")!=null && req.getParameter("feedid") !=null) {
\r
150 StringBuffer subidstr = new StringBuffer();
\r
151 // subidstr.append(" and e.DELIVERY_SUBID in(subid)");
\r
152 // subidstr.append(req.getParameter("subid").replace("|", ",").toString());
\r
153 subidstr.append("and e.DELIVERY_SUBID in(");
\r
155 subidstr.append(req.getParameter("subid").replace("|", ",").toString());
\r
156 subidstr.append(")");
\r
157 map.put("subid", subidstr.toString());
\r
159 if(req.getParameter("subid")!=null && req.getParameter("groupid") !=null) {
\r
160 StringBuffer subidstr = new StringBuffer();
\r
161 // subidstr.append(" and e.DELIVERY_SUBID in(subid)");
\r
162 // subidstr.append(req.getParameter("subid").replace("|", ",").toString());
\r
163 subidstr.append("and e.DELIVERY_SUBID in(");
\r
165 subidstr.append(req.getParameter("subid").replace("|", ",").toString());
\r
166 subidstr.append(")");
\r
167 map.put("subid", subidstr.toString());
\r
169 if(req.getParameter("type")!=null) {
\r
170 map.put("eventType", req.getParameter("type").replace("|", ",").toString());
\r
172 if(req.getParameter("output_type")!=null) {
\r
173 map.put("output_type", req.getParameter("output_type").toString());
\r
175 if(req.getParameter("start_time")!=null) {
\r
176 map.put("start_time", req.getParameter("start_time").toString());
\r
178 if(req.getParameter("end_time")!=null) {
\r
179 map.put("end_time", req.getParameter("end_time").toString());
\r
182 if(req.getParameter("time")!=null) {
\r
183 map.put("start_time", req.getParameter("time").toString());
\r
184 map.put("end_time", null);
\r
189 if(req.getParameter("output_type") !=null)
\r
191 outputType = req.getParameter("output_type");
\r
197 String filterQuery = this.queryGeneretor(map);
\r
198 eventlogger.debug("SQL Query for Statistics resultset. "+filterQuery);
\r
200 ResultSet rs=this.getRecordsForSQL(filterQuery);
\r
202 if(outputType.equals("csv")) {
\r
203 resp.setContentType("application/octet-stream");
\r
204 Date date = new Date() ;
\r
205 SimpleDateFormat dateFormat = new SimpleDateFormat("dd-MM-YYYY HH:mm:ss") ;
\r
206 resp.setHeader("Content-Disposition", "attachment; filename=\"result:"+dateFormat.format(date)+".csv\"");
\r
207 eventlogger.info("Generating CSV file from Statistics resultset");
\r
212 eventlogger.info("Generating JSON for Statistics resultset");
\r
213 this.rsToJson(rs, out);
\r
216 catch (IOException e) {
\r
217 eventlogger.error("IOException - Generating JSON/CSV:"+e);
\r
218 e.printStackTrace();
\r
220 catch (JSONException e) {
\r
221 eventlogger.error("JSONException - executing SQL query:"+e);
\r
222 e.printStackTrace();
\r
223 } catch (SQLException e) {
\r
224 eventlogger.error("SQLException - executing SQL query:"+e);
\r
225 e.printStackTrace();
\r
226 } catch (ParseException e) {
\r
227 eventlogger.error("ParseException - executing SQL query:"+e);
\r
228 e.printStackTrace();
\r
234 * rsToJson - Converting RS to JSON object
\r
235 * @exception IOException, SQLException
\r
236 * @param out ServletOutputStream, rs as ResultSet
\r
238 public void rsToCSV(ResultSet rs, ServletOutputStream out) throws IOException, SQLException {
\r
239 String header = "FEEDNAME,FEEDID,FILES_PUBLISHED,PUBLISH_LENGTH, FILES_DELIVERED, DELIVERED_LENGTH, SUBSCRIBER_URL, SUBID, PUBLISH_TIME,DELIVERY_TIME, AverageDelay\n";
\r
241 // String header = "FEEDNAME,FEEDID,TYPE,REMOTE_ADDR,DELIVERY_SUBID,REQURI,TOTAL CONTENT LENGTH,NO OF FILE,AVERAGE DELAY\n";
\r
243 out.write(header.getBytes());
\r
246 StringBuffer line = new StringBuffer();
\r
247 line.append(rs.getString("FEEDNAME"));
\r
249 line.append(rs.getString("FEEDID"));
\r
251 line.append(rs.getString("FILES_PUBLISHED"));
\r
253 line.append(rs.getString("PUBLISH_LENGTH"));
\r
255 line.append(rs.getString("FILES_DELIVERED"));
\r
257 line.append(rs.getString("DELIVERED_LENGTH"));
\r
259 line.append(rs.getString("SUBSCRIBER_URL"));
\r
261 line.append(rs.getString("SUBID"));
\r
263 line.append(rs.getString("PUBLISH_TIME"));
\r
265 line.append(rs.getString("DELIVERY_TIME"));
\r
267 line.append(rs.getString("AverageDelay"));
\r
271 out.write(line.toString().getBytes());
\r
277 * rsToJson - Converting RS to JSON object
\r
278 * @exception IOException, SQLException
\r
279 * @param out ServletOutputStream, rs as ResultSet
\r
281 public void rsToJson(ResultSet rs, ServletOutputStream out) throws IOException, SQLException {
\r
283 String fields[] = {"FEEDNAME","FEEDID","FILES_PUBLISHED","PUBLISH_LENGTH", "FILES_DELIVERED", "DELIVERED_LENGTH", "SUBSCRIBER_URL", "SUBID", "PUBLISH_TIME","DELIVERY_TIME", "AverageDelay"};
\r
284 StringBuffer line = new StringBuffer();
\r
286 line.append("[\n");
\r
289 LOGJSONObject j2 = new LOGJSONObject();
\r
290 for (String key : fields) {
\r
291 Object v = rs.getString(key);
\r
293 j2.put(key.toLowerCase(), v);
\r
295 j2.put(key.toLowerCase(), "");
\r
297 line = line.append(j2.toString());;
\r
298 line.append(",\n");
\r
301 out.print(line.toString());
\r
305 * getFeedIdsByGroupId - Getting FEEDID's by GROUP ID.
\r
306 * @exception SQL Query SQLException.
\r
309 public StringBuffer getFeedIdsByGroupId(int groupIds) throws SQLException{
\r
312 Connection conn = null;
\r
313 PreparedStatement prepareStatement = null;
\r
314 ResultSet resultSet=null;
\r
315 String sqlGoupid = null;
\r
316 StringBuffer feedIds = new StringBuffer();
\r
320 conn = db.getConnection();
\r
321 sqlGoupid= " SELECT FEEDID from FEEDS WHERE GROUPID = ?";
\r
322 prepareStatement =conn.prepareStatement(sqlGoupid);
\r
323 prepareStatement.setInt(1, groupIds);
\r
324 resultSet=prepareStatement.executeQuery();
\r
325 while(resultSet.next()){
\r
326 feedIds.append(resultSet.getInt("FEEDID"));
\r
327 feedIds.append(",");
\r
329 feedIds.deleteCharAt(feedIds.length()-1);
\r
330 System.out.println("feedIds"+feedIds.toString());
\r
332 } catch (SQLException e) {
\r
333 e.printStackTrace();
\r
336 if(resultSet != null) {
\r
337 resultSet.close();
\r
341 if(prepareStatement != null) {
\r
342 prepareStatement.close();
\r
343 prepareStatement = null;
\r
349 } catch(Exception e) {
\r
350 e.printStackTrace();
\r
358 * queryGeneretor - Generating sql query
\r
359 * @exception SQL Query parse exception.
\r
360 * @param Map as key value pare of all user input fields
\r
362 public String queryGeneretor(Map<String, String> map) throws ParseException{
\r
365 String eventType = null;
\r
366 String feedids = null;
\r
367 String start_time = null;
\r
368 String end_time = null;
\r
370 if(map.get("eventType") != null){
\r
371 eventType=(String) map.get("eventType");
\r
373 if(map.get("feedids") != null){
\r
374 feedids=(String) map.get("feedids");
\r
376 if(map.get("start_time") != null){
\r
377 start_time=(String) map.get("start_time");
\r
379 if(map.get("end_time") != null){
\r
380 end_time=(String) map.get("end_time");
\r
382 if("all".equalsIgnoreCase(eventType)){
\r
383 eventType="PUB','DEL, EXP, PBF";
\r
385 if(map.get("subid") != null){
\r
386 subid=(String) map.get("subid");
\r
389 eventlogger.info("Generating sql query to get Statistics resultset. ");
\r
391 if(end_time==null && start_time==null ){
\r
394 sql="SELECT (SELECT NAME FROM FEEDS AS f WHERE f.FEEDID in("+feedids+") and f.FEEDID=e.FEEDID) AS FEEDNAME, e.FEEDID as FEEDID, (SELECT COUNT(*) FROM LOG_RECORDS AS c WHERE c.FEEDID in("+feedids+") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS FILES_PUBLISHED,(SELECT SUM(content_length) FROM LOG_RECORDS AS c WHERE c.FEEDID in("+feedids+") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS PUBLISH_LENGTH, COUNT(e.EVENT_TIME) as FILES_DELIVERED, sum(m.content_length) as DELIVERED_LENGTH,SUBSTRING_INDEX(e.REQURI,'/',+3) as SUBSCRIBER_URL, e.DELIVERY_SUBID as SUBID, e.EVENT_TIME AS PUBLISH_TIME, m.EVENT_TIME AS DELIVERY_TIME, AVG(e.EVENT_TIME - m.EVENT_TIME)/1000 as AverageDelay FROM LOG_RECORDS e JOIN LOG_RECORDS m ON m.PUBLISH_ID = e.PUBLISH_ID AND e.FEEDID IN ("+feedids+") "+subid+" AND m.STATUS=204 AND e.RESULT=204 group by SUBID";
\r
397 }else if(start_time!=null && end_time==null ){
\r
399 long inputTimeInMilli=60000*Long.parseLong(start_time);
\r
400 Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
\r
401 long currentTimeInMilli=cal.getTimeInMillis();
\r
402 long compareTime=currentTimeInMilli-inputTimeInMilli;
\r
404 sql="SELECT (SELECT NAME FROM FEEDS AS f WHERE f.FEEDID in("+feedids+") and f.FEEDID=e.FEEDID) AS FEEDNAME, e.FEEDID as FEEDID, (SELECT COUNT(*) FROM LOG_RECORDS AS c WHERE c.FEEDID in("+feedids+") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS FILES_PUBLISHED,(SELECT SUM(content_length) FROM LOG_RECORDS AS c WHERE c.FEEDID in("+feedids+") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS PUBLISH_LENGTH, COUNT(e.EVENT_TIME) as FILES_DELIVERED, sum(m.content_length) as DELIVERED_LENGTH,SUBSTRING_INDEX(e.REQURI,'/',+3) as SUBSCRIBER_URL, e.DELIVERY_SUBID as SUBID, e.EVENT_TIME AS PUBLISH_TIME, m.EVENT_TIME AS DELIVERY_TIME, AVG(e.EVENT_TIME - m.EVENT_TIME)/1000 as AverageDelay FROM LOG_RECORDS e JOIN LOG_RECORDS m ON m.PUBLISH_ID = e.PUBLISH_ID AND e.FEEDID IN ("+feedids+") "+subid+" AND m.STATUS=204 AND e.RESULT=204 and e.event_time>="+compareTime+" group by SUBID";
\r
409 SimpleDateFormat inFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
\r
410 Date startDate=inFormat.parse(start_time);
\r
411 Date endDate=inFormat.parse(end_time);
\r
413 long startInMillis=startDate.getTime();
\r
414 long endInMillis=endDate.getTime();
\r
418 sql="SELECT (SELECT NAME FROM FEEDS AS f WHERE f.FEEDID in("+feedids+") and f.FEEDID=e.FEEDID) AS FEEDNAME, e.FEEDID as FEEDID, (SELECT COUNT(*) FROM LOG_RECORDS AS c WHERE c.FEEDID in("+feedids+") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS FILES_PUBLISHED,(SELECT SUM(content_length) FROM LOG_RECORDS AS c WHERE c.FEEDID in("+feedids+") and c.FEEDID=e.FEEDID AND c.TYPE='PUB') AS PUBLISH_LENGTH, COUNT(e.EVENT_TIME) as FILES_DELIVERED, sum(m.content_length) as DELIVERED_LENGTH,SUBSTRING_INDEX(e.REQURI,'/',+3) as SUBSCRIBER_URL, e.DELIVERY_SUBID as SUBID, e.EVENT_TIME AS PUBLISH_TIME, m.EVENT_TIME AS DELIVERY_TIME, AVG(e.EVENT_TIME - m.EVENT_TIME)/1000 as AverageDelay FROM LOG_RECORDS e JOIN LOG_RECORDS m ON m.PUBLISH_ID = e.PUBLISH_ID AND e.FEEDID IN ("+feedids+") "+subid+" AND m.STATUS=204 AND e.RESULT=204 and e.event_time between "+startInMillis+" and "+endInMillis+" group by SUBID";
\r
427 * PUT a Statistics URL -- not supported.
\r
430 public void doPut(HttpServletRequest req, HttpServletResponse resp) throws IOException {
\r
431 String message = "PUT not allowed for the StatisticsURL.";
\r
432 EventLogRecord elr = new EventLogRecord(req);
\r
433 elr.setMessage(message);
\r
434 elr.setResult(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
\r
435 eventlogger.info(elr);
\r
436 resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, message);
\r
439 * POST a Statistics URL -- not supported.
\r
442 public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
\r
443 String message = "POST not allowed for the StatisticsURL.";
\r
444 EventLogRecord elr = new EventLogRecord(req);
\r
445 elr.setMessage(message);
\r
446 elr.setResult(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
\r
447 eventlogger.info(elr);
\r
448 resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, message);
\r
451 private Map<String, String> buildMapFromRequest(HttpServletRequest req) {
\r
452 Map<String, String> map = new HashMap<String, String>();
\r
453 String s = req.getParameter("type");
\r
455 if (s.equals("pub") || s.equals("del") || s.equals("exp")) {
\r
456 map.put("type", s);
\r
458 map.put("err", "bad type");
\r
462 map.put("type", "all");
\r
463 map.put("publishSQL", "");
\r
464 map.put("statusSQL", "");
\r
465 map.put("resultSQL", "");
\r
466 map.put("reasonSQL", "");
\r
468 s = req.getParameter("publishId");
\r
470 if (s.indexOf("'") >= 0) {
\r
471 map.put("err", "bad publishId");
\r
474 map.put("publishSQL", " AND PUBLISH_ID = '"+s+"'");
\r
477 s = req.getParameter("statusCode");
\r
480 if (s.equals("success")) {
\r
481 sql = " AND STATUS >= 200 AND STATUS < 300";
\r
482 } else if (s.equals("redirect")) {
\r
483 sql = " AND STATUS >= 300 AND STATUS < 400";
\r
484 } else if (s.equals("failure")) {
\r
485 sql = " AND STATUS >= 400";
\r
488 Integer n = Integer.parseInt(s);
\r
489 if ((n >= 100 && n < 600) || (n == -1))
\r
490 sql = " AND STATUS = " + n;
\r
491 } catch (NumberFormatException e) {
\r
495 map.put("err", "bad statusCode");
\r
498 map.put("statusSQL", sql);
\r
499 map.put("resultSQL", sql.replaceAll("STATUS", "RESULT"));
\r
502 s = req.getParameter("expiryReason");
\r
504 map.put("type", "exp");
\r
505 if (s.equals("notRetryable")) {
\r
506 map.put("reasonSQL", " AND REASON = 'notRetryable'");
\r
507 } else if (s.equals("retriesExhausted")) {
\r
508 map.put("reasonSQL", " AND REASON = 'retriesExhausted'");
\r
509 } else if (s.equals("diskFull")) {
\r
510 map.put("reasonSQL", " AND REASON = 'diskFull'");
\r
511 } else if (s.equals("other")) {
\r
512 map.put("reasonSQL", " AND REASON = 'other'");
\r
514 map.put("err", "bad expiryReason");
\r
519 long stime = getTimeFromParam(req.getParameter("start"));
\r
521 map.put("err", "bad start");
\r
524 long etime = getTimeFromParam(req.getParameter("end"));
\r
526 map.put("err", "bad end");
\r
529 if (stime == 0 && etime == 0) {
\r
530 etime = System.currentTimeMillis();
\r
531 stime = etime - TWENTYFOUR_HOURS;
\r
532 } else if (stime == 0) {
\r
533 stime = etime - TWENTYFOUR_HOURS;
\r
534 } else if (etime == 0) {
\r
535 etime = stime + TWENTYFOUR_HOURS;
\r
537 map.put("timeSQL", String.format(" AND EVENT_TIME >= %d AND EVENT_TIME <= %d", stime, etime));
\r
540 private long getTimeFromParam(final String s) {
\r
544 // First, look for an RFC 3339 date
\r
545 String fmt = (s.indexOf('.') > 0) ? fmt2 : fmt1;
\r
546 SimpleDateFormat sdf = new SimpleDateFormat(fmt);
\r
547 Date d = sdf.parse(s);
\r
548 return d.getTime();
\r
549 } catch (ParseException e) {
\r
552 // Also allow a long (in ms); useful for testing
\r
553 long n = Long.parseLong(s);
\r
555 } catch (NumberFormatException e) {
\r
557 intlogger.info("Error parsing time="+s);
\r
562 private ResultSet getRecordsForSQL(String sql) {
\r
563 intlogger.debug(sql);
\r
564 long start = System.currentTimeMillis();
\r
566 Connection conn = null;
\r
570 conn = db.getConnection();
\r
571 Statement stmt = conn.createStatement();
\r
572 PreparedStatement pst=conn.prepareStatement(sql);
\r
573 rs=pst.executeQuery();
\r
574 //this.rsToJson(rs)
\r
577 } catch (SQLException e) {
\r
578 e.printStackTrace();
\r
584 intlogger.debug("Time: " + (System.currentTimeMillis()-start) + " ms");
\r