datarouter-prov/src/main/java/org/onap/dmaap/datarouter/reports/VolumeReport.java

   1 /*******************************************************************************
   2  * ============LICENSE_START==================================================
   3  * * org.onap.dmaap
   4  * * ===========================================================================
   5  * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * * ===========================================================================
   7  * * Licensed under the Apache License, Version 2.0 (the "License");
   8  * * you may not use this file except in compliance with the License.
   9  * * You may obtain a copy of the License at
  10  * *
  11  *  *      http://www.apache.org/licenses/LICENSE-2.0
  12  * *
  13  *  * Unless required by applicable law or agreed to in writing, software
  14  * * distributed under the License is distributed on an "AS IS" BASIS,
  15  * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * * See the License for the specific language governing permissions and
  17  * * limitations under the License.
  18  * * ============LICENSE_END====================================================
  19  * *
  20  * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  * *
  22  ******************************************************************************/
  23
  24
  25 package org.onap.dmaap.datarouter.reports;
  26
  27 import java.io.FileNotFoundException;
  28 import java.io.PrintWriter;
  29 import java.sql.Connection;
  30 import java.sql.PreparedStatement;
  31 import java.sql.ResultSet;
  32 import java.sql.SQLException;
  33 import java.text.SimpleDateFormat;
  34 import java.util.Date;
  35 import java.util.HashMap;
  36 import java.util.Map;
  37 import java.util.TreeSet;
  38
  39 import org.apache.log4j.Logger;
  40 import org.onap.dmaap.datarouter.provisioning.utils.DB;
  41
  42 /**
  43  * Generate a traffic volume report. The report is a .csv file containing the following columns:
  44  * <table>
  45  * <tr><td>date</td><td>the date for this record</td></tr>
  46  * <tr><td>feedid</td><td>the Feed ID for this record</td></tr>
  47  * <tr><td>filespublished</td><td>the number of files published on this feed and date</td></tr>
  48  * <tr><td>bytespublished</td><td>the number of bytes published on this feed and date</td></tr>
  49  * <tr><td>filesdelivered</td><td>the number of files delivered on this feed and date</td></tr>
  50  * <tr><td>bytesdelivered</td><td>the number of bytes delivered on this feed and date</td></tr>
  51  * <tr><td>filesexpired</td><td>the number of files expired on this feed and date</td></tr>
  52  * <tr><td>bytesexpired</td><td>the number of bytes expired on this feed and date</td></tr>
  53  * </table>
  54  *
  55  * @author Robert P. Eby
  56  * @version $Id: VolumeReport.java,v 1.3 2014/02/28 15:11:13 eby Exp $
  57  */
  58 public class VolumeReport extends ReportBase {
  59     private static final String SELECT_SQL = "select EVENT_TIME, TYPE, FEEDID, CONTENT_LENGTH, RESULT" +
  60             " from LOG_RECORDS where EVENT_TIME >= ? and EVENT_TIME <= ? LIMIT ?, ?";
  61     private Logger loggerVolumeReport=Logger.getLogger("org.onap.dmaap.datarouter.reports");
  62     private class Counters {
  63         int filespublished, filesdelivered, filesexpired;
  64         long bytespublished, bytesdelivered, bytesexpired;
  65
  66         @Override
  67         public String toString() {
  68             return String.format("%d,%d,%d,%d,%d,%d",
  69                     filespublished, bytespublished, filesdelivered,
  70                     bytesdelivered, filesexpired, bytesexpired);
  71         }
  72     }
  73
  74     @Override
  75     public void run() {
  76         Map<String, Counters> map = new HashMap<String, Counters>();
  77         SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
  78         long start = System.currentTimeMillis();
  79         try {
  80             DB db = new DB();
  81             @SuppressWarnings("resource")
  82             Connection conn = db.getConnection();
  83             // We need to run this SELECT in stages, because otherwise we run out of memory!
  84             final long stepsize = 6000000L;
  85             boolean go_again = true;
  86             for (long i = 0; go_again; i += stepsize) {
  87                 try (PreparedStatement ps = conn.prepareStatement(SELECT_SQL)) {
  88                     ps.setLong(1, from);
  89                     ps.setLong(2, to);
  90                     ps.setLong(3, i);
  91                     ps.setLong(4, stepsize);
  92                     try(ResultSet rs = ps.executeQuery()) {
  93                         go_again = false;
  94                         while (rs.next()) {
  95                             go_again = true;
  96                             long etime = rs.getLong("EVENT_TIME");
  97                             String type = rs.getString("TYPE");
  98                             int feed = rs.getInt("FEEDID");
  99                             long clen = rs.getLong("CONTENT_LENGTH");
 100                             String key = sdf.format(new Date(etime)) + ":" + feed;
 101                             Counters c = map.get(key);
 102                             if (c == null) {
 103                                 c = new Counters();
 104                                 map.put(key, c);
 105                             }
 106                             if (type.equalsIgnoreCase("pub")) {
 107                                 c.filespublished++;
 108                                 c.bytespublished += clen;
 109                             } else if (type.equalsIgnoreCase("del")) {
 110                                 // Only count successful deliveries
 111                                 int statusCode = rs.getInt("RESULT");
 112                                 if (statusCode >= 200 && statusCode < 300) {
 113                                     c.filesdelivered++;
 114                                     c.bytesdelivered += clen;
 115                                 }
 116                             } else if (type.equalsIgnoreCase("exp")) {
 117                                 c.filesexpired++;
 118                                 c.bytesexpired += clen;
 119                             }
 120                         }
 121                     }
 122                 }
 123                 catch (SQLException sqlException)
 124                 {
 125                     loggerVolumeReport.error("SqlException",sqlException);
 126                 }
 127             }
 128
 129             db.release(conn);
 130         } catch (SQLException e) {
 131             loggerVolumeReport.error("SQLException: " + e.getMessage());
 132         }
 133         logger.debug("Query time: " + (System.currentTimeMillis() - start) + " ms");
 134         try (PrintWriter os = new PrintWriter(outfile)) {
 135             os.println("date,feedid,filespublished,bytespublished,filesdelivered,bytesdelivered,filesexpired,bytesexpired");
 136             for(String key :new TreeSet<String>(map.keySet()))
 137             {
 138                 Counters c = map.get(key);
 139                 String[] p = key.split(":");
 140                 os.println(String.format("%s,%s,%s", p[0], p[1], c.toString()));
 141             }
 142         }
 143         catch (FileNotFoundException e) {
 144             System.err.println("File cannot be written: " + outfile);
 145         }
 146     }
 147 }