X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=datarouter-prov%2Fsrc%2Fmain%2Fjava%2Forg%2Fonap%2Fdmaap%2Fdatarouter%2Fprovisioning%2Futils%2FLogfileLoader.java;h=ab1a3a78eb72d1ee39a68820d7f443600fb4cf82;hb=5da50e90e6c78700d48d7468849d4a1599a249b3;hp=ff7893d5a491e6ae68743015870003632bebcba9;hpb=4261823d84c2b911b68cdf4cb4dc3be429ebe285;p=dmaap%2Fdatarouter.git diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java index ff7893d5..ab1a3a78 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java @@ -100,7 +100,7 @@ public class LogfileLoader extends Thread { /** * The PreparedStatement which is loaded by a Loadable. */ - public static final String INSERT_SQL = "insert into LOG_RECORDS values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + public static final String INSERT_SQL = "insert into LOG_RECORDS values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; /** * Each server can assign this many IDs */ @@ -188,6 +188,7 @@ public class LogfileLoader extends Thread { try { Thread.sleep(1000L); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } idle = false; } else { @@ -213,7 +214,6 @@ public class LogfileLoader extends Thread { } } catch (Exception e) { logger.warn("PROV0020: Caught exception in LogfileLoader: " + e); - e.printStackTrace(); } } } @@ -254,27 +254,27 @@ public class LogfileLoader extends Thread { try { // Limit to a million at a time to avoid typing up the DB for too long. conn = db.getConnection(); - PreparedStatement ps = conn.prepareStatement("DELETE from LOG_RECORDS where EVENT_TIME < ? limit 1000000"); - ps.setLong(1, cutoff); - while (count > 0) { - if (!ps.execute()) { - int dcount = ps.getUpdateCount(); - count -= dcount; - logger.debug(" " + dcount + " rows deleted."); - did1 |= (dcount != 0); - if (dcount == 0) - count = 0; // prevent inf. loops - } else { - count = 0; // shouldn't happen! + try(PreparedStatement ps = conn.prepareStatement("DELETE from LOG_RECORDS where EVENT_TIME < ? limit 1000000")) { + ps.setLong(1, cutoff); + while (count > 0) { + if (!ps.execute()) { + int dcount = ps.getUpdateCount(); + count -= dcount; + logger.debug(" " + dcount + " rows deleted."); + did1 |= (dcount != 0); + if (dcount == 0) + count = 0; // prevent inf. loops + } else { + count = 0; // shouldn't happen! + } } } - ps.close(); - Statement stmt = conn.createStatement(); - stmt.execute("OPTIMIZE TABLE LOG_RECORDS"); - stmt.close(); + try(Statement stmt = conn.createStatement()) { + stmt.execute("OPTIMIZE TABLE LOG_RECORDS"); + } } catch (SQLException e) { System.err.println(e); - e.printStackTrace(); + logger.error(e); } finally { db.release(conn); } @@ -287,16 +287,16 @@ public class LogfileLoader extends Thread { Connection conn = null; try { conn = db.getConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("SELECT COUNT(*) as COUNT from LOG_RECORDS"); - if (rs.next()) { - count = rs.getLong("COUNT"); - } - rs.close(); - stmt.close(); - } catch (SQLException e) { + try(Statement stmt = conn.createStatement()) { + try(ResultSet rs = stmt.executeQuery("SELECT COUNT(*) as COUNT from LOG_RECORDS")) { + if (rs.next()) { + count = rs.getLong("COUNT"); + } + } + } + } catch (SQLException e) { System.err.println(e); - e.printStackTrace(); + logger.error(e); } finally { db.release(conn); } @@ -309,19 +309,19 @@ public class LogfileLoader extends Thread { try { logger.debug(" LOG_RECORD table histogram..."); conn = db.getConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("SELECT FLOOR(EVENT_TIME/86400000) AS DAY, COUNT(*) AS COUNT FROM LOG_RECORDS GROUP BY DAY"); - while (rs.next()) { - long day = rs.getLong("DAY"); - long cnt = rs.getLong("COUNT"); - map.put(day, cnt); - logger.debug(" " + day + " " + cnt); + try(Statement stmt = conn.createStatement()) { + try(ResultSet rs = stmt.executeQuery("SELECT FLOOR(EVENT_TIME/86400000) AS DAY, COUNT(*) AS COUNT FROM LOG_RECORDS GROUP BY DAY")) { + while (rs.next()) { + long day = rs.getLong("DAY"); + long cnt = rs.getLong("COUNT"); + map.put(day, cnt); + logger.debug(" " + day + " " + cnt); + } + } } - rs.close(); - stmt.close(); - } catch (SQLException e) { + } catch (SQLException e) { System.err.println(e); - e.printStackTrace(); + logger.error(e); } finally { db.release(conn); } @@ -332,26 +332,25 @@ public class LogfileLoader extends Thread { Connection conn = null; try { conn = db.getConnection(); - Statement stmt = conn.createStatement(); - // Build a bitset of all records in the LOG_RECORDS table - // We need to run this SELECT in stages, because otherwise we run out of memory! RLEBitSet nbs = new RLEBitSet(); - final long stepsize = 6000000L; - boolean go_again = true; - for (long i = 0; go_again; i += stepsize) { - String sql = String.format("select RECORD_ID from LOG_RECORDS LIMIT %d,%d", i, stepsize); - ResultSet rs = stmt.executeQuery(sql); - go_again = false; - while (rs.next()) { - long n = rs.getLong("RECORD_ID"); - nbs.set(n); - go_again = true; + try(Statement stmt = conn.createStatement()) { + // Build a bitset of all records in the LOG_RECORDS table + // We need to run this SELECT in stages, because otherwise we run out of memory! + final long stepsize = 6000000L; + boolean go_again = true; + for (long i = 0; go_again; i += stepsize) { + String sql = String.format("select RECORD_ID from LOG_RECORDS LIMIT %d,%d", i, stepsize); + try (ResultSet rs = stmt.executeQuery(sql)) { + go_again = false; + while (rs.next()) { + long n = rs.getLong("RECORD_ID"); + nbs.set(n); + go_again = true; + } + } } - rs.close(); } - stmt.close(); seq_set = nbs; - // Compare with the range for this server // Determine the next ID for this set of record IDs RLEBitSet tbs = (RLEBitSet) nbs.clone(); @@ -376,7 +375,7 @@ public class LogfileLoader extends Thread { logger.debug(String.format("initializeNextid, next ID is %d (%x)", nextid, nextid)); } catch (SQLException e) { System.err.println(e); - e.printStackTrace(); + logger.error(e); } finally { db.release(conn); } @@ -391,49 +390,45 @@ public class LogfileLoader extends Thread { Reader r = f.getPath().endsWith(".gz") ? new InputStreamReader(new GZIPInputStream(new FileInputStream(f))) : new FileReader(f); - LineNumberReader in = new LineNumberReader(r); - String line; - while ((line = in.readLine()) != null) { - try { - for (Loadable rec : buildRecords(line)) { - rec.load(ps); - if (rec instanceof LogRecord) { - LogRecord lr = ((LogRecord) rec); - if (!seq_set.get(lr.getRecordId())) { + try(LineNumberReader in = new LineNumberReader(r)) { + String line; + while ((line = in.readLine()) != null) { + try { + for (Loadable rec : buildRecords(line)) { + rec.load(ps); + if (rec instanceof LogRecord) { + LogRecord lr = ((LogRecord) rec); + if (!seq_set.get(lr.getRecordId())) { + ps.executeUpdate(); + seq_set.set(lr.getRecordId()); + } else + logger.debug("Duplicate record ignored: " + lr.getRecordId()); + } else { + if (++nextid > set_end) + nextid = set_start; + ps.setLong(18, nextid); ps.executeUpdate(); - seq_set.set(lr.getRecordId()); - } else - logger.debug("Duplicate record ignored: " + lr.getRecordId()); - } else { - if (++nextid > set_end) - nextid = set_start; - ps.setLong(18, nextid); - ps.executeUpdate(); - seq_set.set(nextid); + seq_set.set(nextid); + } + ps.clearParameters(); + ok++; } - ps.clearParameters(); - ok++; + } catch (SQLException e) { + logger.warn("PROV8003 Invalid value in record: " + line); + logger.debug(e); + } catch (NumberFormatException e) { + logger.warn("PROV8004 Invalid number in record: " + line); + logger.debug(e); + } catch (ParseException e) { + logger.warn("PROV8005 Invalid date in record: " + line); + logger.debug(e); + } catch (Exception e) { + logger.warn("PROV8006 Invalid pattern in record: " + line); + logger.debug(e); } - } catch (SQLException e) { - logger.warn("PROV8003 Invalid value in record: " + line); - logger.debug(e); - e.printStackTrace(); - } catch (NumberFormatException e) { - logger.warn("PROV8004 Invalid number in record: " + line); - logger.debug(e); - e.printStackTrace(); - } catch (ParseException e) { - logger.warn("PROV8005 Invalid date in record: " + line); - logger.debug(e); - e.printStackTrace(); - } catch (Exception e) { - logger.warn("PROV8006 Invalid pattern in record: " + line); - logger.debug(e); - e.printStackTrace(); + total++; } - total++; } - in.close(); ps.close(); db.release(conn); conn = null;