- // Compare with the range for this server\r
- // Determine the next ID for this set of record IDs\r
- RLEBitSet tbs = (RLEBitSet) nbs.clone();\r
- RLEBitSet idset = new RLEBitSet();\r
- idset.set(set_start, set_start+SET_SIZE);\r
- tbs.and(idset);\r
- long t = tbs.length();\r
- nextid = (t == 0) ? set_start : (t - 1);\r
- if (nextid >= set_start+SET_SIZE) {\r
- // Handle wraparound, when the IDs reach the end of our "range"\r
- Long[] last = null;\r
- Iterator<Long[]> li = tbs.getRangeIterator();\r
- while (li.hasNext()) {\r
- last = li.next();\r
- }\r
- if (last != null) {\r
- tbs.clear(last[0], last[1]+1);\r
- t = tbs.length();\r
- nextid = (t == 0) ? set_start : (t - 1);\r
- }\r
- }\r
- logger.debug(String.format("initializeNextid, next ID is %d (%x)", nextid, nextid));\r
- } catch (SQLException e) {\r
- System.err.println(e);\r
- e.printStackTrace();\r
- } finally {\r
- db.release(conn);\r
- }\r
- }\r
-// OLD CODE - commented here for historical purposes\r
-//\r
-// private boolean pruneRecordsOldAlgorithm() {\r
-// // Determine space available -- available space must be at least 20% under /opt/app/mysql\r
-// int pct = getFreePercentage();\r
-// boolean did1 = false;\r
-// while (pct < REQUIRED_FREE_PCT) {\r
-// logger.info("PROV8008: Free space is " + pct + "% - removing old log entries");\r
-// boolean didit = removeOldestEntries();\r
-// pct = didit ? getFreePercentage() : 100; // don't loop endlessly\r
-// did1 |= didit;\r
-// }\r
-// return did1;\r
-// }\r
-// private int getFreePercentage() {\r
-// FileSystem fs = (Paths.get("/opt/app/mysql")).getFileSystem();\r
-// long total = 0;\r
-// long avail = 0;\r
-// try {\r
-// for (FileStore store : fs.getFileStores()) {\r
-// total += store.getTotalSpace();\r
-// avail += store.getUsableSpace();\r
-// }\r
-// } catch (IOException e) {\r
-// }\r
-// try { fs.close(); } catch (Exception e) { }\r
-// return (int)((avail * 100) / total);\r
-// }\r
-// private boolean removeOldestEntries() {\r
-// // Remove the last days worth of entries\r
-// Connection conn = null;\r
-// try {\r
-// conn = db.getConnection();\r
-// Statement stmt = conn.createStatement();\r
-// ResultSet rs = stmt.executeQuery("select min(event_time) as MIN from LOG_RECORDS");\r
-// if (rs != null) {\r
-// if (rs.next()) {\r
-// // Compute the end of the first day of logs\r
-// long first = rs.getLong("MIN");\r
-// Calendar cal = new GregorianCalendar();\r
-// cal.setTime(new Date(first));\r
-// cal.add(Calendar.DAY_OF_YEAR, 1);\r
-// cal.set(Calendar.HOUR_OF_DAY, 0);\r
-// cal.set(Calendar.MINUTE, 0);\r
-// cal.set(Calendar.SECOND, 0);\r
-// cal.set(Calendar.MILLISECOND, 0);\r
-// if (!stmt.execute("delete from LOG_RECORDS where event_time < " + cal.getTimeInMillis())) {\r
-// int count = stmt.getUpdateCount();\r
-// logger.info("PROV0009: Removed "+count+" old log entries.");\r
-// stmt.execute("OPTIMIZE TABLE LOG_RECORDS");\r
-// }\r
-// rs.close();\r
-// stmt.close();\r
-// return true;\r
-// }\r
-// rs.close();\r
-// }\r
-// stmt.close();\r
-// } catch (SQLException e) {\r
-// System.err.println(e);\r
-// e.printStackTrace();\r
-// } finally {\r
-// db.release(conn);\r
-// }\r
-// return false;\r
-// }\r
- @SuppressWarnings("resource")\r
- private int[] process(File f) {\r
- int ok = 0, total = 0;\r
- try {\r
- Connection conn = db.getConnection();\r
- PreparedStatement ps = conn.prepareStatement(INSERT_SQL);\r
- Reader r = f.getPath().endsWith(".gz")\r
- ? new InputStreamReader(new GZIPInputStream(new FileInputStream(f)))\r
- : new FileReader(f);\r
- LineNumberReader in = new LineNumberReader(r);\r
- String line;\r
- while ((line = in.readLine()) != null) {\r
- try {\r
- for (Loadable rec : buildRecords(line)) {\r
- rec.load(ps);\r
- if (rec instanceof LogRecord) {\r
- LogRecord lr = ((LogRecord)rec);\r
- if (!seq_set.get(lr.getRecordId())) {\r
- ps.executeUpdate();\r
- seq_set.set(lr.getRecordId());\r
- } else\r
- logger.debug("Duplicate record ignored: "+lr.getRecordId());\r
- } else {\r
- if (++nextid > set_end)\r
- nextid = set_start;\r
- ps.setLong(18, nextid);\r
- ps.executeUpdate();\r
- seq_set.set(nextid);\r
- }\r
- ps.clearParameters();\r
- ok++;\r
- }\r
- } catch (SQLException e) {\r
- logger.warn("PROV8003 Invalid value in record: "+line);\r
- logger.debug(e);\r
- e.printStackTrace();\r
- } catch (NumberFormatException e) {\r
- logger.warn("PROV8004 Invalid number in record: "+line);\r
- logger.debug(e);\r
- e.printStackTrace();\r
- } catch (ParseException e) {\r
- logger.warn("PROV8005 Invalid date in record: "+line);\r
- logger.debug(e);\r
- e.printStackTrace();\r
- } catch (Exception e) {\r
- logger.warn("PROV8006 Invalid pattern in record: "+line);\r
- logger.debug(e);\r
- e.printStackTrace();\r
- }\r
- total++;\r
- }\r
- in.close();\r
- ps.close();\r
- db.release(conn);\r
- conn = null;\r
- } catch (FileNotFoundException e) {\r
- logger.warn("PROV8007 Exception reading "+f+": "+e);\r
- } catch (IOException e) {\r
- logger.warn("PROV8007 Exception reading "+f+": "+e);\r
- } catch (SQLException e) {\r
- logger.warn("PROV8007 Exception reading "+f+": "+e);\r
- }\r
- return new int[] { ok, total };\r
- }\r
- private Loadable[] buildRecords(String line) throws ParseException {\r
- String[] pp = line.split("\\|");\r
- if (pp != null && pp.length >= 7) {\r
- String rtype = pp[1].toUpperCase();\r
- if (rtype.equals("PUB") && pp.length == 11) {\r
- // Fields are: date|PUB|pubid|feedid|requrl|method|ctype|clen|srcip|user|status\r
- return new Loadable[] { new PublishRecord(pp) };\r
- }\r
- if (rtype.equals("DEL") && pp.length == 12) {\r
- // Fields are: date|DEL|pubid|feedid|subid|requrl|method|ctype|clen|user|status|xpubid\r
- String[] subs = pp[4].split("\\s+");\r
- if (subs != null) {\r
- Loadable[] rv = new Loadable[subs.length];\r
- for (int i = 0; i < subs.length; i++) {\r
- // create a new record for each individual sub\r
- pp[4] = subs[i];\r
- rv[i] = new DeliveryRecord(pp);\r
- }\r
- return rv;\r
- }\r
- }\r
- if (rtype.equals("EXP") && pp.length == 11) {\r
- // Fields are: date|EXP|pubid|feedid|subid|requrl|method|ctype|clen|reason|attempts\r
- ExpiryRecord e = new ExpiryRecord(pp);\r
- if (e.getReason().equals("other"))\r
- logger.info("Invalid reason '"+pp[9]+"' changed to 'other' for record: "+e.getPublishId());\r
- return new Loadable[] { e };\r
- }\r
- if (rtype.equals("PBF") && pp.length == 12) {\r
- // Fields are: date|PBF|pubid|feedid|requrl|method|ctype|clen-expected|clen-received|srcip|user|error\r
- return new Loadable[] { new PubFailRecord(pp) };\r
- }\r
- if (rtype.equals("DLX") && pp.length == 7) {\r
- // Fields are: date|DLX|pubid|feedid|subid|clen-tosend|clen-sent\r
- return new Loadable[] { new DeliveryExtraRecord(pp) };\r
- }\r
- if (rtype.equals("LOG") && (pp.length == 19 || pp.length == 20)) {\r
- // Fields are: date|LOG|pubid|feedid|requrl|method|ctype|clen|type|feedFileid|remoteAddr|user|status|subid|fileid|result|attempts|reason|record_id\r
- return new Loadable[] { new LogRecord(pp) };\r
- }\r
- }\r
- logger.warn("PROV8002 bad record: "+line);\r
- return new Loadable[0];\r
- }\r
+ private Loadable[] buildRecords(String line) throws ParseException {\r
+ String[] pp = line.split("\\|");\r
+ if (pp != null && pp.length >= 7) {\r
+ String rtype = pp[1].toUpperCase();\r
+ if (rtype.equals("PUB") && pp.length == 11) {\r
+ // Fields are: date|PUB|pubid|feedid|requrl|method|ctype|clen|srcip|user|status\r
+ return new Loadable[]{new PublishRecord(pp)};\r
+ }\r
+ if (rtype.equals("DEL") && pp.length == 12) {\r
+ // Fields are: date|DEL|pubid|feedid|subid|requrl|method|ctype|clen|user|status|xpubid\r
+ String[] subs = pp[4].split("\\s+");\r
+ if (subs != null) {\r
+ Loadable[] rv = new Loadable[subs.length];\r
+ for (int i = 0; i < subs.length; i++) {\r
+ // create a new record for each individual sub\r
+ pp[4] = subs[i];\r
+ rv[i] = new DeliveryRecord(pp);\r
+ }\r
+ return rv;\r
+ }\r
+ }\r
+ if (rtype.equals("EXP") && pp.length == 11) {\r
+ // Fields are: date|EXP|pubid|feedid|subid|requrl|method|ctype|clen|reason|attempts\r
+ ExpiryRecord e = new ExpiryRecord(pp);\r
+ if (e.getReason().equals("other"))\r
+ logger.info("Invalid reason '" + pp[9] + "' changed to 'other' for record: " + e.getPublishId());\r
+ return new Loadable[]{e};\r
+ }\r
+ if (rtype.equals("PBF") && pp.length == 12) {\r
+ // Fields are: date|PBF|pubid|feedid|requrl|method|ctype|clen-expected|clen-received|srcip|user|error\r
+ return new Loadable[]{new PubFailRecord(pp)};\r
+ }\r
+ if (rtype.equals("DLX") && pp.length == 7) {\r
+ // Fields are: date|DLX|pubid|feedid|subid|clen-tosend|clen-sent\r
+ return new Loadable[]{new DeliveryExtraRecord(pp)};\r
+ }\r
+ if (rtype.equals("LOG") && (pp.length == 19 || pp.length == 20)) {\r
+ // Fields are: date|LOG|pubid|feedid|requrl|method|ctype|clen|type|feedFileid|remoteAddr|user|status|subid|fileid|result|attempts|reason|record_id\r
+ return new Loadable[]{new LogRecord(pp)};\r
+ }\r
+ }\r
+ logger.warn("PROV8002 bad record: " + line);\r
+ return new Loadable[0];\r
+ }\r