From 5da50e90e6c78700d48d7468849d4a1599a249b3 Mon Sep 17 00:00:00 2001 From: EmmettCox Date: Tue, 5 Feb 2019 16:16:40 +0000 Subject: [PATCH] adding FILENAME value to LOG_RECORDS table. Change-Id: Iea49d196f324b3a03a18d3d2084e5f6c7ad226ea Issue-ID: DMAAP-1022 Signed-off-by: EmmettCox --- .../src/main/resources/database/sql_init_01.sql | 3 ++- .../org/onap/dmaap/datarouter/node/LogManager.java | 2 +- .../dmaap/datarouter/node/NodeConfigManager.java | 8 ++++---- .../org/onap/dmaap/datarouter/node/StatusLog.java | 2 +- .../provisioning/beans/DeliveryExtraRecord.java | 1 + .../provisioning/beans/DeliveryRecord.java | 1 + .../provisioning/beans/ExpiryRecord.java | 1 + .../datarouter/provisioning/beans/LogRecord.java | 10 +++++++++- .../provisioning/beans/PubFailRecord.java | 1 + .../provisioning/beans/PublishRecord.java | 22 ++++++++++++++-------- .../provisioning/utils/LogfileLoader.java | 2 +- .../src/main/resources/misc/sql_init_01.sql | 3 ++- .../provisioning/beans/PublishRecordTest.java | 2 ++ datarouter-prov/src/test/resources/create.sql | 1 + 14 files changed, 41 insertions(+), 18 deletions(-) diff --git a/datarouter-docker-compose/src/main/resources/database/sql_init_01.sql b/datarouter-docker-compose/src/main/resources/database/sql_init_01.sql index 356a67a9..01258320 100644 --- a/datarouter-docker-compose/src/main/resources/database/sql_init_01.sql +++ b/datarouter-docker-compose/src/main/resources/database/sql_init_01.sql @@ -77,6 +77,7 @@ CREATE TABLE LOG_RECORDS ( RECORD_ID BIGINT UNSIGNED NOT NULL PRIMARY KEY, /* unique ID for this record */ CONTENT_LENGTH_2 BIGINT, + FILENAME VARCHAR(256), /* Name of the file being published on DR */ INDEX (FEEDID) USING BTREE, INDEX (DELIVERY_SUBID) USING BTREE, @@ -134,7 +135,7 @@ INSERT INTO PARAMETERS VALUES ('DELIVERY_MAX_AGE', '86400'), ('DELIVERY_MAX_RETRY_INTERVAL', '3600'), ('DELIVERY_RETRY_RATIO', '2'), - ('LOGROLL_INTERVAL', '300'), + ('LOGROLL_INTERVAL', '30'), ('PROV_AUTH_ADDRESSES', 'dmaap-dr-prov|dmaap-dr-node'), ('PROV_AUTH_SUBJECTS', ''), ('PROV_MAXFEED_COUNT', '10000'), diff --git a/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/LogManager.java b/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/LogManager.java index ee81a321..1d5f76f6 100644 --- a/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/LogManager.java +++ b/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/LogManager.java @@ -188,7 +188,7 @@ public class LogManager extends TimerTask { uploaddir = logdir + "/.spool"; (new File(uploaddir)).mkdirs(); long now = System.currentTimeMillis(); - long intvl = StatusLog.parseInterval(config.getEventLogInterval(), 300000); + long intvl = StatusLog.parseInterval(config.getEventLogInterval(), 30000); long when = now - now % intvl + intvl + 20000L; config.getTimer().scheduleAtFixedRate(this, when - now, intvl); worker = new Uploader(); diff --git a/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/NodeConfigManager.java b/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/NodeConfigManager.java index 50baef5c..474f5dde 100644 --- a/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/NodeConfigManager.java +++ b/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/NodeConfigManager.java @@ -185,7 +185,7 @@ public class NodeConfigManager implements DeliveryQueueHelper { private void localconfig() { followredirects = Boolean.parseBoolean(getProvParam("FOLLOW_REDIRECTS", "false")); - eventloginterval = getProvParam("LOGROLL_INTERVAL", "5m"); + eventloginterval = getProvParam("LOGROLL_INTERVAL", "30s"); initfailuretimer = 10000; maxfailuretimer = 3600000; expirationtimer = 86400000; @@ -365,13 +365,13 @@ public class NodeConfigManager implements DeliveryQueueHelper { * Get a provisioned configuration parameter (from the provisioning server configuration) * * @param name The name of the parameter - * @param deflt The value to use if the parameter is not defined + * @param defaultValue The value to use if the parameter is not defined * @return The value of the parameter or deflt if it is not defined. */ - public String getProvParam(String name, String deflt) { + public String getProvParam(String name, String defaultValue) { name = config.getProvParam(name); if (name == null) { - name = deflt; + name = defaultValue; } return (name); } diff --git a/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/StatusLog.java b/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/StatusLog.java index 43c4fda4..c8a7bd0c 100644 --- a/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/StatusLog.java +++ b/datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/StatusLog.java @@ -53,7 +53,7 @@ public class StatusLog { } /** - * Parse an interval of the form xxhyymzzs and round it to the nearest whole fraction of 24 hours. If no units are specified, assume seconds. + * Parse an interval of the form xxhyymzzs and round it to the nearest whole fraction of 24 hours.If no units are specified, assume seconds. */ public static long parseInterval(String interval, int def) { try { diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryExtraRecord.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryExtraRecord.java index 5aa51302..2a959f3d 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryExtraRecord.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryExtraRecord.java @@ -64,5 +64,6 @@ public class DeliveryExtraRecord extends BaseLogRecord { ps.setNull(16, Types.INTEGER); ps.setNull(17, Types.VARCHAR); ps.setLong(19, contentLength2); + ps.setNull(20, Types.VARCHAR); } } diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryRecord.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryRecord.java index aa7f5d0b..929c6f90 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryRecord.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/DeliveryRecord.java @@ -145,5 +145,6 @@ public class DeliveryRecord extends BaseLogRecord { ps.setNull(16, Types.INTEGER); ps.setNull(17, Types.VARCHAR); ps.setNull(19, Types.BIGINT); + ps.setNull(20, Types.VARCHAR); } } diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/ExpiryRecord.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/ExpiryRecord.java index 3f85aec6..9e6987a5 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/ExpiryRecord.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/ExpiryRecord.java @@ -141,5 +141,6 @@ public class ExpiryRecord extends BaseLogRecord { ps.setInt(16, getAttempts()); ps.setString(17, getReason()); ps.setNull(19, Types.BIGINT); + ps.setNull(20, Types.VARCHAR); } } diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/LogRecord.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/LogRecord.java index 1791bdc8..b7b82729 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/LogRecord.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/LogRecord.java @@ -91,6 +91,7 @@ public class LogRecord extends BaseLogRecord { private final String reason; private final long record_id; private final long clength2; + private final String fileName; public LogRecord(ResultSet rs) throws SQLException { super(rs); @@ -109,6 +110,7 @@ public class LogRecord extends BaseLogRecord { this.record_id = rs.getLong("RECORD_ID"); this.clength2 = rs.getLong("CONTENT_LENGTH_2"); + this.fileName = rs.getString("FILENAME"); } public LogRecord(String[] pp) throws ParseException { @@ -127,7 +129,8 @@ public class LogRecord extends BaseLogRecord { this.reason = pp[17]; this.record_id = Long.parseLong(pp[18]); - this.clength2 = (pp.length == 20) ? Long.parseLong(pp[19]) : 0; + this.clength2 = (pp.length == 21) ? Long.parseLong(pp[19]) : 0; + this.fileName = pp[20]; } public long getRecordId() { @@ -176,6 +179,7 @@ public class LogRecord extends BaseLogRecord { ps.setNull(17, Types.VARCHAR); ps.setLong(18, record_id); ps.setNull(19, Types.BIGINT); + ps.setString(20, fileName); } else if (type.equals("del")) { ps.setNull(9, Types.VARCHAR); ps.setNull(10, Types.VARCHAR); @@ -188,6 +192,7 @@ public class LogRecord extends BaseLogRecord { ps.setNull(17, Types.VARCHAR); ps.setLong(18, record_id); ps.setNull(19, Types.BIGINT); + ps.setString(20, fileName); } else if (type.equals("exp")) { ps.setNull(9, Types.VARCHAR); ps.setNull(10, Types.VARCHAR); @@ -200,6 +205,7 @@ public class LogRecord extends BaseLogRecord { ps.setString(17, reason); ps.setLong(18, record_id); ps.setNull(19, Types.BIGINT); + ps.setString(20, fileName); } else if (type.equals("pbf")) { ps.setString(9, feedFileid); ps.setString(10, remoteAddr); @@ -212,6 +218,7 @@ public class LogRecord extends BaseLogRecord { ps.setNull(17, Types.VARCHAR); ps.setLong(18, record_id); ps.setLong(19, clength2); + ps.setString(20, fileName); } else if (type.equals("dlx")) { ps.setNull(9, Types.VARCHAR); ps.setNull(10, Types.VARCHAR); @@ -224,6 +231,7 @@ public class LogRecord extends BaseLogRecord { ps.setNull(17, Types.VARCHAR); ps.setLong(18, record_id); ps.setLong(19, clength2); + ps.setString(20, fileName); } } diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PubFailRecord.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PubFailRecord.java index cead8b05..88d48826 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PubFailRecord.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PubFailRecord.java @@ -88,5 +88,6 @@ public class PubFailRecord extends BaseLogRecord { ps.setNull(16, Types.INTEGER); ps.setNull(17, Types.VARCHAR); ps.setLong(19, getContentLengthReceived()); + ps.setNull(20, Types.VARCHAR); } } diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecord.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecord.java index e7575ff3..2cde3971 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecord.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecord.java @@ -30,6 +30,7 @@ import java.sql.Types; import java.text.ParseException; import java.util.LinkedHashMap; +import org.apache.commons.lang3.StringUtils; import org.onap.dmaap.datarouter.provisioning.utils.LOGJSONObject; @@ -44,14 +45,10 @@ public class PublishRecord extends BaseLogRecord { private String remoteAddr; private String user; private int status; + private String fileName; public PublishRecord(String[] pp) throws ParseException { super(pp); -// This is too slow! -// Matcher m = Pattern.compile(".*/publish/(\\d+)/(.*)$").matcher(pp[4]); -// if (!m.matches()) -// throw new ParseException("bad pattern", 0); -// this.feedFileid = m.group(2); int ix = pp[4].indexOf("/publish/"); if (ix < 0) throw new ParseException("bad pattern", 0); @@ -62,6 +59,7 @@ public class PublishRecord extends BaseLogRecord { this.remoteAddr = pp[8]; this.user = pp[9]; this.status = Integer.parseInt(pp[10]); + this.fileName = StringUtils.substringAfterLast(this.getRequestUri(), "/"); } public PublishRecord(ResultSet rs) throws SQLException { @@ -70,6 +68,7 @@ public class PublishRecord extends BaseLogRecord { this.remoteAddr = rs.getString("REMOTE_ADDR"); this.user = rs.getString("USER"); this.status = rs.getInt("STATUS"); + this.fileName = StringUtils.substringAfterLast(this.getRequestUri(), "/"); } public String getFeedFileid() { @@ -104,6 +103,10 @@ public class PublishRecord extends BaseLogRecord { this.status = status; } + public String getFileName() { return fileName;} + + public void setFileName(String fileName) { this.fileName = fileName; } + public LOGJSONObject reOrderObject(LOGJSONObject jo) { LinkedHashMap logrecordObj = new LinkedHashMap(); @@ -119,6 +122,7 @@ public class PublishRecord extends BaseLogRecord { logrecordObj.put("type", jo.get("type")); logrecordObj.put("date", jo.get("date")); logrecordObj.put("contentLength", jo.get("contentLength")); + logrecordObj.put("fileName", jo.get("fileName")); LOGJSONObject newjo = new LOGJSONObject(logrecordObj); return newjo; @@ -128,12 +132,13 @@ public class PublishRecord extends BaseLogRecord { public LOGJSONObject asJSONObject() { LOGJSONObject jo = super.asJSONObject(); jo.put("type", "pub"); -// jo.put("feedFileid", feedFileid); -// jo.put("remoteAddr", remoteAddr); -// jo.put("user", user); + jo.put("feedFileid", feedFileid); + jo.put("remoteAddr", remoteAddr); + jo.put("user", user); jo.put("sourceIP", remoteAddr); jo.put("endpointId", user); jo.put("statusCode", status); + jo.put("fileName", fileName); LOGJSONObject newjo = this.reOrderObject(jo); @@ -154,5 +159,6 @@ public class PublishRecord extends BaseLogRecord { ps.setNull(16, Types.INTEGER); ps.setNull(17, Types.VARCHAR); ps.setNull(19, Types.BIGINT); + ps.setString(20, getFileName()); } } diff --git a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java index b6ad8e43..ab1a3a78 100644 --- a/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java +++ b/datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/utils/LogfileLoader.java @@ -100,7 +100,7 @@ public class LogfileLoader extends Thread { /** * The PreparedStatement which is loaded by a Loadable. */ - public static final String INSERT_SQL = "insert into LOG_RECORDS values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + public static final String INSERT_SQL = "insert into LOG_RECORDS values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; /** * Each server can assign this many IDs */ diff --git a/datarouter-prov/src/main/resources/misc/sql_init_01.sql b/datarouter-prov/src/main/resources/misc/sql_init_01.sql index 356a67a9..01258320 100755 --- a/datarouter-prov/src/main/resources/misc/sql_init_01.sql +++ b/datarouter-prov/src/main/resources/misc/sql_init_01.sql @@ -77,6 +77,7 @@ CREATE TABLE LOG_RECORDS ( RECORD_ID BIGINT UNSIGNED NOT NULL PRIMARY KEY, /* unique ID for this record */ CONTENT_LENGTH_2 BIGINT, + FILENAME VARCHAR(256), /* Name of the file being published on DR */ INDEX (FEEDID) USING BTREE, INDEX (DELIVERY_SUBID) USING BTREE, @@ -134,7 +135,7 @@ INSERT INTO PARAMETERS VALUES ('DELIVERY_MAX_AGE', '86400'), ('DELIVERY_MAX_RETRY_INTERVAL', '3600'), ('DELIVERY_RETRY_RATIO', '2'), - ('LOGROLL_INTERVAL', '300'), + ('LOGROLL_INTERVAL', '30'), ('PROV_AUTH_ADDRESSES', 'dmaap-dr-prov|dmaap-dr-node'), ('PROV_AUTH_SUBJECTS', ''), ('PROV_MAXFEED_COUNT', '10000'), diff --git a/datarouter-prov/src/test/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecordTest.java b/datarouter-prov/src/test/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecordTest.java index eaeb91cc..373839ca 100644 --- a/datarouter-prov/src/test/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecordTest.java +++ b/datarouter-prov/src/test/java/org/onap/dmaap/datarouter/provisioning/beans/PublishRecordTest.java @@ -56,6 +56,7 @@ public class PublishRecordTest { publishRecord.setRemoteAddr("172.100.0.4"); publishRecord.setStatus(201); publishRecord.setUser("user2"); + publishRecord.setFileName("fileName"); LOGJSONObject publishRecordJson = createPublishRecordJson(); Assert.assertEquals(publishRecordJson.toString(), publishRecord.asJSONObject().toString()); } @@ -72,6 +73,7 @@ public class PublishRecordTest { publishRecordMap.put("type", "pub"); publishRecordMap.put("date", "2018-08-29T10:10:10.543Z"); publishRecordMap.put("contentLength", 285); + publishRecordMap.put("fileName", "fileName"); return new LOGJSONObject(publishRecordMap); } } diff --git a/datarouter-prov/src/test/resources/create.sql b/datarouter-prov/src/test/resources/create.sql index cd87dde0..b2906035 100755 --- a/datarouter-prov/src/test/resources/create.sql +++ b/datarouter-prov/src/test/resources/create.sql @@ -75,6 +75,7 @@ CREATE TABLE LOG_RECORDS ( RECORD_ID BIGINT UNSIGNED NOT NULL PRIMARY KEY, /* unique ID for this record */ CONTENT_LENGTH_2 BIGINT, + FILENAME VARCHAR(256), /* Name of the file being published on DR */ INDEX (FEEDID) USING BTREE, INDEX (DELIVERY_SUBID) USING BTREE, -- 2.16.6