2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.onap.xacml.parser;
24 import java.io.FileInputStream;
25 import java.io.FileReader;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.LineNumberReader;
29 import java.io.RandomAccessFile;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.Paths;
34 import java.sql.Connection;
35 import java.sql.DriverManager;
36 import java.sql.PreparedStatement;
37 import java.sql.SQLException;
38 import java.text.Format;
39 import java.text.ParseException;
40 import java.text.SimpleDateFormat;
41 import java.util.Date;
42 import java.util.Properties;
43 import java.util.Timer;
44 import java.util.regex.Matcher;
45 import java.util.regex.Pattern;
46 import java.util.stream.Stream;
48 import org.apache.log4j.Logger;
49 import org.onap.policy.common.im.AdministrativeStateException;
50 import org.onap.policy.common.im.IntegrityMonitor;
51 import org.onap.policy.common.im.StandbyStatusException;
52 import org.onap.policy.common.logging.flexlogger.FlexLogger;
53 import org.onap.xacml.parser.LogEntryObject.LOGTYPE;
56 * Parse log files and store the information in a H2 database.
60 public class ParseLog {
62 // only logging last line of each log file processed to the log4j log file defined by property - PARSERLOGPATH
63 private static final Logger log4jlogger = Logger.getLogger(ParseLog.class.getName());
66 private static org.onap.policy.common.logging.flexlogger.Logger logger = FlexLogger.getLogger(ParseLog.class.getName());
68 private static String system;
69 private static int lastNumberRead = 0;
70 private static int debuglastNumberRead = 0;
71 private static int errorlastNumberRead = 0;
72 private static String type;
73 private static long startFileSize;
74 private static long debugStartFileSize;
75 private static long errorStartFileSize;
76 private static String systemLogFile;
77 private static String logFile;
78 private static String debuglogFile;
79 private static String errorlogFile;
80 private static String JDBC_URL;
81 private static String JDBC_USER;
82 private static String JDBC_PASSWORD;
83 private static String JDBC_DRIVER;
84 private static int maxLength = 255; //Max length that is allowed in the DB table
85 private static String resourceName;
86 private static long sleepTimer = 50000;
87 static IntegrityMonitor im;
88 private static boolean isMissingLogFile;
89 //Default:Timer initial delay and the delay between in milliseconds before task is to be execute
90 private static int TIMER_DELAY_TIME = 1000;
91 //Default:Timer scheduleAtFixedRate period - time in milliseconds between successive task executions
92 private static int CHECK_INTERVAL = 86400000; // run this clean up once a day
93 private static RandomAccessFile randomAccessFile;
94 private static String loggingProcess = "Error processing line in ";
95 private static int TIME_FRAME = 5;
96 private static String message =" value read in: ";
97 private static String lineFormat = "(\\r\\n|\\n)";
98 private static String lineRead = "-line-Read:";
99 private static String br= "<br />";
100 private static String last = "Last-";
101 private static String breakLoop = "break the loop.";
102 private static String dateFormat = "yyyy-MM-dd HH:mm:ss";
104 public static void main(String[] args) throws Exception {
106 Properties logProperties = getPropertiesValue("parserlog.properties");
108 if(logProperties == null || isMissingLogFile){
109 // missing the path of log file in the properties file, so stop the process
110 logger.error("logProperties is null or LOGPATH is missing in parserlog.properties, so stop the process.");
114 //trigger the cleanup systemLogDb timer
117 File fileLog = new File(systemLogFile);
119 im = IntegrityMonitor.getInstance(resourceName,logProperties );
121 startDebugLogParser(fileLog);
122 startErrorLogParser(fileLog);
123 startAPIRestLogParser(fileLog);
127 private static boolean processLine(Path debugfilePath, String dataFileName, int lastNmRead, LOGTYPE logType){
128 // log4jlogger must use .info
129 Stream<String> lines = null;
131 lines = Files.lines(debugfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info(last+dataFileName+ lineRead + lastNmRead)).skip(lastNmRead);
132 lines.forEachOrdered(line -> process(line, type, logType));
133 } catch (IOException e) {
134 logger.error(loggingProcess + dataFileName, e);
135 logger.error(breakLoop);
145 private static void processDebugLogParser(File debugfile, Path debugfilePath, String dataFileName){
147 Runnable runnable = new Runnable (){
148 boolean isStop = false;
151 if (debugfile.isFile()){
152 isStop = processLine(debugfilePath, dataFileName, debuglastNumberRead, LOGTYPE.DEBUG);
155 Thread.sleep(sleepTimer);
156 debugStartFileSize = countLines(debuglogFile);
157 } catch (Exception e) {
158 logger.error(loggingProcess + dataFileName, e);
159 logger.error(breakLoop);
162 logger.debug("File Line Count of debug.log: " + debugStartFileSize + message + debuglastNumberRead);
163 if (debugStartFileSize < debuglastNumberRead ){
164 logger.debug("Failed Rolled: set Last number read to 0");
165 debuglastNumberRead = 0;
171 Thread thread = new Thread(runnable);
175 private static void startDebugLogParser(File fileLog) throws IOException{
177 if(debuglogFile != null && !debuglogFile.isEmpty()){
179 // pull the last line number
180 String dataFileName = "debug.log";
181 String filesRead = pullLastLineRead(fileLog, dataFileName);
182 if (filesRead!= null){
183 filesRead = filesRead.replaceAll(lineFormat, br);
184 debuglastNumberRead= Integer.parseInt(filesRead.trim());
186 debuglastNumberRead = 0;
189 debugStartFileSize = countLines(debuglogFile);
190 if (debugStartFileSize < debuglastNumberRead ){
191 logger.error("Filed Rolled: set Last debug number read to 0");
192 debuglastNumberRead = 0;
195 isMissingLogFile = false;
196 Path debugfilePath = Paths.get(debuglogFile);
197 File debugfile = new File(debuglogFile);
198 debugStartFileSize = debugfile.length();
200 // start process debug.log file
201 processDebugLogParser(debugfile, debugfilePath, dataFileName);
206 private static void processErrorLogParser(File errorfile, Path errorfilePath, String dataFileName){
207 Runnable runnable = new Runnable (){
208 boolean isStop = false;
212 if (errorfile.isFile()){
213 isStop = processLine(errorfilePath, dataFileName, errorlastNumberRead, LOGTYPE.ERROR);
216 Thread.sleep(sleepTimer);
217 errorStartFileSize = countLines(errorlogFile);
218 } catch (Exception e) {
219 logger.error(loggingProcess + dataFileName, e);
220 logger.error(breakLoop);
224 logger.debug("File Line Count of error.log: " + errorStartFileSize + message + errorlastNumberRead);
225 if (errorStartFileSize < errorlastNumberRead ){
226 logger.debug("Failed Rolled: set Last error number read to 0");
227 errorlastNumberRead = 0;
233 Thread thread = new Thread(runnable);
237 private static void startErrorLogParser(File fileLog) throws IOException{
239 if(errorlogFile != null && !errorlogFile.isEmpty()){
241 // pull the last line number
242 String dataFileName = "error.log";
243 String filesRead = pullLastLineRead(fileLog, dataFileName);
244 if (filesRead!= null){
245 filesRead = filesRead.replaceAll(lineFormat, br);
246 errorlastNumberRead= Integer.parseInt(filesRead.trim());
248 errorlastNumberRead = 0;
251 errorStartFileSize = countLines(errorlogFile);
252 if (errorStartFileSize < errorlastNumberRead ){
253 logger.error("Filed Rolled: set Last error number read to 0");
254 errorlastNumberRead = 0;
257 isMissingLogFile = false;
258 Path errorfilePath = Paths.get(errorlogFile);
259 File errorfile = new File(errorlogFile);
260 errorStartFileSize = errorfile.length();
261 // start process error.log file
262 processErrorLogParser(errorfile, errorfilePath, dataFileName);
267 private static void processAPIRestLog(File file, Path filePath, String dataFileName){
269 Runnable runnable = new Runnable () {
270 boolean isStop = false;
275 isStop = processLine(filePath, dataFileName, lastNumberRead, LOGTYPE.INFO);
278 Thread.sleep(sleepTimer);
279 startFileSize = countLines(logFile);
280 } catch (Exception e) {
281 logger.error(loggingProcess + dataFileName, e);
282 logger.error(breakLoop);
286 logger.debug("File Line Count of " + dataFileName+": " + startFileSize + message + lastNumberRead);
287 if (startFileSize < lastNumberRead ){
288 logger.debug("Failed Rolled: set Last number read to 0");
295 Thread thread = new Thread(runnable);
299 private static void startAPIRestLogParser(File fileLog) throws IOException{
301 if(logFile != null && !logFile.isEmpty()){
303 // pull the last line number
304 String dataFileName = type.toLowerCase()+"-rest.log";
305 String filesRead = pullLastLineRead(fileLog, dataFileName);
306 if (filesRead!= null){
307 filesRead = filesRead.replaceAll(lineFormat, br);
308 lastNumberRead= Integer.parseInt(filesRead.trim());
312 startFileSize = countLines(logFile);
313 if (startFileSize < lastNumberRead ){
314 logger.error("Filed Rolled: set Last number read to 0");
318 isMissingLogFile = false;
319 Path filePath = Paths.get(logFile);
320 File file = new File(logFile);
321 startFileSize = file.length();
322 // start process pap/pdp-rest.log file
323 processAPIRestLog(file, filePath, dataFileName);
327 public static int countLines(String filename){
330 FileReader freader = new FileReader(filename);
331 LineNumberReader reader = new LineNumberReader(freader);
334 while ((line = reader.readLine()) != null) {
335 logger.debug("Reading the Logs"+line);
337 cnt = reader.getLineNumber();
338 logger.info("Line number:"+cnt);
349 public static String pullLastLineRead(File file, String dataFileName) throws IOException {
351 file.createNewFile();
354 randomAccessFile = new RandomAccessFile(file, "r");
355 StringBuilder builder = new StringBuilder();
356 long length = file.length();
357 logger.debug("dataFileName: " +dataFileName);
360 randomAccessFile.seek(length);
361 for(long seek = length; seek >= 0; --seek){
362 randomAccessFile.seek(seek);
363 char c = (char)randomAccessFile.read();
366 builder = builder.reverse();
367 logger.debug("builder.toString(): " +builder.toString());
368 if (builder.toString().contains(last+dataFileName+lineRead)){
369 String[] parseString = builder.toString().split(last+dataFileName+lineRead);
370 String returnValue = parseString[1].replace("\r", "");
371 return returnValue.trim();
373 builder = new StringBuilder();
381 private static LogEntryObject getDebugOutLogValue (String line, String type){
384 LogEntryObject logEntry = new LogEntryObject();
385 logEntry.setSystemType(type);
386 logEntry.setSystem(system);
387 String info1 = "||INFO||";
388 String info2 = "INFO:";
389 String error1 = "||ERROR||";
390 String error2 = "ERROR:";
392 if(line.contains(info1) || line.contains(error1) || line.contains(info2) || line.contains(error2)){
393 String[] splitString = null;
394 if(line.contains(info1) || line.contains(error1)){
395 splitString = line.split("[||]");
396 }else if(line.contains(info2)){
397 splitString = line.split(info2);
399 splitString = line.split(error2);
401 String dateString = splitString[0].substring(0, 19);
402 logEntry.setDescription(splitString[splitString.length-1]);
405 date = parseDate(dateString.replace("T", " "), dateFormat, false);
406 logEntry.setDate(date);
408 logEntry.setRemote(parseRemoteSystem(line));
409 if (line.contains(info2) || line.contains(info1)){
410 logEntry.setLogType(LOGTYPE.INFO);
412 logEntry.setLogType(LOGTYPE.ERROR);
421 private static LogEntryObject getRestAPIOutLogValue (String line, String type){
423 LogEntryObject logEntry = new LogEntryObject();
424 logEntry.setSystemType(type);
425 logEntry.setSystem(system);
426 String info3 = "INFO";
428 // from PDP/PAP rest log file below
429 if (line.contains(info3) && line.contains(")-")){
430 //parse out description
431 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
433 date = parseDate(line, dateFormat, true);
434 logEntry.setDate(date);
436 logEntry.setRemote(parseRemoteSystem(line));
437 logEntry.setLogType(LOGTYPE.INFO);
445 private static LogEntryObject getInfoOutLogValue (String line, String type){
447 LogEntryObject logEntry = new LogEntryObject();
448 logEntry.setSystemType(type);
449 logEntry.setSystem(system);
450 String info3 = "INFO";
452 if (line.contains(info3) && line.contains("--- [")){
453 //parse out description
454 String temp = line.substring(line.indexOf("---")+1);
455 String[] split = temp.split(":");
457 logEntry.setDescription(split[1]);
460 date = parseDate(line, dateFormat, false);
461 logEntry.setDate(date);
464 logEntry.setRemote(parseRemoteSystem(line));
465 logEntry.setLogType(LOGTYPE.INFO);
473 private static LogEntryObject getSevereOutLogValue (String line, String type){
475 LogEntryObject logEntry = new LogEntryObject();
476 logEntry.setSystemType(type);
477 logEntry.setSystem(system);
478 if (line.contains("SEVERE") && line.contains("[main]")){
479 String[] splitString = line.split(" ");
480 StringBuilder description = new StringBuilder();
481 for (int i = 5; i < splitString.length; i++){
482 description.append(" " + splitString[i]);
485 logEntry.setDescription(description.toString());
487 date = parseDate(line, dateFormat, false);
488 logEntry.setDate(date);
489 logEntry.setLogType(LOGTYPE.SEVERE);
494 if (line.contains("ERROR") && line.contains(")-")){
495 //parse out description
496 StringBuilder description = new StringBuilder();
497 description.append(line.substring(line.indexOf(")-")+3));
499 date = parseDate(line, dateFormat, true);
500 logEntry.setDate(date);
501 logEntry.setDescription(description.toString());
503 logEntry.setRemote(parseRemoteSystem(line));
504 logEntry.setLogType(LOGTYPE.ERROR);
512 private static LogEntryObject getWarnOutLogValue (String line, String type){
514 LogEntryObject logEntry = new LogEntryObject();
515 logEntry.setSystemType(type);
516 logEntry.setSystem(system);
517 if (line.contains("WARN") && line.contains(")-")){
518 //parse out description
520 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
523 date = parseDate(line, dateFormat, true);
524 logEntry.setDate(date);
527 logEntry.setRemote(parseRemoteSystem(line));
528 logEntry.setLogType(LOGTYPE.WARN);
533 if (line.contains("WARNING") && type =="PyPDP"){
534 String[] splitString = line.split(" ");
535 StringBuilder description = new StringBuilder();
537 for (int i = 5; i < splitString.length; i++){
538 description.append(" " + splitString[i]);
542 date = parseDate(line, dateFormat, false);
543 logEntry.setDate(date);
544 logEntry.setLogType(LOGTYPE.WARN);
545 logEntry.setDescription(description.toString());
552 public static LogEntryObject pullOutLogValues(String line, String type){
554 LogEntryObject logEntry = getDebugOutLogValue(line, type);
556 if(logEntry == null){
557 logEntry = getRestAPIOutLogValue(line, type);
559 if(logEntry == null){
560 logEntry = getInfoOutLogValue(line, type);
562 if(logEntry == null){
563 logEntry = getSevereOutLogValue(line, type);
565 if(logEntry == null){
566 logEntry = getWarnOutLogValue(line, type);
572 private static void dbClose(Connection conn) {
575 } catch (SQLException e) {
576 logger.error("Error closing DB Connection: " + e);
581 public static void process(String line, String type, LOGTYPE logFile) {
583 LogEntryObject returnLogValue = null;
586 im.startTransaction();
587 } catch (AdministrativeStateException e) {
588 logger.error("Error received" + e);
589 } catch (StandbyStatusException ex) {
590 logger.error("Error received" + ex);
593 returnLogValue = pullOutLogValues(line, type);
595 if(logFile.equals(LOGTYPE.DEBUG)){
596 debuglastNumberRead++;
597 }else if(logFile.equals(LOGTYPE.ERROR)){
598 errorlastNumberRead++;
599 }else if(logFile.equals(LOGTYPE.INFO)){
602 if (returnLogValue!=null){
603 writeDB(returnLogValue);
610 private static void writeDB(LogEntryObject returnLogValue) {
612 Connection conn = dbConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
613 dbAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
614 returnLogValue.getDate(), returnLogValue.getRemote(),
615 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
619 private static Connection dbConnection(String driver, String jdbc, String user, String pass){
622 Class.forName(driver);
623 Connection conn = DriverManager.getConnection(jdbc, user, pass);
625 } catch ( Exception e) {
626 logger.error("Error connecting to DB: " + e);
630 private static void dbAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
634 Format formatter = new SimpleDateFormat(dateFormat);
635 sdate = formatter.format(date);
636 logger.debug("DBAccesss : sdate : " + sdate);
638 logger.debug("DBAccesss : sdate is null");
641 //ensure the length of description is less than the maximumm db char length
642 if (description.length() > maxLength) {
643 description = description.substring(0, maxLength);
647 PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
650 prep.setString(1, system);
651 prep.setString(2, description);
652 prep.setString(3, remote);
653 prep.setString(4, type);
654 prep.setString(5, sdate);
655 prep.setString(6, logType);
657 prep.executeUpdate();
660 } catch (SQLException e1) {
661 logger.error("Error trying to excute SQL Statment: " + e1);
665 public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
668 String[] splitString = dateline.split(" ");
669 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
672 returnDate = formatter.parse(dateline);
673 } catch (ParseException e) {
674 logger.error("Unable to parse date for line: " + dateline);
678 String tmpString = splitString[0] + " " + splitString[1];
680 returnDate = formatter.parse(tmpString);
681 } catch (ParseException e) {
682 logger.error("Unable to parse date for line: " + dateline);
691 public static String parseRemoteSystem(String line) {
693 if (line.contains("http") && !(line.contains("www.w3.org"))){
695 Pattern pattern = Pattern.compile("://(.+?)/");
696 Matcher remote = pattern.matcher(line);
699 return remote.group(1);
705 public static String[] getPaths(String logPath){
706 String[] oneFile = null;
707 if(logPath != null && !logPath.isEmpty()){
708 if(logPath.contains(";")){
709 return logPath.split(";");
711 oneFile = new String[1];
712 oneFile[0] = logPath;
719 private static void setCleanUpProperties(String cleanupInterval, String timeFrame){
720 if(cleanupInterval != null && !cleanupInterval.isEmpty()){
721 int intCheckInterval = Integer.parseInt(cleanupInterval);
722 if(intCheckInterval > 300000) {//must be longer than 5 minutes
723 CHECK_INTERVAL = intCheckInterval;
726 logger.debug("No value defined for CHECK_INTERVAL in parserlog.properties, so use its default value:" + CHECK_INTERVAL + " milliseconds");
729 if(timeFrame != null && !timeFrame.isEmpty()){
730 int intTimeFrame = Integer.parseInt(timeFrame);
731 if(intTimeFrame > 0){
732 TIME_FRAME = intTimeFrame;
735 logger.debug("No value defined for TIME_FRAME in parserlog.properties, so use its default value:" + TIME_FRAME + " days");
739 private static void setDebuglogFile(String fileName){
740 debuglogFile = fileName;
741 if(debuglogFile != null && !debuglogFile.isEmpty()){
742 debuglogFile = debuglogFile.trim();
748 private static void setErrorlogFile(String fileName){
749 errorlogFile = fileName;
750 if(errorlogFile != null && !errorlogFile.isEmpty()){
751 errorlogFile = errorlogFile.trim();
757 private static void setLogFileProperties(String[] splitString){
758 if(splitString != null){
759 for(int i=0; i < splitString.length; i++){
761 if(splitString[i].contains("debug")){
762 // get path of debug.log file
763 setDebuglogFile(splitString[i]);
764 }else if(splitString[i].contains("error")){
765 // get path of error.log file
766 setErrorlogFile(splitString[i]);
768 // get path of default file
769 logFile = splitString[i];
770 if(logFile != null && !logFile.isEmpty()){
771 logFile = logFile.trim();
780 public static Properties getPropertiesValue(String fileName) {
781 Properties config = new Properties();
782 Path file = Paths.get(fileName);
783 if (file.toFile().exists()) {
785 if (file.toString().endsWith(".properties")) {
788 in = new FileInputStream(file.toFile());
791 resourceName = config.getProperty("RESOURCE_NAME");
792 system = config.getProperty("SERVER");
793 type = config.getProperty("LOGTYPE");
794 systemLogFile = config.getProperty("PARSERLOGPATH");
795 String logFiles = config.getProperty("LOGPATH");
796 String cleanupInterval= config.getProperty("CHECK_INTERVAL");
797 String timeFrame = config.getProperty("TIME_FRAME");
799 setCleanUpProperties(cleanupInterval, timeFrame);
801 if(logFiles == null || logFiles.isEmpty()){
802 isMissingLogFile = true;
806 String[] splitString = getPaths(logFiles);
808 setLogFileProperties(splitString);
810 JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
811 JDBC_USER = config.getProperty("JDBC_USER");
812 JDBC_DRIVER = config.getProperty("JDBC_DRIVER");
813 JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
816 } catch (IOException e) {
817 logger.error("Error porcessing Config file will be unable to create Health Check" + e);
819 logger.error("Error getPropertiesValue on TIME_FRAME", e);
820 logger.debug("Error getPropertiesValue on TIME_FRAME, so use its default value:" + TIME_FRAME + " days");
826 logger.debug("File doesn't exist in the specified Path " + file.toString());
831 public static Connection getDbConnection(){
832 return dbConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
834 private static void startCleanUp(){
835 Connection conn = dbConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
836 CleanUpSystemLogDB cleanUp = new CleanUpSystemLogDB(conn, TIME_FRAME);
837 Timer timer = new Timer(true);
838 timer.scheduleAtFixedRate(cleanUp, TIMER_DELAY_TIME, CHECK_INTERVAL);
839 logger.info("startCleanUp begins! : " + new Date());