2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.onap.xacml.parser;
24 import java.io.FileInputStream;
25 import java.io.FileReader;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.LineNumberReader;
29 import java.io.RandomAccessFile;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.Paths;
34 import java.sql.Connection;
35 import java.sql.DriverManager;
36 import java.sql.PreparedStatement;
37 import java.sql.SQLException;
38 import java.text.Format;
39 import java.text.ParseException;
40 import java.text.SimpleDateFormat;
41 import java.util.Date;
42 import java.util.Properties;
43 import java.util.Timer;
44 import java.util.regex.Matcher;
45 import java.util.regex.Pattern;
46 import java.util.stream.Stream;
48 import org.apache.log4j.Logger;
49 import org.onap.policy.common.im.IntegrityMonitor;
50 import org.onap.policy.common.im.IntegrityMonitorException;
51 import org.onap.policy.common.logging.flexlogger.FlexLogger;
52 import org.onap.policy.utils.CryptoUtils;
53 import org.onap.xacml.parser.LogEntryObject.LOGTYPE;
56 * Parse log files and store the information in a H2 database.
60 public class ParseLog {
62 // only logging last line of each log file processed to the log4j log file defined by property - PARSERLOGPATH
63 private static final Logger log4jlogger = Logger.getLogger(ParseLog.class.getName());
66 private static org.onap.policy.common.logging.flexlogger.Logger logger = FlexLogger.getLogger(ParseLog.class.getName());
68 private static String system;
69 private static int lastNumberRead = 0;
70 private static int debuglastNumberRead = 0;
71 private static int errorlastNumberRead = 0;
72 private static String type;
73 private static long startFileSize;
74 private static long debugStartFileSize;
75 private static long errorStartFileSize;
76 private static String systemLogFile;
77 private static String logFile;
78 private static String debuglogFile;
79 private static String errorlogFile;
80 private static String jdbcUrl;
81 private static String jdbcUser;
82 private static String jdbcPassword;
83 private static String jdbcDriver;
84 private static int maxLength = 255; //Max length that is allowed in the DB table
85 private static String resourceName;
86 private static long sleepTimer = 50000;
87 static IntegrityMonitor im;
88 private static boolean isMissingLogFile;
89 //Default:Timer initial delay and the delay between in milliseconds before task is to be execute
90 private static final int TIMER_DELAY_TIME = 1000;
91 //Default:Timer scheduleAtFixedRate period - time in milliseconds between successive task executions
92 private static int checkInterval = 86400000; // run this clean up once a day
93 private static String loggingProcess = "Error processing line in ";
94 private static int defaultTimeFrame = 5;
95 private static String message =" value read in: ";
96 private static String lineFormat = "(\\r\\n|\\n)";
97 private static String lineRead = "-line-Read:";
98 private static String br= "<br />";
99 private static String last = "Last-";
100 private static String breakLoop = "break the loop.";
101 private static String dateFormat = "yyyy-MM-dd HH:mm:ss";
103 public static void main(String[] args) throws Exception {
105 Properties logProperties = getPropertiesValue("parserlog.properties");
107 if(logProperties == null || isMissingLogFile){
108 // missing the path of log file in the properties file, so stop the process
109 logger.error("logProperties is null or LOGPATH is missing in parserlog.properties, so stop the process.");
113 //trigger the cleanup systemLogDb timer
116 File fileLog = new File(systemLogFile);
118 im = IntegrityMonitor.getInstance(resourceName,logProperties );
120 startDebugLogParser(fileLog);
121 startErrorLogParser(fileLog);
122 startAPIRestLogParser(fileLog);
126 private static boolean processLine(Path debugfilePath, String dataFileName, int lastNmRead, LOGTYPE logType){
127 // log4jlogger must use .info
128 try(Stream<String> lines = Files.lines(debugfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info(last+dataFileName+ lineRead + lastNmRead)).skip(lastNmRead)){
129 lines.forEachOrdered(line -> process(line, type, logType));
130 } catch (IOException e) {
131 logger.error(loggingProcess + dataFileName, e);
132 logger.error(breakLoop);
137 private static void processDebugLogParser(File debugfile, Path debugfilePath, String dataFileName){
139 Runnable runnable = new Runnable (){
140 boolean isStop = false;
143 if (debugfile.isFile()){
144 isStop = processLine(debugfilePath, dataFileName, debuglastNumberRead, LOGTYPE.DEBUG);
147 Thread.sleep(sleepTimer);
148 debugStartFileSize = countLines(debuglogFile);
149 } catch (Exception e) {
150 logger.error(loggingProcess + dataFileName, e);
151 logger.error(breakLoop);
154 logger.debug("File Line Count of debug.log: " + debugStartFileSize + message + debuglastNumberRead);
155 if (debugStartFileSize < debuglastNumberRead ){
156 logger.debug("Failed Rolled: set Last number read to 0");
157 debuglastNumberRead = 0;
163 Thread thread = new Thread(runnable);
167 private static void startDebugLogParser(File fileLog) throws IOException{
169 if(debuglogFile != null && !debuglogFile.isEmpty()){
171 // pull the last line number
172 String dataFileName = "debug.log";
173 String filesRead = pullLastLineRead(fileLog, dataFileName);
174 if (filesRead!= null){
175 filesRead = filesRead.replaceAll(lineFormat, br);
176 debuglastNumberRead= Integer.parseInt(filesRead.trim());
178 debuglastNumberRead = 0;
181 debugStartFileSize = countLines(debuglogFile);
182 if (debugStartFileSize < debuglastNumberRead ){
183 logger.error("Filed Rolled: set Last debug number read to 0");
184 debuglastNumberRead = 0;
187 isMissingLogFile = false;
188 Path debugfilePath = Paths.get(debuglogFile);
189 File debugfile = new File(debuglogFile);
190 debugStartFileSize = debugfile.length();
192 // start process debug.log file
193 processDebugLogParser(debugfile, debugfilePath, dataFileName);
198 private static void processErrorLogParser(File errorfile, Path errorfilePath, String dataFileName){
199 Runnable runnable = new Runnable (){
200 boolean isStop = false;
204 if (errorfile.isFile()){
205 isStop = processLine(errorfilePath, dataFileName, errorlastNumberRead, LOGTYPE.ERROR);
208 Thread.sleep(sleepTimer);
209 errorStartFileSize = countLines(errorlogFile);
210 } catch (Exception e) {
211 logger.error(loggingProcess + dataFileName, e);
212 logger.error(breakLoop);
216 logger.debug("File Line Count of error.log: " + errorStartFileSize + message + errorlastNumberRead);
217 if (errorStartFileSize < errorlastNumberRead ){
218 logger.debug("Failed Rolled: set Last error number read to 0");
219 errorlastNumberRead = 0;
225 Thread thread = new Thread(runnable);
229 private static void startErrorLogParser(File fileLog) throws IOException{
231 if(errorlogFile != null && !errorlogFile.isEmpty()){
233 // pull the last line number
234 String dataFileName = "error.log";
235 String filesRead = pullLastLineRead(fileLog, dataFileName);
236 if (filesRead!= null){
237 filesRead = filesRead.replaceAll(lineFormat, br);
238 errorlastNumberRead= Integer.parseInt(filesRead.trim());
240 errorlastNumberRead = 0;
243 errorStartFileSize = countLines(errorlogFile);
244 if (errorStartFileSize < errorlastNumberRead ){
245 logger.error("Filed Rolled: set Last error number read to 0");
246 errorlastNumberRead = 0;
249 isMissingLogFile = false;
250 Path errorfilePath = Paths.get(errorlogFile);
251 File errorfile = new File(errorlogFile);
252 errorStartFileSize = errorfile.length();
253 // start process error.log file
254 processErrorLogParser(errorfile, errorfilePath, dataFileName);
259 private static void processAPIRestLog(File file, Path filePath, String dataFileName){
261 Runnable runnable = new Runnable () {
262 boolean isStop = false;
267 isStop = processLine(filePath, dataFileName, lastNumberRead, LOGTYPE.INFO);
270 Thread.sleep(sleepTimer);
271 startFileSize = countLines(logFile);
272 } catch (Exception e) {
273 logger.error(loggingProcess + dataFileName, e);
274 logger.error(breakLoop);
278 logger.debug("File Line Count of " + dataFileName+": " + startFileSize + message + lastNumberRead);
279 if (startFileSize < lastNumberRead ){
280 logger.debug("Failed Rolled: set Last number read to 0");
287 Thread thread = new Thread(runnable);
291 private static void startAPIRestLogParser(File fileLog) throws IOException{
293 if(logFile != null && !logFile.isEmpty()){
295 // pull the last line number
296 String dataFileName = type.toLowerCase()+"-rest.log";
297 String filesRead = pullLastLineRead(fileLog, dataFileName);
298 if (filesRead!= null){
299 filesRead = filesRead.replaceAll(lineFormat, br);
300 lastNumberRead= Integer.parseInt(filesRead.trim());
304 startFileSize = countLines(logFile);
305 if (startFileSize < lastNumberRead ){
306 logger.error("Filed Rolled: set Last number read to 0");
310 isMissingLogFile = false;
311 Path filePath = Paths.get(logFile);
312 File file = new File(logFile);
313 startFileSize = file.length();
314 // start process pap/pdp-rest.log file
315 processAPIRestLog(file, filePath, dataFileName);
319 public static int countLines(String filename){
322 FileReader freader = new FileReader(filename);
323 LineNumberReader reader = new LineNumberReader(freader);
326 while ((line = reader.readLine()) != null) {
327 logger.debug("Reading the Logs"+line);
329 cnt = reader.getLineNumber();
330 logger.info("Line number:"+cnt);
341 public static String pullLastLineRead(File file, String dataFileName) throws IOException {
343 file.createNewFile();
346 try(RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");){
347 StringBuilder builder = new StringBuilder();
348 long length = file.length();
349 logger.debug("dataFileName: " +dataFileName);
352 randomAccessFile.seek(length);
353 for(long seek = length; seek >= 0; --seek){
354 randomAccessFile.seek(seek);
355 char c = (char)randomAccessFile.read();
358 builder = builder.reverse();
359 logger.debug("builder.toString(): " +builder.toString());
360 if (builder.toString().contains(last+dataFileName+lineRead)){
361 String[] parseString = builder.toString().split(last+dataFileName+lineRead);
362 String returnValue = parseString[1].replace("\r", "");
363 randomAccessFile.close();
364 return returnValue.trim();
366 builder = new StringBuilder();
370 randomAccessFile.close();
375 private static LogEntryObject getDebugOutLogValue (String line, String type){
378 LogEntryObject logEntry = new LogEntryObject();
379 logEntry.setSystemType(type);
380 logEntry.setSystem(system);
381 String info1 = "||INFO||";
382 String info2 = "INFO:";
383 String error1 = "||ERROR||";
384 String error2 = "ERROR:";
386 if(line.contains(info1) || line.contains(error1) || line.contains(info2) || line.contains(error2)){
387 String[] splitString = null;
388 if(line.contains(info1) || line.contains(error1)){
389 splitString = line.split("[||]");
390 }else if(line.contains(info2)){
391 splitString = line.split(info2);
393 splitString = line.split(error2);
395 String dateString = splitString[0].substring(0, 19);
396 logEntry.setDescription(splitString[splitString.length-1]);
399 date = parseDate(dateString.replace("T", " "), dateFormat, false);
400 logEntry.setDate(date);
402 logEntry.setRemote(parseRemoteSystem(line));
403 if (line.contains(info2) || line.contains(info1)){
404 logEntry.setLogType(LOGTYPE.INFO);
406 logEntry.setLogType(LOGTYPE.ERROR);
415 private static LogEntryObject getRestAPIOutLogValue (String line, String type){
417 LogEntryObject logEntry = new LogEntryObject();
418 logEntry.setSystemType(type);
419 logEntry.setSystem(system);
420 String info3 = "INFO";
422 // from PDP/PAP rest log file below
423 if (line.contains(info3) && line.contains(")-")){
424 //parse out description
425 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
427 date = parseDate(line, dateFormat, true);
428 logEntry.setDate(date);
430 logEntry.setRemote(parseRemoteSystem(line));
431 logEntry.setLogType(LOGTYPE.INFO);
439 private static LogEntryObject getInfoOutLogValue (String line, String type){
441 LogEntryObject logEntry = new LogEntryObject();
442 logEntry.setSystemType(type);
443 logEntry.setSystem(system);
444 String info3 = "INFO";
446 if (line.contains(info3) && line.contains("--- [")){
447 //parse out description
448 String temp = line.substring(line.indexOf("---")+1);
449 String[] split = temp.split(":");
451 logEntry.setDescription(split[1]);
454 date = parseDate(line, dateFormat, false);
455 logEntry.setDate(date);
458 logEntry.setRemote(parseRemoteSystem(line));
459 logEntry.setLogType(LOGTYPE.INFO);
467 private static LogEntryObject getSevereOutLogValue (String line, String type){
469 LogEntryObject logEntry = new LogEntryObject();
470 logEntry.setSystemType(type);
471 logEntry.setSystem(system);
472 if (line.contains("SEVERE") && line.contains("[main]")){
473 String[] splitString = line.split(" ");
474 StringBuilder description = new StringBuilder();
475 for (int i = 5; i < splitString.length; i++){
476 description.append(" " + splitString[i]);
479 logEntry.setDescription(description.toString());
481 date = parseDate(line, dateFormat, false);
482 logEntry.setDate(date);
483 logEntry.setLogType(LOGTYPE.SEVERE);
488 if (line.contains("ERROR") && line.contains(")-")){
489 //parse out description
490 StringBuilder description = new StringBuilder();
491 description.append(line.substring(line.indexOf(")-")+3));
493 date = parseDate(line, dateFormat, true);
494 logEntry.setDate(date);
495 logEntry.setDescription(description.toString());
497 logEntry.setRemote(parseRemoteSystem(line));
498 logEntry.setLogType(LOGTYPE.ERROR);
506 private static LogEntryObject getWarnOutLogValue (String line, String type){
508 LogEntryObject logEntry = new LogEntryObject();
509 logEntry.setSystemType(type);
510 logEntry.setSystem(system);
511 if (line.contains("WARN") && line.contains(")-")){
512 //parse out description
514 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
517 date = parseDate(line, dateFormat, true);
518 logEntry.setDate(date);
521 logEntry.setRemote(parseRemoteSystem(line));
522 logEntry.setLogType(LOGTYPE.WARN);
527 if (line.contains("WARNING") && type =="PyPDP"){
528 String[] splitString = line.split(" ");
529 StringBuilder description = new StringBuilder();
531 for (int i = 5; i < splitString.length; i++){
532 description.append(" " + splitString[i]);
536 date = parseDate(line, dateFormat, false);
537 logEntry.setDate(date);
538 logEntry.setLogType(LOGTYPE.WARN);
539 logEntry.setDescription(description.toString());
546 public static LogEntryObject pullOutLogValues(String line, String type){
548 LogEntryObject logEntry = getDebugOutLogValue(line, type);
550 if(logEntry == null){
551 logEntry = getRestAPIOutLogValue(line, type);
553 if(logEntry == null){
554 logEntry = getInfoOutLogValue(line, type);
556 if(logEntry == null){
557 logEntry = getSevereOutLogValue(line, type);
559 if(logEntry == null){
560 logEntry = getWarnOutLogValue(line, type);
566 private static void dbClose(Connection conn) {
569 } catch (SQLException e) {
570 logger.error("Error closing DB Connection: " + e);
575 public static void process(String line, String type, LOGTYPE logFile) {
577 LogEntryObject returnLogValue = null;
580 im.startTransaction();
581 } catch (IntegrityMonitorException e) {
582 logger.error("Error received" + e);
585 returnLogValue = pullOutLogValues(line, type);
587 if(logFile.equals(LOGTYPE.DEBUG)){
588 debuglastNumberRead++;
589 }else if(logFile.equals(LOGTYPE.ERROR)){
590 errorlastNumberRead++;
591 }else if(logFile.equals(LOGTYPE.INFO)){
594 if (returnLogValue!=null){
595 writeDB(returnLogValue);
602 private static void writeDB(LogEntryObject returnLogValue) {
604 Connection conn = dbConnection(jdbcDriver, jdbcUrl, jdbcUser,jdbcPassword);
605 dbAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
606 returnLogValue.getDate(), returnLogValue.getRemote(),
607 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
611 private static Connection dbConnection(String driver, String jdbc, String user, String pass){
614 Class.forName(driver);
615 return DriverManager.getConnection(jdbc, user, pass);
616 } catch ( Exception e) {
617 logger.error("Error connecting to DB: " + e);
621 private static void dbAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
625 Format formatter = new SimpleDateFormat(dateFormat);
626 sdate = formatter.format(date);
627 logger.debug("DBAccesss : sdate : " + sdate);
629 logger.debug("DBAccesss : sdate is null");
632 //ensure the length of description is less than the maximumm db char length
633 if (description.length() > maxLength) {
634 description = description.substring(0, maxLength);
638 PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
641 prep.setString(1, system);
642 prep.setString(2, description);
643 prep.setString(3, remote);
644 prep.setString(4, type);
645 prep.setString(5, sdate);
646 prep.setString(6, logType);
648 prep.executeUpdate();
651 } catch (SQLException e1) {
652 logger.error("Error trying to excute SQL Statment: " + e1);
656 public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
659 String[] splitString = dateline.split(" ");
660 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
663 returnDate = formatter.parse(dateline);
664 } catch (ParseException e) {
665 logger.error("Unable to parse date for line: " + dateline);
669 String tmpString = splitString[0] + " " + splitString[1];
671 returnDate = formatter.parse(tmpString);
672 } catch (ParseException e) {
673 logger.error("Unable to parse date for line: " + dateline);
682 public static String parseRemoteSystem(String line) {
684 if (line.contains("http") && !(line.contains("www.w3.org"))){
686 Pattern pattern = Pattern.compile("://(.+?)/");
687 Matcher remote = pattern.matcher(line);
690 return remote.group(1);
696 public static String[] getPaths(String logPath){
697 String[] oneFile = null;
698 if(logPath != null && !logPath.isEmpty()){
699 if(logPath.contains(";")){
700 return logPath.split(";");
702 oneFile = new String[1];
703 oneFile[0] = logPath;
710 private static void setCleanUpProperties(String cleanupInterval, String timeFrame){
711 if(cleanupInterval != null && !cleanupInterval.isEmpty()){
712 int intCheckInterval = Integer.parseInt(cleanupInterval);
713 if(intCheckInterval > 300000) {//must be longer than 5 minutes
714 checkInterval = intCheckInterval;
717 logger.debug("No value defined for CHECK_INTERVAL in parserlog.properties, so use its default value:" + checkInterval + " milliseconds");
720 if(timeFrame != null && !timeFrame.trim().isEmpty()){
721 int intTimeFrame = defaultTimeFrame;
723 intTimeFrame = Integer.parseInt(timeFrame);
724 }catch(NumberFormatException e){
725 logger.debug("Improper value defined for TIME_FRAME in parserlog.properties, so use its default value:" + defaultTimeFrame + " days");
727 if(intTimeFrame > 0){
728 defaultTimeFrame = intTimeFrame;
731 logger.debug("No value defined for TIME_FRAME in parserlog.properties, so use its default value:" + defaultTimeFrame + " days");
735 private static void setDebuglogFile(String fileName){
736 debuglogFile = fileName;
737 if(debuglogFile != null && !debuglogFile.isEmpty()){
738 debuglogFile = debuglogFile.trim();
744 private static void setErrorlogFile(String fileName){
745 errorlogFile = fileName;
746 if(errorlogFile != null && !errorlogFile.isEmpty()){
747 errorlogFile = errorlogFile.trim();
753 private static void setLogFileProperties(String[] splitString){
754 if(splitString != null){
755 for(int i=0; i < splitString.length; i++){
757 if(splitString[i].contains("debug")){
758 // get path of debug.log file
759 setDebuglogFile(splitString[i]);
760 }else if(splitString[i].contains("error")){
761 // get path of error.log file
762 setErrorlogFile(splitString[i]);
764 // get path of default file
765 logFile = splitString[i];
766 if(logFile != null && !logFile.isEmpty()){
767 logFile = logFile.trim();
776 public static Properties getPropertiesValue(String fileName) {
777 Properties config = new Properties();
778 Path file = Paths.get(fileName);
779 if (file.toFile().exists()) {
781 if (file.toString().endsWith(".properties")) {
784 in = new FileInputStream(file.toFile());
787 resourceName = config.getProperty("RESOURCE_NAME");
788 system = config.getProperty("SERVER");
789 type = config.getProperty("LOGTYPE");
790 systemLogFile = config.getProperty("PARSERLOGPATH");
791 String logFiles = config.getProperty("LOGPATH");
792 String cleanupInterval= config.getProperty("CHECK_INTERVAL");
793 String timeFrame = config.getProperty("TIME_FRAME");
795 setCleanUpProperties(cleanupInterval, timeFrame);
797 if(logFiles == null || logFiles.isEmpty()){
798 isMissingLogFile = true;
802 String[] splitString = getPaths(logFiles);
804 setLogFileProperties(splitString);
806 jdbcUrl = config.getProperty("JDBC_URL").replace("'", "");
807 jdbcUser = config.getProperty("JDBC_USER");
808 jdbcDriver = config.getProperty("JDBC_DRIVER");
809 jdbcPassword = CryptoUtils.decryptTxtNoExStr(config.getProperty("JDBC_PASSWORD", ""));
810 config.setProperty("javax.persistence.jdbc.password", CryptoUtils.decryptTxtNoExStr(config.getProperty("javax.persistence.jdbc.password", "")));
813 } catch (IOException e) {
814 logger.error("Error porcessing Config file will be unable to create Health Check" + e);
816 logger.error("Error getPropertiesValue on TIME_FRAME", e);
817 logger.debug("Error getPropertiesValue on TIME_FRAME, so use its default value:" + defaultTimeFrame + " days");
823 logger.debug("File doesn't exist in the specified Path " + file.toString());
828 public static Connection getDbConnection(){
829 return dbConnection(jdbcDriver, jdbcUrl, jdbcUser,jdbcPassword);
831 private static void startCleanUp(){
832 Connection conn = dbConnection(jdbcDriver, jdbcUrl, jdbcUser,jdbcPassword);
833 CleanUpSystemLogDB cleanUp = new CleanUpSystemLogDB(conn, defaultTimeFrame);
834 Timer timer = new Timer(true);
835 timer.scheduleAtFixedRate(cleanUp, TIMER_DELAY_TIME, checkInterval);
836 logger.info("startCleanUp begins! : " + new Date());