2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.onap.xacml.parser;
24 import java.io.FileInputStream;
25 import java.io.FileReader;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.LineNumberReader;
29 import java.io.RandomAccessFile;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.Paths;
34 import java.sql.Connection;
35 import java.sql.DriverManager;
36 import java.sql.PreparedStatement;
37 import java.sql.SQLException;
38 import java.text.Format;
39 import java.text.ParseException;
40 import java.text.SimpleDateFormat;
41 import java.util.Date;
42 import java.util.Properties;
43 import java.util.regex.Matcher;
44 import java.util.regex.Pattern;
45 import java.util.stream.Stream;
47 import org.apache.log4j.Logger;
48 import org.onap.policy.common.im.AdministrativeStateException;
49 import org.onap.policy.common.im.IntegrityMonitor;
50 import org.onap.policy.common.im.StandbyStatusException;
51 import org.onap.policy.common.logging.flexlogger.FlexLogger;
52 import org.onap.xacml.parser.LogEntryObject.LOGTYPE;
55 * Parse log files and store the information in a H2 database.
59 public class ParseLog {
61 // only logging last line of each log file processed to the log4j log file defined by property - PARSERLOGPATH
62 private static final Logger log4jlogger = Logger.getLogger(ParseLog.class.getName());
65 private static org.onap.policy.common.logging.flexlogger.Logger logger = FlexLogger.getLogger(ParseLog.class.getName());
67 private static String system;
68 private static int lastNumberRead = 0;
69 private static int debuglastNumberRead = 0;
70 private static int errorlastNumberRead = 0;
71 private static String type;
72 private static long startFileSize;
73 private static long debugStartFileSize;
74 private static long errorStartFileSize;
75 private static String systemLogFile;
76 private static String logFile;
77 private static String debuglogFile;
78 private static String errorlogFile;
79 private static String JDBC_URL;
80 private static String JDBC_USER;
81 private static String JDBC_PASSWORD;
82 private static String JDBC_DRIVER;
83 private static int maxLength = 255; //Max length that is allowed in the DB table
84 private static String resourceName;
85 private static long sleepTimer = 50000;
86 static IntegrityMonitor im;
87 private static boolean isMissingLogFile;
89 private static RandomAccessFile randomAccessFile;
91 public static void main(String[] args) throws Exception {
93 Properties logProperties = getPropertiesValue("parserlog.properties");
95 if(logProperties == null || isMissingLogFile){
96 // missing the path of log file in the properties file, so stop the process
97 logger.error("logProperties is null or LOGPATH is missing in parserlog.properties, so stop the process.");
101 File fileLog = new File(systemLogFile);
103 im = IntegrityMonitor.getInstance(resourceName,logProperties );
105 startDebugLogParser(fileLog);
106 startErrorLogParser(fileLog);
107 startAPIRestLogParser(fileLog);
111 private static void startDebugLogParser(File fileLog){
113 if(debuglogFile != null && !debuglogFile.isEmpty()){
115 // pull the last line number
116 String dataFileName = "debug.log";
117 String filesRead = PullLastLineRead(fileLog, dataFileName);
118 if (filesRead!= null){
119 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
120 debuglastNumberRead= Integer.parseInt(filesRead.trim());
122 debuglastNumberRead = 0;
125 debugStartFileSize = countLines(debuglogFile);
126 if (debugStartFileSize < debuglastNumberRead ){
127 logger.error("Filed Rolled: set Last debug number read to 0");
128 debuglastNumberRead = 0;
131 isMissingLogFile = false;
132 Path debugfilePath = Paths.get(debuglogFile);
133 File debugfile = new File(debuglogFile);
134 debugStartFileSize = debugfile.length();
135 // start process debug.log file
137 Runnable runnable = new Runnable (){
138 boolean isStop = false;
143 if (debugfile.isFile()){
144 // log4jlogger must use .info
145 Stream<String> lines = null;
147 lines = Files.lines(debugfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + debuglastNumberRead)).skip(debuglastNumberRead);
148 lines.forEachOrdered(line -> process(line, type, LOGTYPE.DEBUG));
149 } catch (IOException e) {
150 logger.error("Error processing line in " + dataFileName + ":" + e);
151 logger.error("break the loop.");
160 Thread.sleep(sleepTimer);
161 debugStartFileSize = countLines(debuglogFile);
162 } catch (InterruptedException | IOException e) {
163 logger.error("Error processing line in " + dataFileName + ":" + e);
164 logger.error("break the loop.");
168 logger.debug("File Line Count of debug.log: " + debugStartFileSize + " value read in: " + debuglastNumberRead);
169 if (debugStartFileSize < debuglastNumberRead ){
170 logger.debug("Failed Rolled: set Last number read to 0");
171 debuglastNumberRead = 0;
177 Thread thread = new Thread(runnable);
182 logger.error("Exception occured in DebugLogParser" +e);
186 private static void startErrorLogParser(File fileLog){
188 if(errorlogFile != null && !errorlogFile.isEmpty()){
190 // pull the last line number
191 String dataFileName = "error.log";
192 String filesRead = PullLastLineRead(fileLog, dataFileName);
193 if (filesRead!= null){
194 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
195 errorlastNumberRead= Integer.parseInt(filesRead.trim());
197 errorlastNumberRead = 0;
200 errorStartFileSize = countLines(errorlogFile);
201 if (errorStartFileSize < errorlastNumberRead ){
202 logger.error("Filed Rolled: set Last error number read to 0");
203 errorlastNumberRead = 0;
206 isMissingLogFile = false;
207 Path errorfilePath = Paths.get(errorlogFile);
208 File errorfile = new File(errorlogFile);
209 errorStartFileSize = errorfile.length();
210 // start process error.log file
211 Runnable runnable = new Runnable (){
212 boolean isStop = false;
216 if (errorfile.isFile()){
217 // log4jlogger must use .info
218 Stream<String> lines = null;
220 lines = Files.lines(errorfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + errorlastNumberRead)).skip(errorlastNumberRead);
221 lines.forEachOrdered(line -> process(line, type, LOGTYPE.ERROR));
222 } catch (IOException e) {
223 logger.error("Error processing line in " + dataFileName + ":" + e);
224 logger.error("break the loop.");
233 Thread.sleep(sleepTimer);
234 errorStartFileSize = countLines(errorlogFile);
235 } catch (InterruptedException | IOException e) {
236 logger.error("Error processing line in " + dataFileName + ":" + e);
237 logger.error("break the loop.");
241 logger.debug("File Line Count of error.log: " + errorStartFileSize + " value read in: " + errorlastNumberRead);
242 if (errorStartFileSize < errorlastNumberRead ){
243 logger.debug("Failed Rolled: set Last error number read to 0");
244 errorlastNumberRead = 0;
250 Thread thread = new Thread(runnable);
254 logger.error("Exception occured in startErrorLogParser" +e);
258 private static void startAPIRestLogParser(File fileLog){
260 if(logFile != null && !logFile.isEmpty()){
262 // pull the last line number
263 String dataFileName = type.toLowerCase()+"-rest.log";
264 String filesRead = PullLastLineRead(fileLog, dataFileName);
265 if (filesRead!= null){
266 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
267 lastNumberRead= Integer.parseInt(filesRead.trim());
271 startFileSize = countLines(logFile);
272 if (startFileSize < lastNumberRead ){
273 logger.error("Filed Rolled: set Last number read to 0");
277 isMissingLogFile = false;
278 Path filePath = Paths.get(logFile);
279 File file = new File(logFile);
280 startFileSize = file.length();
281 // start process pap/pdp-rest.log file
282 Runnable runnable = new Runnable () {
283 boolean isStop = false;
288 // log4jlogger must use .info
289 Stream<String> lines = null;;
291 lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + lastNumberRead)).skip(lastNumberRead);
292 lines.forEachOrdered(line -> process(line, type, LOGTYPE.INFO));
293 } catch (IOException e) {
294 logger.error("Error processing line in " + dataFileName + ":" + e);
295 logger.error("break the loop.");
304 Thread.sleep(sleepTimer);
305 startFileSize = countLines(logFile);
306 } catch (InterruptedException | IOException e) {
307 logger.error("Error processing line in " + dataFileName + ":" + e);
308 logger.error("break the loop.");
312 logger.debug("File Line Count of " + dataFileName+": " + startFileSize + " value read in: " + lastNumberRead);
313 if (startFileSize < lastNumberRead ){
314 logger.debug("Failed Rolled: set Last number read to 0");
321 Thread thread = new Thread(runnable);
325 logger.error("Exception occured in StartAPIRestLogParser" +e);
329 public static int countLines(String filename) throws IOException {
330 LineNumberReader reader = new LineNumberReader(new FileReader(filename));
333 while ((line = reader.readLine()) != null) {
334 logger.info("Reading the Logs"+line);
336 cnt = reader.getLineNumber();
341 public static String PullLastLineRead(File file, String dataFileName) throws IOException {
343 file.createNewFile();
346 randomAccessFile = new RandomAccessFile(file, "r");
347 StringBuilder builder = new StringBuilder();
348 long length = file.length();
349 logger.debug("dataFileName: " +dataFileName);
352 randomAccessFile.seek(length);
353 for(long seek = length; seek >= 0; --seek){
354 randomAccessFile.seek(seek);
355 char c = (char)randomAccessFile.read();
358 builder = builder.reverse();
359 logger.debug("builder.toString(): " +builder.toString());
360 if (builder.toString().contains("Last-"+dataFileName+"-line-Read:")){
361 String[] parseString = builder.toString().split("Last-"+dataFileName+"-line-Read:");
362 String returnValue = parseString[1].replace("\r", "");
363 return returnValue.trim();
366 builder = new StringBuilder();
374 public static LogEntryObject pullOutLogValues(String line, String type){
376 LogEntryObject logEntry = new LogEntryObject();
377 String description = "";
378 logEntry.setSystemType(type);
379 logEntry.setSystem(system);
380 logger.debug("In pullOutLogValues ...");
381 //Values for PDP/PAP debug.log file contains "INFO:", error.log file contains ""ERROR:", others are in PDP/PAP rest log file
382 if(line.contains("||INFO||") || line.contains("||ERROR||") || line.contains("INFO:") || line.contains("ERROR:")){
383 String[] splitString = null;
384 if(line.contains("||INFO||") || line.contains("||ERROR||")){
385 splitString = line.split("[||]");
386 }else if(line.contains("INFO:")){
387 splitString = line.split("INFO:");
389 splitString = line.split("ERROR:");
391 String dateString = splitString[0].substring(0, 19);
392 logEntry.setDescription(splitString[splitString.length-1]);
395 date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
396 logEntry.setDate(date);
398 logEntry.setRemote(parseRemoteSystem(line));
399 if (line.contains("INFO:") || line.contains("||INFO||")){
400 logEntry.setLogType(LOGTYPE.INFO);
402 logEntry.setLogType(LOGTYPE.ERROR);
404 // from PDP/PAP rest log file below
405 }else if (line.contains("INFO") && line.contains(")-")){
406 //parse out description
407 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
409 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
410 logEntry.setDate(date);
412 logEntry.setRemote(parseRemoteSystem(line));
413 logEntry.setLogType(LOGTYPE.INFO);
414 } else if (line.contains("INFO") && line.contains("--- [")){
415 //parse out description
416 String temp = line.substring(line.indexOf("---")+1);
417 String[] split = temp.split(":");
419 logEntry.setDescription(split[1]);
422 date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
423 logEntry.setDate(date);
426 logEntry.setRemote(parseRemoteSystem(line));
427 logEntry.setLogType(LOGTYPE.INFO);
428 }else if (line.contains("SEVERE") && line.contains("[main]")){
429 String[] splitString = line.split(" ");
431 for (int i = 5; i < splitString.length; i++){
432 description = description + " " + splitString[i];
435 logEntry.setDescription(description);
437 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
438 logEntry.setDate(date);
439 logEntry.setLogType(LOGTYPE.SEVERE);
440 } else if (line.contains("WARN") && line.contains(")-")){
441 //parse out description
443 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
446 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
447 logEntry.setDate(date);
450 logEntry.setRemote(parseRemoteSystem(line));
451 logEntry.setLogType(LOGTYPE.WARN);
452 }else if (line.contains("WARNING") && type =="PyPDP"){
453 String[] splitString = line.split(" ");
454 for (int i = 5; i < splitString.length; i++){
455 description = description + " " + splitString[i];
459 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
460 logEntry.setDate(date);
461 logEntry.setLogType(LOGTYPE.WARN);
462 }else if (line.contains("ERROR") && line.contains(")-")){
463 //parse out description
464 description = line.substring(line.indexOf(")-")+3);
467 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
468 logEntry.setDate(date);
470 logEntry.setRemote(parseRemoteSystem(line));
471 logEntry.setLogType(LOGTYPE.ERROR);
479 private static void DBClose(Connection conn) {
482 } catch (SQLException e) {
483 logger.error("Error closing DB Connection: " + e);
488 public static void process(String line, String type, LOGTYPE logFile) {
490 logger.debug("In process: processing line : " + line);
491 LogEntryObject returnLogValue = null;
494 im.startTransaction();
495 } catch (AdministrativeStateException e) {
496 logger.error("Error received" + e);
497 } catch (StandbyStatusException e) {
498 logger.error("Error received" + e);
501 returnLogValue = pullOutLogValues(line, type);
503 if(logFile.equals(LOGTYPE.DEBUG)){
504 debuglastNumberRead++;
505 }else if(logFile.equals(LOGTYPE.ERROR)){
506 errorlastNumberRead++;
507 }else if(logFile.equals(LOGTYPE.INFO)){
510 if (returnLogValue!=null){
511 writeDB(returnLogValue);
518 private static void writeDB(LogEntryObject returnLogValue) {
520 Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
521 DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
522 returnLogValue.getDate(), returnLogValue.getRemote(),
523 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
527 private static Connection DBConnection(String driver, String jdbc, String user, String pass){
530 Class.forName(driver);
531 Connection conn = DriverManager.getConnection(jdbc, user, pass);
533 } catch ( Exception e) {
534 logger.error("Error connecting to DB: " + e);
538 private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
543 Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
544 sdate = formatter.format(date);
545 logger.debug("DBAccesss : sdate : " + sdate);
547 logger.debug("DBAccesss : sdate is null");
550 //ensure the length of description is less than the maximumm db char length
551 if (description.length() > maxLength) {
552 description = description.substring(0, maxLength);
556 PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
557 prep.setString(1, system);
558 prep.setString(2, description);
559 prep.setString(3, remote);
560 prep.setString(4, type);
561 prep.setString(5, sdate);
562 prep.setString(6, logType);
564 prep.executeUpdate();
567 } catch (SQLException e1) {
568 logger.error("Error trying to excute SQL Statment: " + e1);
572 public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
575 String[] splitString = dateline.split(" ");
576 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
579 returnDate = formatter.parse(splitString[0]);
580 } catch (ParseException e) {
581 logger.error("Unable to parse date for line: " + dateline);
585 String tmpString = splitString[0] + " " + splitString[1];
587 returnDate = formatter.parse(tmpString);
588 } catch (ParseException e) {
589 logger.error("Unable to parse date for line: " + dateline);
598 public static String parseRemoteSystem(String line) {
600 if (line.contains("http") && !(line.contains("www.w3.org"))){
602 Pattern pattern = Pattern.compile("://(.+?)/");
603 Matcher remote = pattern.matcher(line);
606 return remote.group(1);
612 public static String[] getPaths(String logPath){
614 if(logPath != null && !logPath.isEmpty()){
615 if(logPath.contains(";")){
616 return logPath.split(";");
618 String[] oneFile = new String[1];
619 oneFile[0] = logPath;
627 public static Properties getPropertiesValue(String fileName) {
628 Properties config = new Properties();
629 Path file = Paths.get(fileName);
630 if (Files.notExists(file)) {
631 logger.debug("File doesn't exist in the specified Path " + file.toString());
633 if (file.toString().endsWith(".properties")) {
636 in = new FileInputStream(file.toFile());
639 resourceName = config.getProperty("RESOURCE_NAME");
640 system = config.getProperty("SERVER");
641 type = config.getProperty("LOGTYPE");
642 systemLogFile = config.getProperty("PARSERLOGPATH");
643 String logFiles = config.getProperty("LOGPATH");
644 if(logFiles == null || logFiles.isEmpty()){
645 isMissingLogFile = true;
649 String[] splitString = getPaths(logFiles);
651 if(splitString != null){
652 for(int i=0; i < splitString.length; i++){
654 if(splitString[i].contains("debug")){
655 // get path of debug.log file
656 debuglogFile = splitString[i];
657 if(debuglogFile != null && !debuglogFile.isEmpty()){
658 debuglogFile = debuglogFile.trim();
660 }else if(splitString[i].contains("error")){
661 // get path of error.log file
662 errorlogFile = splitString[i];
663 if(errorlogFile != null && !errorlogFile.isEmpty()){
664 errorlogFile = errorlogFile.trim();
667 // get path of default file
668 logFile = splitString[i];
669 if(logFile != null && !logFile.isEmpty()){
670 logFile = logFile.trim();
681 JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
682 JDBC_USER = config.getProperty("JDBC_USER");
683 JDBC_DRIVER = config.getProperty("JDBC_DRIVER");
684 JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
687 } catch (IOException e) {
688 logger.debug("Error porcessing Config file will be unable to create Health Check" + e);