2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.xacml.parser;
24 import java.io.FileInputStream;
25 import java.io.FileReader;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.LineNumberReader;
29 import java.io.RandomAccessFile;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.Paths;
34 import java.sql.Connection;
35 import java.sql.DriverManager;
36 import java.sql.PreparedStatement;
37 import java.sql.SQLException;
38 import java.text.Format;
39 import java.text.ParseException;
40 import java.text.SimpleDateFormat;
41 import java.util.Date;
42 import java.util.Properties;
43 import java.util.regex.Matcher;
44 import java.util.regex.Pattern;
45 import java.util.stream.Stream;
47 import org.apache.log4j.Logger;
48 import org.openecomp.policy.common.im.AdministrativeStateException;
49 import org.openecomp.policy.common.im.IntegrityMonitor;
50 import org.openecomp.policy.common.im.StandbyStatusException;
51 import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
52 import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE;
55 * Parse log files and store the information in a H2 database.
59 public class ParseLog {
61 // only logging last line of each log file processed to the log4j log file defined by property - PARSERLOGPATH
62 private static final Logger log4jlogger = Logger.getLogger(ParseLog.class.getName());
65 private static org.openecomp.policy.common.logging.flexlogger.Logger logger = FlexLogger.getLogger(ParseLog.class.getName());
67 private static String system;
68 private static int lastNumberRead = 0;
69 private static int debuglastNumberRead = 0;
70 private static int errorlastNumberRead = 0;
71 private static String type;
72 private static long startFileSize;
73 private static long debugStartFileSize;
74 private static long errorStartFileSize;
75 private static String systemLogFile;
76 private static String logFile;
77 private static String debuglogFile;
78 private static String errorlogFile;
79 private static String JDBC_URL;
80 private static String JDBC_USER;
81 private static String JDBC_PASSWORD = "";
82 private static String JDBC_DRIVER;
83 private static int maxLength = 255; //Max length that is allowed in the DB table
84 private static String resourceName;
85 private static long sleepTimer = 50000;
86 static IntegrityMonitor im;
87 private static boolean isMissingLogFile;
89 private static RandomAccessFile randomAccessFile;
91 public static void main(String[] args) throws Exception {
93 Properties logProperties = getPropertiesValue("parserlog.properties");
95 if(logProperties == null || isMissingLogFile){
96 // missing the path of log file in the properties file, so stop the process
97 logger.error("logProperties is null or LOGPATH is missing in parserlog.properties, so stop the process.");
101 File fileLog = new File(systemLogFile);
103 im = IntegrityMonitor.getInstance(resourceName,logProperties );
105 startDebugLogParser(fileLog);
106 startErrorLogParser(fileLog);
107 startAPIRestLogParser(fileLog);
111 private static void startDebugLogParser(File fileLog) throws Exception{
112 if(debuglogFile != null && !debuglogFile.isEmpty()){
114 // pull the last line number
115 String dataFileName = "debug.log";
116 String filesRead = PullLastLineRead(fileLog, dataFileName);
117 if (filesRead!= null){
118 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
119 debuglastNumberRead= Integer.parseInt(filesRead.trim());
121 debuglastNumberRead = 0;
124 debugStartFileSize = countLines(debuglogFile);
125 if (debugStartFileSize < debuglastNumberRead ){
126 logger.error("Filed Rolled: set Last debug number read to 0");
127 debuglastNumberRead = 0;
130 isMissingLogFile = false;
131 Path debugfilePath = Paths.get(debuglogFile);
132 File debugfile = new File(debuglogFile);
133 debugStartFileSize = debugfile.length();
134 // start process debug.log file
136 Runnable runnable = new Runnable (){
137 boolean isStop = false;
142 if (debugfile.isFile()){
143 // log4jlogger must use .info
144 Stream<String> lines = null;
146 lines = Files.lines(debugfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + debuglastNumberRead)).skip(debuglastNumberRead);
147 lines.forEachOrdered(line -> process(line, type, LOGTYPE.DEBUG));
148 } catch (IOException e) {
149 logger.error("Error processing line in " + dataFileName + ":" + e);
150 logger.error("break the loop.");
159 Thread.sleep(sleepTimer);
160 debugStartFileSize = countLines(debuglogFile);
161 } catch (InterruptedException | IOException e) {
162 logger.error("Error processing line in " + dataFileName + ":" + e);
163 logger.error("break the loop.");
167 logger.debug("File Line Count of debug.log: " + debugStartFileSize + " value read in: " + debuglastNumberRead);
168 if (debugStartFileSize < debuglastNumberRead ){
169 logger.debug("Failed Rolled: set Last number read to 0");
170 debuglastNumberRead = 0;
176 Thread thread = new Thread(runnable);
182 private static void startErrorLogParser(File fileLog) throws Exception{
184 if(errorlogFile != null && !errorlogFile.isEmpty()){
186 // pull the last line number
187 String dataFileName = "error.log";
188 String filesRead = PullLastLineRead(fileLog, dataFileName);
189 if (filesRead!= null){
190 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
191 errorlastNumberRead= Integer.parseInt(filesRead.trim());
193 errorlastNumberRead = 0;
196 errorStartFileSize = countLines(errorlogFile);
197 if (errorStartFileSize < errorlastNumberRead ){
198 logger.error("Filed Rolled: set Last error number read to 0");
199 errorlastNumberRead = 0;
202 isMissingLogFile = false;
203 Path errorfilePath = Paths.get(errorlogFile);
204 File errorfile = new File(errorlogFile);
205 errorStartFileSize = errorfile.length();
206 // start process error.log file
207 Runnable runnable = new Runnable (){
208 boolean isStop = false;
212 if (errorfile.isFile()){
213 // log4jlogger must use .info
214 Stream<String> lines = null;
216 lines = Files.lines(errorfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + errorlastNumberRead)).skip(errorlastNumberRead);
217 lines.forEachOrdered(line -> process(line, type, LOGTYPE.ERROR));
218 } catch (IOException e) {
219 logger.error("Error processing line in " + dataFileName + ":" + e);
220 logger.error("break the loop.");
229 Thread.sleep(sleepTimer);
230 errorStartFileSize = countLines(errorlogFile);
231 } catch (InterruptedException | IOException e) {
232 logger.error("Error processing line in " + dataFileName + ":" + e);
233 logger.error("break the loop.");
237 logger.debug("File Line Count of error.log: " + errorStartFileSize + " value read in: " + errorlastNumberRead);
238 if (errorStartFileSize < errorlastNumberRead ){
239 logger.debug("Failed Rolled: set Last error number read to 0");
240 errorlastNumberRead = 0;
246 Thread thread = new Thread(runnable);
251 private static void startAPIRestLogParser(File fileLog) throws Exception{
253 if(logFile != null && !logFile.isEmpty()){
255 // pull the last line number
256 String dataFileName = type.toLowerCase()+"-rest.log";
257 String filesRead = PullLastLineRead(fileLog, dataFileName);
258 if (filesRead!= null){
259 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
260 lastNumberRead= Integer.parseInt(filesRead.trim());
264 startFileSize = countLines(logFile);
265 if (startFileSize < lastNumberRead ){
266 logger.error("Filed Rolled: set Last number read to 0");
270 isMissingLogFile = false;
271 Path filePath = Paths.get(logFile);
272 File file = new File(logFile);
273 startFileSize = file.length();
274 // start process pap/pdp-rest.log file
275 Runnable runnable = new Runnable () {
276 boolean isStop = false;
281 // log4jlogger must use .info
282 Stream<String> lines = null;;
284 lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + lastNumberRead)).skip(lastNumberRead);
285 lines.forEachOrdered(line -> process(line, type, LOGTYPE.INFO));
286 } catch (IOException e) {
287 logger.error("Error processing line in " + dataFileName + ":" + e);
288 logger.error("break the loop.");
297 Thread.sleep(sleepTimer);
298 startFileSize = countLines(logFile);
299 } catch (InterruptedException | IOException e) {
300 logger.error("Error processing line in " + dataFileName + ":" + e);
301 logger.error("break the loop.");
305 logger.debug("File Line Count of " + dataFileName+": " + startFileSize + " value read in: " + lastNumberRead);
306 if (startFileSize < lastNumberRead ){
307 logger.debug("Failed Rolled: set Last number read to 0");
314 Thread thread = new Thread(runnable);
319 public static int countLines(String filename) throws IOException {
320 LineNumberReader reader = new LineNumberReader(new FileReader(filename));
323 while ((line = reader.readLine()) != null) {
324 logger.info("Reading the Logs"+line);
326 cnt = reader.getLineNumber();
331 public static String PullLastLineRead(File file, String dataFileName) throws IOException {
333 file.createNewFile();
336 randomAccessFile = new RandomAccessFile(file, "r");
337 StringBuilder builder = new StringBuilder();
338 long length = file.length();
339 logger.debug("dataFileName: " +dataFileName);
342 randomAccessFile.seek(length);
343 for(long seek = length; seek >= 0; --seek){
344 randomAccessFile.seek(seek);
345 char c = (char)randomAccessFile.read();
348 builder = builder.reverse();
349 logger.debug("builder.toString(): " +builder.toString());
350 if (builder.toString().contains("Last-"+dataFileName+"-line-Read:")){
351 String[] parseString = builder.toString().split("Last-"+dataFileName+"-line-Read:");
352 String returnValue = parseString[1].replace("\r", "");
353 return returnValue.trim();
356 builder = new StringBuilder();
364 public static LogEntryObject pullOutLogValues(String line, String type){
366 LogEntryObject logEntry = new LogEntryObject();
367 String description = "";
368 logEntry.setSystemType(type);
369 logEntry.setSystem(system);
370 logger.debug("In pullOutLogValues ...");
371 //Values for PDP/PAP debug.log file contains "INFO:", error.log file contains ""ERROR:", others are in PDP/PAP rest log file
372 if(line.contains("||INFO||") || line.contains("||ERROR||") || line.contains("INFO:") || line.contains("ERROR:")){
373 String[] splitString = null;
374 if(line.contains("||INFO||") || line.contains("||ERROR||")){
375 splitString = line.split("[||]");
376 }else if(line.contains("INFO:")){
377 splitString = line.split("INFO:");
379 splitString = line.split("ERROR:");
381 String dateString = splitString[0].substring(0, 19);
382 logEntry.setDescription(splitString[splitString.length-1]);
385 date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
386 logEntry.setDate(date);
388 logEntry.setRemote(parseRemoteSystem(line));
389 if (line.contains("INFO:") || line.contains("||INFO||")){
390 logEntry.setLogType(LOGTYPE.INFO);
392 logEntry.setLogType(LOGTYPE.ERROR);
394 // from PDP/PAP rest log file below
395 }else if (line.contains("INFO") && line.contains(")-")){
396 //parse out description
397 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
399 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
400 logEntry.setDate(date);
402 logEntry.setRemote(parseRemoteSystem(line));
403 logEntry.setLogType(LOGTYPE.INFO);
404 } else if (line.contains("INFO") && line.contains("--- [")){
405 //parse out description
406 String temp = line.substring(line.indexOf("---")+1);
407 String[] split = temp.split(":");
409 logEntry.setDescription(split[1]);
412 date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
413 logEntry.setDate(date);
416 logEntry.setRemote(parseRemoteSystem(line));
417 logEntry.setLogType(LOGTYPE.INFO);
418 }else if (line.contains("SEVERE") && line.contains("[main]")){
419 String[] splitString = line.split(" ");
421 for (int i = 5; i < splitString.length; i++){
422 description = description + " " + splitString[i];
425 logEntry.setDescription(description);
427 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
428 logEntry.setDate(date);
429 logEntry.setLogType(LOGTYPE.SEVERE);
430 } else if (line.contains("WARN") && line.contains(")-")){
431 //parse out description
433 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
436 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
437 logEntry.setDate(date);
440 logEntry.setRemote(parseRemoteSystem(line));
441 logEntry.setLogType(LOGTYPE.WARN);
442 }else if (line.contains("WARNING") && type =="PyPDP"){
443 String[] splitString = line.split(" ");
444 for (int i = 5; i < splitString.length; i++){
445 description = description + " " + splitString[i];
449 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
450 logEntry.setDate(date);
451 logEntry.setLogType(LOGTYPE.WARN);
452 }else if (line.contains("ERROR") && line.contains(")-")){
453 //parse out description
454 description = line.substring(line.indexOf(")-")+3);
457 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
458 logEntry.setDate(date);
460 logEntry.setRemote(parseRemoteSystem(line));
461 logEntry.setLogType(LOGTYPE.ERROR);
469 private static void DBClose(Connection conn) {
472 } catch (SQLException e) {
473 logger.error("Error closing DB Connection: " + e);
478 public static void process(String line, String type, LOGTYPE logFile) {
480 logger.debug("In process: processing line : " + line);
481 LogEntryObject returnLogValue = null;
484 im.startTransaction();
485 } catch (AdministrativeStateException e) {
486 logger.error("Error received" + e);
487 } catch (StandbyStatusException e) {
488 logger.error("Error received" + e);
491 returnLogValue = pullOutLogValues(line, type);
493 if(logFile.equals(LOGTYPE.DEBUG)){
494 debuglastNumberRead++;
495 }else if(logFile.equals(LOGTYPE.ERROR)){
496 errorlastNumberRead++;
497 }else if(logFile.equals(LOGTYPE.INFO)){
500 if (returnLogValue!=null){
501 writeDB(returnLogValue);
508 private static void writeDB(LogEntryObject returnLogValue) {
510 Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
511 DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
512 returnLogValue.getDate(), returnLogValue.getRemote(),
513 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
517 private static Connection DBConnection(String driver, String jdbc, String user, String pass){
520 Class.forName(driver);
521 Connection conn = DriverManager.getConnection(jdbc, user, pass);
523 } catch ( Exception e) {
524 logger.error("Error connecting to DB: " + e);
528 private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
533 Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
534 sdate = formatter.format(date);
535 logger.debug("DBAccesss : sdate : " + sdate);
537 logger.debug("DBAccesss : sdate is null");
540 //ensure the length of description is less than the maximumm db char length
541 if (description.length() > maxLength) {
542 description = description.substring(0, maxLength);
546 PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
547 prep.setString(1, system);
548 prep.setString(2, description);
549 prep.setString(3, remote);
550 prep.setString(4, type);
551 prep.setString(5, sdate);
552 prep.setString(6, logType);
554 prep.executeUpdate();
557 } catch (SQLException e1) {
558 logger.error("Error trying to excute SQL Statment: " + e1);
562 public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
565 String[] splitString = dateline.split(" ");
566 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
569 returnDate = formatter.parse(splitString[0]);
570 } catch (ParseException e) {
571 logger.error("Unable to parse date for line: " + dateline);
575 String tmpString = splitString[0] + " " + splitString[1];
577 returnDate = formatter.parse(tmpString);
578 } catch (ParseException e) {
579 logger.error("Unable to parse date for line: " + dateline);
588 public static String parseRemoteSystem(String line) {
590 if (line.contains("http") && !(line.contains("www.w3.org"))){
592 Pattern pattern = Pattern.compile("://(.+?)/");
593 Matcher remote = pattern.matcher(line);
596 return remote.group(1);
602 public static String[] getPaths(String logPath){
604 if(logPath != null && !logPath.isEmpty()){
605 if(logPath.contains(";")){
606 return logPath.split(";");
608 String[] oneFile = new String[1];
609 oneFile[0] = logPath;
617 public static Properties getPropertiesValue(String fileName) {
618 Properties config = new Properties();
619 Path file = Paths.get(fileName);
620 if (Files.notExists(file)) {
621 logger.debug("File doesn't exist in the specified Path " + file.toString());
623 if (file.toString().endsWith(".properties")) {
626 in = new FileInputStream(file.toFile());
629 resourceName = config.getProperty("RESOURCE_NAME");
630 system = config.getProperty("SERVER");
631 type = config.getProperty("LOGTYPE");
632 systemLogFile = config.getProperty("PARSERLOGPATH");
633 String logFiles = config.getProperty("LOGPATH");
634 if(logFiles == null || logFiles.isEmpty()){
635 isMissingLogFile = true;
639 String[] splitString = getPaths(logFiles);
641 if(splitString != null){
642 for(int i=0; i < splitString.length; i++){
644 if(splitString[i].contains("debug")){
645 // get path of debug.log file
646 debuglogFile = splitString[i];
647 if(debuglogFile != null && !debuglogFile.isEmpty()){
648 debuglogFile = debuglogFile.trim();
650 }else if(splitString[i].contains("error")){
651 // get path of error.log file
652 errorlogFile = splitString[i];
653 if(errorlogFile != null && !errorlogFile.isEmpty()){
654 errorlogFile = errorlogFile.trim();
657 // get path of default file
658 logFile = splitString[i];
659 if(logFile != null && !logFile.isEmpty()){
660 logFile = logFile.trim();
671 JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
672 JDBC_USER = config.getProperty("JDBC_USER");
673 JDBC_DRIVER = config.getProperty("JDBC_DRIVER");
674 JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
677 } catch (IOException e) {
678 logger.debug("Error porcessing Config file will be unable to create Health Check" + e);