2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.xacml.parser;
24 import java.io.FileInputStream;
25 import java.io.FileReader;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.LineNumberReader;
29 import java.io.RandomAccessFile;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.Paths;
34 import java.sql.Connection;
35 import java.sql.DriverManager;
36 import java.sql.PreparedStatement;
37 import java.sql.SQLException;
38 import java.text.Format;
39 import java.text.ParseException;
40 import java.text.SimpleDateFormat;
41 import java.util.Date;
42 import java.util.Properties;
43 import java.util.regex.Matcher;
44 import java.util.regex.Pattern;
45 import java.util.stream.Stream;
47 import org.apache.log4j.Logger;
48 import org.openecomp.policy.common.im.AdministrativeStateException;
49 import org.openecomp.policy.common.im.IntegrityMonitor;
50 import org.openecomp.policy.common.im.StandbyStatusException;
51 import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
52 import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE;
55 * Parse log files and store the information in a H2 database.
59 public class ParseLog {
61 // only logging last line of each log file processed to the log4j log file defined by property - PARSERLOGPATH
62 private static final Logger log4jlogger = Logger.getLogger(ParseLog.class.getName());
65 private static org.openecomp.policy.common.logging.flexlogger.Logger logger = FlexLogger.getLogger(ParseLog.class.getName());
67 private static String system;
68 private static int lastNumberRead = 0;
69 private static int debuglastNumberRead = 0;
70 private static int errorlastNumberRead = 0;
71 private static String type;
72 private static long startFileSize;
73 private static long debugStartFileSize;
74 private static long errorStartFileSize;
75 private static String systemLogFile;
76 private static String logFile;
77 private static String debuglogFile;
78 private static String errorlogFile;
79 private static String JDBC_URL;
80 private static String JDBC_USER;
81 private static String JDBC_PASSWORD = "";
82 private static String JDBC_DRIVER;
83 private static int maxLength = 255; //Max length that is allowed in the DB table
84 private static String resourceName;
85 private static long sleepTimer = 50000;
86 static IntegrityMonitor im;
87 private static boolean isMissingLogFile;
89 private static RandomAccessFile randomAccessFile;
91 public static void main(String[] args) throws Exception {
93 Properties logProperties = getPropertiesValue("parserlog.properties");
95 if(logProperties == null || isMissingLogFile){
96 // missing the path of log file in the properties file, so stop the process
97 logger.error("logProperties is null or LOGPATH is missing in parserlog.properties, so stop the process.");
101 File fileLog = new File(systemLogFile);
103 im = IntegrityMonitor.getInstance(resourceName,logProperties );
105 startDebugLogParser(fileLog);
106 startErrorLogParser(fileLog);
107 startAPIRestLogParser(fileLog);
111 private static void startDebugLogParser(File fileLog) throws Exception{
112 if(debuglogFile != null && !debuglogFile.isEmpty()){
114 // pull the last line number
115 String dataFileName = "debug.log";
116 String filesRead = PullLastLineRead(fileLog, dataFileName);
117 if (filesRead!= null){
118 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
119 debuglastNumberRead= Integer.parseInt(filesRead.trim());
121 debuglastNumberRead = 0;
124 debugStartFileSize = countLines(debuglogFile);
125 if (debugStartFileSize < debuglastNumberRead ){
126 logger.error("Filed Rolled: set Last debug number read to 0");
127 debuglastNumberRead = 0;
130 isMissingLogFile = false;
131 Path debugfilePath = Paths.get(debuglogFile);
132 File debugfile = new File(debuglogFile);
133 debugStartFileSize = debugfile.length();
134 // start process debug.log file
136 Runnable runnable = new Runnable (){
137 boolean isStop = false;
142 if (debugfile.isFile()){
143 // log4jlogger must use .info
144 try (Stream<String> lines = Files.lines(debugfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + debuglastNumberRead)).skip(debuglastNumberRead)) {
146 lines.forEachOrdered(line -> process(line, type, LOGTYPE.DEBUG));
148 } catch (IOException e) {
149 logger.error("Error processing line in " + dataFileName + ":" + e);
150 logger.error("break the loop.");
155 Thread.sleep(sleepTimer);
156 debugStartFileSize = countLines(debuglogFile);
157 } catch (InterruptedException | IOException e) {
158 logger.error("Error processing line in " + dataFileName + ":" + e);
159 logger.error("break the loop.");
163 logger.debug("File Line Count of debug.log: " + debugStartFileSize + " value read in: " + debuglastNumberRead);
164 if (debugStartFileSize < debuglastNumberRead ){
165 logger.debug("Failed Rolled: set Last number read to 0");
166 debuglastNumberRead = 0;
172 Thread thread = new Thread(runnable);
178 private static void startErrorLogParser(File fileLog) throws Exception{
180 if(errorlogFile != null && !errorlogFile.isEmpty()){
182 // pull the last line number
183 String dataFileName = "error.log";
184 String filesRead = PullLastLineRead(fileLog, dataFileName);
185 if (filesRead!= null){
186 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
187 errorlastNumberRead= Integer.parseInt(filesRead.trim());
189 errorlastNumberRead = 0;
192 errorStartFileSize = countLines(errorlogFile);
193 if (errorStartFileSize < errorlastNumberRead ){
194 logger.error("Filed Rolled: set Last error number read to 0");
195 errorlastNumberRead = 0;
198 isMissingLogFile = false;
199 Path errorfilePath = Paths.get(errorlogFile);
200 File errorfile = new File(errorlogFile);
201 errorStartFileSize = errorfile.length();
202 // start process error.log file
203 Runnable runnable = new Runnable (){
204 boolean isStop = false;
208 if (errorfile.isFile()){
209 // log4jlogger must use .info
210 try (Stream<String> lines = Files.lines(errorfilePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + errorlastNumberRead)).skip(errorlastNumberRead)) {
212 lines.forEachOrdered(line -> process(line, type, LOGTYPE.ERROR));
214 } catch (IOException e) {
215 logger.error("Error processing line in " + dataFileName + ":" + e);
216 logger.error("break the loop.");
221 Thread.sleep(sleepTimer);
222 errorStartFileSize = countLines(errorlogFile);
223 } catch (InterruptedException | IOException e) {
224 logger.error("Error processing line in " + dataFileName + ":" + e);
225 logger.error("break the loop.");
229 logger.debug("File Line Count of error.log: " + errorStartFileSize + " value read in: " + errorlastNumberRead);
230 if (errorStartFileSize < errorlastNumberRead ){
231 logger.debug("Failed Rolled: set Last error number read to 0");
232 errorlastNumberRead = 0;
238 Thread thread = new Thread(runnable);
243 private static void startAPIRestLogParser(File fileLog) throws Exception{
245 if(logFile != null && !logFile.isEmpty()){
247 // pull the last line number
248 String dataFileName = type.toLowerCase()+"-rest.log";
249 String filesRead = PullLastLineRead(fileLog, dataFileName);
250 if (filesRead!= null){
251 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
252 lastNumberRead= Integer.parseInt(filesRead.trim());
256 startFileSize = countLines(logFile);
257 if (startFileSize < lastNumberRead ){
258 logger.error("Filed Rolled: set Last number read to 0");
262 isMissingLogFile = false;
263 Path filePath = Paths.get(logFile);
264 File file = new File(logFile);
265 startFileSize = file.length();
266 // start process pap/pdp-rest.log file
267 Runnable runnable = new Runnable () {
268 boolean isStop = false;
273 // log4jlogger must use .info
274 try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> log4jlogger.info("Last-"+dataFileName+"-line-Read:" + lastNumberRead)).skip(lastNumberRead)) {
276 lines.forEachOrdered(line -> process(line, type, LOGTYPE.INFO));
278 } catch (IOException e) {
279 logger.error("Error processing line in " + dataFileName + ":" + e);
280 logger.error("break the loop.");
285 Thread.sleep(sleepTimer);
286 startFileSize = countLines(logFile);
287 } catch (InterruptedException | IOException e) {
288 logger.error("Error processing line in " + dataFileName + ":" + e);
289 logger.error("break the loop.");
293 logger.debug("File Line Count of " + dataFileName+": " + startFileSize + " value read in: " + lastNumberRead);
294 if (startFileSize < lastNumberRead ){
295 logger.debug("Failed Rolled: set Last number read to 0");
302 Thread thread = new Thread(runnable);
307 public static int countLines(String filename) throws IOException {
308 LineNumberReader reader = new LineNumberReader(new FileReader(filename));
311 while ((line = reader.readLine()) != null) {
312 logger.info("Reading the Logs"+line);
314 cnt = reader.getLineNumber();
319 public static String PullLastLineRead(File file, String dataFileName) throws IOException {
321 file.createNewFile();
324 randomAccessFile = new RandomAccessFile(file, "r");
325 StringBuilder builder = new StringBuilder();
326 long length = file.length();
327 logger.debug("dataFileName: " +dataFileName);
330 randomAccessFile.seek(length);
331 for(long seek = length; seek >= 0; --seek){
332 randomAccessFile.seek(seek);
333 char c = (char)randomAccessFile.read();
336 builder = builder.reverse();
337 logger.debug("builder.toString(): " +builder.toString());
338 if (builder.toString().contains("Last-"+dataFileName+"-line-Read:")){
339 String[] parseString = builder.toString().split("Last-"+dataFileName+"-line-Read:");
340 String returnValue = parseString[1].replace("\r", "");
341 return returnValue.trim();
344 builder = new StringBuilder();
352 public static LogEntryObject pullOutLogValues(String line, String type){
354 LogEntryObject logEntry = new LogEntryObject();
355 String description = "";
356 logEntry.setSystemType(type);
357 logEntry.setSystem(system);
358 logger.debug("In pullOutLogValues ...");
359 //Values for PDP/PAP debug.log file contains "INFO:", error.log file contains ""ERROR:", others are in PDP/PAP rest log file
360 if(line.contains("||INFO||") || line.contains("||ERROR||") || line.contains("INFO:") || line.contains("ERROR:")){
361 String[] splitString = null;
362 if(line.contains("||INFO||") || line.contains("||ERROR||")){
363 splitString = line.split("[||]");
364 }else if(line.contains("INFO:")){
365 splitString = line.split("INFO:");
367 splitString = line.split("ERROR:");
369 String dateString = splitString[0].substring(0, 19);
370 logEntry.setDescription(splitString[splitString.length-1]);
373 date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
374 logEntry.setDate(date);
376 logEntry.setRemote(parseRemoteSystem(line));
377 if (line.contains("INFO:") || line.contains("||INFO||")){
378 logEntry.setLogType(LOGTYPE.INFO);
380 logEntry.setLogType(LOGTYPE.ERROR);
382 // from PDP/PAP rest log file below
383 }else if (line.contains("INFO") && line.contains(")-")){
384 //parse out description
385 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
387 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
388 logEntry.setDate(date);
390 logEntry.setRemote(parseRemoteSystem(line));
391 logEntry.setLogType(LOGTYPE.INFO);
392 } else if (line.contains("INFO") && line.contains("--- [")){
393 //parse out description
394 String temp = line.substring(line.indexOf("---")+1);
395 String[] split = temp.split(":");
397 logEntry.setDescription(split[1]);
400 date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
401 logEntry.setDate(date);
404 logEntry.setRemote(parseRemoteSystem(line));
405 logEntry.setLogType(LOGTYPE.INFO);
406 }else if (line.contains("SEVERE") && line.contains("[main]")){
407 String[] splitString = line.split(" ");
409 for (int i = 5; i < splitString.length; i++){
410 description = description + " " + splitString[i];
413 logEntry.setDescription(description);
415 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
416 logEntry.setDate(date);
417 logEntry.setLogType(LOGTYPE.SEVERE);
418 } else if (line.contains("WARN") && line.contains(")-")){
419 //parse out description
421 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
424 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
425 logEntry.setDate(date);
428 logEntry.setRemote(parseRemoteSystem(line));
429 logEntry.setLogType(LOGTYPE.WARN);
430 }else if (line.contains("WARNING") && type =="PyPDP"){
431 String[] splitString = line.split(" ");
432 for (int i = 5; i < splitString.length; i++){
433 description = description + " " + splitString[i];
437 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
438 logEntry.setDate(date);
439 logEntry.setLogType(LOGTYPE.WARN);
440 }else if (line.contains("ERROR") && line.contains(")-")){
441 //parse out description
442 description = line.substring(line.indexOf(")-")+3);
445 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
446 logEntry.setDate(date);
448 logEntry.setRemote(parseRemoteSystem(line));
449 logEntry.setLogType(LOGTYPE.ERROR);
457 private static void DBClose(Connection conn) {
460 } catch (SQLException e) {
461 logger.error("Error closing DB Connection: " + e);
466 public static void process(String line, String type, LOGTYPE logFile) {
468 logger.debug("In process: processing line : " + line);
469 LogEntryObject returnLogValue = null;
472 im.startTransaction();
473 } catch (AdministrativeStateException e) {
474 logger.error("Error received" + e);
475 } catch (StandbyStatusException e) {
476 logger.error("Error received" + e);
479 returnLogValue = pullOutLogValues(line, type);
481 if(logFile.equals(LOGTYPE.DEBUG)){
482 debuglastNumberRead++;
483 }else if(logFile.equals(LOGTYPE.ERROR)){
484 errorlastNumberRead++;
485 }else if(logFile.equals(LOGTYPE.INFO)){
488 if (returnLogValue!=null){
489 writeDB(returnLogValue);
496 private static void writeDB(LogEntryObject returnLogValue) {
498 Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
499 DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
500 returnLogValue.getDate(), returnLogValue.getRemote(),
501 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
505 private static Connection DBConnection(String driver, String jdbc, String user, String pass){
508 Class.forName(driver);
509 Connection conn = DriverManager.getConnection(jdbc, user, pass);
511 } catch ( Exception e) {
512 logger.error("Error connecting to DB: " + e);
516 private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
521 Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
522 sdate = formatter.format(date);
523 logger.debug("DBAccesss : sdate : " + sdate);
525 logger.debug("DBAccesss : sdate is null");
528 //ensure the length of description is less than the maximumm db char length
529 if (description.length() > maxLength) {
530 description = description.substring(0, maxLength);
534 PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
535 prep.setString(1, system);
536 prep.setString(2, description);
537 prep.setString(3, remote);
538 prep.setString(4, type);
539 prep.setString(5, sdate);
540 prep.setString(6, logType);
542 prep.executeUpdate();
545 } catch (SQLException e1) {
546 logger.error("Error trying to excute SQL Statment: " + e1);
550 public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
553 String[] splitString = dateline.split(" ");
554 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
557 returnDate = formatter.parse(splitString[0]);
558 } catch (ParseException e) {
559 logger.error("Unable to parse date for line: " + dateline);
563 String tmpString = splitString[0] + " " + splitString[1];
565 returnDate = formatter.parse(tmpString);
566 } catch (ParseException e) {
567 logger.error("Unable to parse date for line: " + dateline);
576 public static String parseRemoteSystem(String line) {
578 if (line.contains("http") && !(line.contains("www.w3.org"))){
580 Pattern pattern = Pattern.compile("://(.+?)/");
581 Matcher remote = pattern.matcher(line);
584 return remote.group(1);
590 public static String[] getPaths(String logPath){
592 if(logPath != null && !logPath.isEmpty()){
593 if(logPath.contains(";")){
594 return logPath.split(";");
596 String[] oneFile = new String[1];
597 oneFile[0] = logPath;
605 public static Properties getPropertiesValue(String fileName) {
606 Properties config = new Properties();
607 Path file = Paths.get(fileName);
608 if (Files.notExists(file)) {
609 logger.debug("File doesn't exist in the specified Path " + file.toString());
611 if (file.toString().endsWith(".properties")) {
614 in = new FileInputStream(file.toFile());
617 resourceName = config.getProperty("RESOURCE_NAME");
618 system = config.getProperty("SERVER");
619 type = config.getProperty("LOGTYPE");
620 systemLogFile = config.getProperty("PARSERLOGPATH");
621 String logFiles = config.getProperty("LOGPATH");
622 if(logFiles == null || logFiles.isEmpty()){
623 isMissingLogFile = true;
627 String[] splitString = getPaths(logFiles);
629 if(splitString != null){
630 for(int i=0; i < splitString.length; i++){
632 if(splitString[i].contains("debug")){
633 // get path of debug.log file
634 debuglogFile = splitString[i];
635 if(debuglogFile != null && !debuglogFile.isEmpty()){
636 debuglogFile = debuglogFile.trim();
638 }else if(splitString[i].contains("error")){
639 // get path of error.log file
640 errorlogFile = splitString[i];
641 if(errorlogFile != null && !errorlogFile.isEmpty()){
642 errorlogFile = errorlogFile.trim();
645 // get path of default file
646 logFile = splitString[i];
647 if(logFile != null && !logFile.isEmpty()){
648 logFile = logFile.trim();
659 JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
660 JDBC_USER = config.getProperty("JDBC_USER");
661 JDBC_DRIVER = config.getProperty("JDBC_DRIVER");
662 JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
665 } catch (IOException e) {
666 logger.debug("Error porcessing Config file will be unable to create Health Check" + e);