2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.xacml.parser;
24 import java.io.FileInputStream;
25 import java.io.FileReader;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.LineNumberReader;
29 import java.io.RandomAccessFile;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.Paths;
34 import java.sql.Connection;
35 import java.sql.DriverManager;
36 import java.sql.PreparedStatement;
37 import java.sql.SQLException;
38 import java.text.Format;
39 import java.text.ParseException;
40 import java.text.SimpleDateFormat;
41 import java.util.Date;
42 import java.util.Properties;
43 import java.util.regex.Matcher;
44 import java.util.regex.Pattern;
45 import java.util.stream.Stream;
47 import org.openecomp.policy.common.im.AdministrativeStateException;
48 import org.openecomp.policy.common.im.IntegrityMonitor;
49 import org.openecomp.policy.common.im.StandbyStatusException;
51 import org.apache.log4j.Logger;
52 import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE;
55 * Parse log files and store the information in a H2 database.
59 public class ParseLog {
61 private static final Logger logger = Logger.getLogger(ParseLog.class.getName());
63 private static String system;
64 private static int lastNumberRead = 0;
65 private static String type;
66 private static long startFileSize;
67 private static String systemLogFile;
68 private static String logFile;
70 private static String JDBC_URL;
71 private static String JDBC_USER;
72 private static String JDBC_PASSWORD = "";
73 private static String JDBC_DRIVER;
74 private static int maxLength = 255; //Max length that is allowed in the DB table
75 private static String resourceName;
76 private static long sleepTimer = 50000;
77 static IntegrityMonitor im;
79 public static void main(String[] args) throws Exception {
81 Properties logProperties = getPropertiesValue("parserlog.properties");
82 Path filePath = Paths.get(logFile);
83 File file = new File(logFile);
84 File fileLog = new File(systemLogFile);
85 startFileSize = file.length();
87 im = IntegrityMonitor.getInstance(resourceName,logProperties );
89 logger.info("System: " + system );
90 logger.info("System type: " + type );
91 logger.info("Logging File: " + systemLogFile );
92 logger.info("log file: " + logFile);
93 logger.info("JDBC_URL: " + JDBC_URL);
94 logger.info("JDBC_DRIVER: " + JDBC_DRIVER);
96 String filesRead = PullLastLineRead(fileLog);
97 if (filesRead!= null){
98 filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
99 lastNumberRead= Integer.parseInt(filesRead.trim());
103 startFileSize = countLines(logFile);
104 logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
105 if (startFileSize < lastNumberRead ){
106 logger.error("Filed Rolled: set Last number read to 0");
109 Runnable runnable = new Runnable (){
114 try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> logger.info("Last line Read: " + lastNumberRead)).skip(lastNumberRead)) {
116 lines.forEachOrdered(line -> process(line, type));
118 } catch (IOException e) {
119 logger.error("Error processing line in log file: " + e);
123 Thread.sleep(sleepTimer);
124 startFileSize = countLines(logFile);
125 } catch (InterruptedException | IOException e) {
126 logger.error("Error: " + e);
129 logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
130 if (startFileSize < lastNumberRead ){
131 logger.info("Failed Rolled: set Last number read to 0");
138 Thread thread = new Thread(runnable);
143 public static int countLines(String filename) throws IOException {
144 LineNumberReader reader = new LineNumberReader(new FileReader(filename));
146 String lineRead = "";
147 while ((lineRead = reader.readLine()) != null) {}
149 cnt = reader.getLineNumber();
154 public static String PullLastLineRead(File file) throws IOException {
156 file.createNewFile();
159 RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
161 StringBuilder builder = new StringBuilder();
162 long length = file.length();
164 randomAccessFile.seek(length);
165 for(long seek = length; seek >= 0; --seek){
166 randomAccessFile.seek(seek);
167 char c = (char)randomAccessFile.read();
170 builder = builder.reverse();
171 if (builder.toString().contains("Last line Read:")){
172 String[] parseString = builder.toString().split("Last line Read:");
173 String returnValue = parseString[1].replace("\r", "");
174 return returnValue.trim();
175 //return parseString[2].replace("\r", "");
179 builder = new StringBuilder();
186 public static LogEntryObject pullOutLogValues(String line, String type){
188 LogEntryObject logEntry = new LogEntryObject();
189 logEntry.setSystemType(type);
190 String description = null;
192 logEntry.setSystem(system);
194 //Values for PDP/PAP log file
195 if(line.contains("||INFO||") || line.contains("||ERROR||")){
196 String[] splitString = line.split("[||]");
197 String dateString = splitString[0].substring(0, 19);
198 logEntry.setDescription(splitString[splitString.length-1]);
201 date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
202 logEntry.setDate(date);
204 logEntry.setRemote(parseRemoteSystem(line));
205 if (line.contains("||INFO||")){
206 logEntry.setLogType(LOGTYPE.INFO);
208 logEntry.setLogType(LOGTYPE.ERROR);
210 }else if (line.contains("INFO") && line.contains(")-")){
211 //parse out description
212 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
214 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
215 logEntry.setDate(date);
217 logEntry.setRemote(parseRemoteSystem(line));
218 logEntry.setLogType(LOGTYPE.INFO);
219 } else if (line.contains("INFO") && line.contains("--- [")){
220 //parse out description
221 String temp = line.substring(line.indexOf("---")+1);
222 String[] split = temp.split(":");
224 logEntry.setDescription(split[1]);
227 date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
228 logEntry.setDate(date);
231 logEntry.setRemote(parseRemoteSystem(line));
232 logEntry.setLogType(LOGTYPE.INFO);
233 }else if (line.contains("SEVERE") && line.contains("[main]")){
234 String[] splitString = line.split(" ");
236 for (int i = 5; i < splitString.length; i++){
237 description = description + " " + splitString[i];
240 logEntry.setDescription(description);
242 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
243 logEntry.setDate(date);
244 logEntry.setLogType(LOGTYPE.SEVERE);
245 } else if (line.contains("WARN") && line.contains(")-")){
246 //parse out description
248 logEntry.setDescription(line.substring(line.indexOf(")-")+3));
251 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
252 logEntry.setDate(date);
255 logEntry.setRemote(parseRemoteSystem(line));
256 logEntry.setLogType(LOGTYPE.WARN);
257 }else if (line.contains("WARNING") && type =="PyPDP"){
258 String[] splitString = line.split(" ");
259 for (int i = 5; i < splitString.length; i++){
260 description = description + " " + splitString[i];
264 date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
265 logEntry.setDate(date);
266 logEntry.setLogType(LOGTYPE.WARN);
267 }else if (line.contains("ERROR") && line.contains(")-")){
268 //parse out description
269 description = line.substring(line.indexOf(")-")+3);
272 date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
273 logEntry.setDate(date);
275 logEntry.setRemote(parseRemoteSystem(line));
276 logEntry.setLogType(LOGTYPE.ERROR);
285 private static void DBClose(Connection conn) {
288 } catch (SQLException e) {
289 logger.error("Error closing DB Connection: " + e);
294 public static void process(String line, String type) {
295 LogEntryObject returnLogValue = null;
298 im.startTransaction();
299 } catch (AdministrativeStateException e) {
300 logger.error("Error received" + e);
302 } catch (StandbyStatusException e) {
303 logger.error("Error received" + e);
306 returnLogValue = pullOutLogValues(line, type);
308 if (returnLogValue!=null){
309 writeDB(returnLogValue);
316 private static void writeDB(LogEntryObject returnLogValue) {
317 Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
318 DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
319 returnLogValue.getDate(), returnLogValue.getRemote(),
320 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
324 private static Connection DBConnection(String driver, String jdbc, String user, String pass){
327 Class.forName(driver);
328 Connection conn = DriverManager.getConnection(jdbc, user, pass);
330 } catch ( Exception e) {
331 logger.error("Error connecting to DB: " + e);
335 private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
340 Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
341 sdate = formatter.format(date);
344 //ensure the length of description is less than the maximumm db char length
345 if (description.length() > maxLength) {
346 description = description.substring(0, maxLength);
350 PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
351 prep.setString(1, system);
352 prep.setString(2, description);
353 prep.setString(3, remote);
354 prep.setString(4, type);
355 prep.setString(5, sdate);
356 prep.setString(6, logType);
358 prep.executeUpdate();
361 } catch (SQLException e1) {
362 logger.error("Error trying to excute SQL Statment: " + e1);
366 public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
369 String[] splitString = dateline.split(" ");
370 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
373 returnDate = formatter.parse(splitString[0]);
374 } catch (ParseException e) {
375 logger.error("Unable to parse date for line: " + dateline);
379 String tmpString = splitString[0] + " " + splitString[1];
381 returnDate = formatter.parse(tmpString);
382 } catch (ParseException e) {
383 logger.error("Unable to parse date for line: " + dateline);
392 public static String parseRemoteSystem(String line) {
394 if (line.contains("http") && !(line.contains("www.w3.org"))){
396 Pattern pattern = Pattern.compile("://(.+?)/");
397 Matcher remote = pattern.matcher(line);
400 return remote.group(1);
406 public static Properties getPropertiesValue(String fileName) {
407 Properties config = new Properties();
408 Path file = Paths.get(fileName);
409 if (Files.notExists(file)) {
410 logger.info("File doesn't exist in the specified Path " + file.toString());
412 if (file.toString().endsWith(".properties")) {
415 in = new FileInputStream(file.toFile());
418 resourceName = config.getProperty("RESOURCE_NAME");
419 system = config.getProperty("SERVER");
420 type = config.getProperty("LOGTYPE");
421 systemLogFile = config.getProperty("PARSERLOGPATH");
422 logFile = config.getProperty("LOGPATH");
423 JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
424 JDBC_USER = config.getProperty("JDBC_USER");
425 JDBC_DRIVER = config.getProperty("JDBC_DRIVER");
426 JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
429 } catch (IOException e) {
430 logger.info("Error porcessing Cofnig file will be unable to create Health Check");