--- /dev/null
+/*-
+ * ============LICENSE_START=======================================================
+ * LogParser
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.xacml.parser;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.LineNumberReader;
+import java.io.RandomAccessFile;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.text.Format;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.openecomp.policy.common.im.AdministrativeStateException;
+import org.openecomp.policy.common.im.IntegrityMonitor;
+import org.openecomp.policy.common.im.StandbyStatusException;
+
+import org.apache.log4j.Logger;
+import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE;
+
+/**
+ * Parse log files and store the information in a H2 database.
+ *
+ *
+ */
+public class ParseLog {
+
+ private static final Logger logger = Logger.getLogger(ParseLog.class.getName());
+
+ private static String system;
+ private static int lastNumberRead = 0;
+ private static String type;
+ private static long startFileSize;
+ private static String systemLogFile;
+ private static String logFile;
+
+ private static String JDBC_URL;
+ private static String JDBC_USER;
+ private static String JDBC_PASSWORD = "";
+ private static String JDBC_DRIVER;
+ private static int maxLength = 255; //Max length that is allowed in the DB table
+ private static String resourceName;
+ private static long sleepTimer = 50000;
+ static IntegrityMonitor im;
+
+ public static void main(String[] args) throws Exception {
+
+ Properties logProperties = getPropertiesValue("parserlog.properties");
+ Path filePath = Paths.get(logFile);
+ File file = new File(logFile);
+ File fileLog = new File(systemLogFile);
+ startFileSize = file.length();
+
+ im = IntegrityMonitor.getInstance(resourceName,logProperties );
+
+ logger.info("System: " + system );
+ logger.info("System type: " + type );
+ logger.info("Logging File: " + systemLogFile );
+ logger.info("log file: " + logFile);
+ logger.info("JDBC_URL: " + JDBC_URL);
+ logger.info("JDBC_DRIVER: " + JDBC_DRIVER);
+
+ String filesRead = PullLastLineRead(fileLog);
+ if (filesRead!= null){
+ filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
+ lastNumberRead= Integer.parseInt(filesRead.trim());
+ }else{
+ lastNumberRead = 0;
+ }
+ startFileSize = countLines(logFile);
+ logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
+ if (startFileSize < lastNumberRead ){
+ logger.error("Filed Rolled: set Last number read to 0");
+ lastNumberRead = 0;
+ }
+ Runnable runnable = new Runnable (){
+ public void run(){
+ while (true){
+
+ if (file.isFile()){
+ try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> logger.info("Last line Read: " + lastNumberRead)).skip(lastNumberRead)) {
+
+ lines.forEachOrdered(line -> process(line, type));
+
+ } catch (IOException e) {
+ logger.error("Error processing line in log file: " + e);
+ }
+ }
+ try {
+ Thread.sleep(sleepTimer);
+ startFileSize = countLines(logFile);
+ } catch (InterruptedException | IOException e) {
+ logger.error("Error: " + e);
+ }
+
+ logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
+ if (startFileSize < lastNumberRead ){
+ logger.info("Failed Rolled: set Last number read to 0");
+ lastNumberRead = 0;
+ }
+ }
+ }
+ };
+
+ Thread thread = new Thread(runnable);
+ thread.start();
+
+ }
+
+ public static int countLines(String filename) throws IOException {
+ LineNumberReader reader = new LineNumberReader(new FileReader(filename));
+ int cnt = 0;
+ String lineRead = "";
+ while ((lineRead = reader.readLine()) != null) {}
+
+ cnt = reader.getLineNumber();
+ reader.close();
+ return cnt;
+ }
+
+ public static String PullLastLineRead(File file) throws IOException {
+ if(!file.exists()){
+ file.createNewFile();
+ return null;
+ }
+ RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
+ int lines = 0;
+ StringBuilder builder = new StringBuilder();
+ long length = file.length();
+ length--;
+ randomAccessFile.seek(length);
+ for(long seek = length; seek >= 0; --seek){
+ randomAccessFile.seek(seek);
+ char c = (char)randomAccessFile.read();
+ builder.append(c);
+ if(c == '\n'){
+ builder = builder.reverse();
+ if (builder.toString().contains("Last line Read:")){
+ String[] parseString = builder.toString().split("Last line Read:");
+ String returnValue = parseString[1].replace("\r", "");
+ return returnValue.trim();
+ //return parseString[2].replace("\r", "");
+ }
+ lines++;
+ builder = null;
+ builder = new StringBuilder();
+ }
+
+ }
+ return null;
+ }
+
+ public static LogEntryObject pullOutLogValues(String line, String type){
+ Date date;
+ LogEntryObject logEntry = new LogEntryObject();
+ logEntry.setSystemType(type);
+ String description = null;
+
+ logEntry.setSystem(system);
+
+ //Values for PDP/PAP log file
+ if(line.contains("||INFO||") || line.contains("||ERROR||")){
+ String[] splitString = line.split("[||]");
+ String dateString = splitString[0].substring(0, 19);
+ logEntry.setDescription(splitString[splitString.length-1]);
+
+ //parse out date
+ date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
+ logEntry.setDate(date);
+
+ logEntry.setRemote(parseRemoteSystem(line));
+ if (line.contains("||INFO||")){
+ logEntry.setLogType(LOGTYPE.INFO);
+ }else{
+ logEntry.setLogType(LOGTYPE.ERROR);
+ }
+ }else if (line.contains("INFO") && line.contains(")-")){
+ //parse out description
+ logEntry.setDescription(line.substring(line.indexOf(")-")+3));
+
+ date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
+ logEntry.setDate(date);
+
+ logEntry.setRemote(parseRemoteSystem(line));
+ logEntry.setLogType(LOGTYPE.INFO);
+ } else if (line.contains("INFO") && line.contains("--- [")){
+ //parse out description
+ String temp = line.substring(line.indexOf("---")+1);
+ String[] split = temp.split(":");
+
+ logEntry.setDescription(split[1]);
+
+ //parse out date
+ date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
+ logEntry.setDate(date);
+
+ //remote system
+ logEntry.setRemote(parseRemoteSystem(line));
+ logEntry.setLogType(LOGTYPE.INFO);
+ }else if (line.contains("SEVERE") && line.contains("[main]")){
+ String[] splitString = line.split(" ");
+
+ for (int i = 5; i < splitString.length; i++){
+ description = description + " " + splitString[i];
+ }
+
+ logEntry.setDescription(description);
+ //parse out date
+ date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
+ logEntry.setDate(date);
+ logEntry.setLogType(LOGTYPE.SEVERE);
+ } else if (line.contains("WARN") && line.contains(")-")){
+ //parse out description
+
+ logEntry.setDescription(line.substring(line.indexOf(")-")+3));
+
+ //parse out date
+ date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
+ logEntry.setDate(date);
+
+ //remote system
+ logEntry.setRemote(parseRemoteSystem(line));
+ logEntry.setLogType(LOGTYPE.WARN);
+ }else if (line.contains("WARNING") && type =="PyPDP"){
+ String[] splitString = line.split(" ");
+ for (int i = 5; i < splitString.length; i++){
+ description = description + " " + splitString[i];
+ }
+
+ //parse out date
+ date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
+ logEntry.setDate(date);
+ logEntry.setLogType(LOGTYPE.WARN);
+ }else if (line.contains("ERROR") && line.contains(")-")){
+ //parse out description
+ description = line.substring(line.indexOf(")-")+3);
+
+ //parse out date
+ date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
+ logEntry.setDate(date);
+ //remote system
+ logEntry.setRemote(parseRemoteSystem(line));
+ logEntry.setLogType(LOGTYPE.ERROR);
+ }else {
+ return null;
+ }
+
+
+ return logEntry;
+ }
+
+ private static void DBClose(Connection conn) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ logger.error("Error closing DB Connection: " + e);
+
+ }
+ }
+
+ public static void process(String line, String type) {
+ LogEntryObject returnLogValue = null;
+ if (im!=null){
+ try {
+ im.startTransaction();
+ } catch (AdministrativeStateException e) {
+ logger.error("Error received" + e);
+
+ } catch (StandbyStatusException e) {
+ logger.error("Error received" + e);
+ }
+ }
+ returnLogValue = pullOutLogValues(line, type);
+ lastNumberRead++;
+ if (returnLogValue!=null){
+ writeDB(returnLogValue);
+ }
+ if (im!=null){
+ im.endTransaction();
+ }
+ }
+
+ private static void writeDB(LogEntryObject returnLogValue) {
+ Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
+ DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
+ returnLogValue.getDate(), returnLogValue.getRemote(),
+ returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
+ DBClose(conn);
+ }
+
+ private static Connection DBConnection(String driver, String jdbc, String user, String pass){
+
+ try {
+ Class.forName(driver);
+ Connection conn = DriverManager.getConnection(jdbc, user, pass);
+ return conn;
+ } catch ( Exception e) {
+ logger.error("Error connecting to DB: " + e);
+ }
+ return null;
+ }
+ private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType) {
+
+ String sdate = null;
+
+ if (date!=null){
+ Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdate = formatter.format(date);
+ }
+
+ //ensure the length of description is less than the maximumm db char length
+ if (description.length() > maxLength) {
+ description = description.substring(0, maxLength);
+ }
+
+ try {
+ PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?, ?, ?, ?);");
+ prep.setString(1, system);
+ prep.setString(2, description);
+ prep.setString(3, remote);
+ prep.setString(4, type);
+ prep.setString(5, sdate);
+ prep.setString(6, logType);
+
+ prep.executeUpdate();
+ prep.close();
+
+ } catch (SQLException e1) {
+ logger.error("Error trying to excute SQL Statment: " + e1);
+ }
+ }
+
+ public static Date parseDate(String dateline, String pattern, boolean singleSplit) {
+
+ Date returnDate;
+ String[] splitString = dateline.split(" ");
+ SimpleDateFormat formatter = new SimpleDateFormat(pattern);
+ if (singleSplit){
+ try {
+ returnDate = formatter.parse(splitString[0]);
+ } catch (ParseException e) {
+ logger.error("Unable to parse date for line: " + dateline);
+ returnDate = null;
+ }
+ }else{
+ String tmpString = splitString[0] + " " + splitString[1];
+ try {
+ returnDate = formatter.parse(tmpString);
+ } catch (ParseException e) {
+ logger.error("Unable to parse date for line: " + dateline);
+ returnDate = null;
+ }
+ }
+
+ return returnDate;
+ }
+
+
+ public static String parseRemoteSystem(String line) {
+
+ if (line.contains("http") && !(line.contains("www.w3.org"))){
+
+ Pattern pattern = Pattern.compile("://(.+?)/");
+ Matcher remote = pattern.matcher(line);
+ if (remote.find())
+ {
+ return remote.group(1);
+ }
+ }
+ return null;
+ }
+
+ public static Properties getPropertiesValue(String fileName) {
+ Properties config = new Properties();
+ Path file = Paths.get(fileName);
+ if (Files.notExists(file)) {
+ logger.info("File doesn't exist in the specified Path " + file.toString());
+ }else{
+ if (file.toString().endsWith(".properties")) {
+ InputStream in;
+ try {
+ in = new FileInputStream(file.toFile());
+ config.load(in);
+
+ resourceName = config.getProperty("RESOURCE_NAME");
+ system = config.getProperty("SERVER");
+ type = config.getProperty("LOGTYPE");
+ systemLogFile = config.getProperty("PARSERLOGPATH");
+ logFile = config.getProperty("LOGPATH");
+ JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
+ JDBC_USER = config.getProperty("JDBC_USER");
+ JDBC_DRIVER = config.getProperty("JDBC_DRIVER");
+ JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
+ return config;
+
+ } catch (IOException e) {
+ logger.info("Error porcessing Cofnig file will be unable to create Health Check");
+ }
+
+ }
+ }
+ return null;
+ }
+}
Code Review / policy / engine.git / blobdiff