LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * LogParser
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.xacml.parser;
  22
  23 import java.io.File;
  24 import java.io.FileInputStream;
  25 import java.io.FileReader;
  26 import java.io.IOException;
  27 import java.io.InputStream;
  28 import java.io.LineNumberReader;
  29 import java.io.RandomAccessFile;
  30 import java.nio.charset.Charset;
  31 import java.nio.file.Files;
  32 import java.nio.file.Path;
  33 import java.nio.file.Paths;
  34 import java.sql.Connection;
  35 import java.sql.DriverManager;
  36 import java.sql.PreparedStatement;
  37 import java.sql.SQLException;
  38 import java.text.Format;
  39 import java.text.ParseException;
  40 import java.text.SimpleDateFormat;
  41 import java.util.Date;
  42 import java.util.Properties;
  43 import java.util.regex.Matcher;
  44 import java.util.regex.Pattern;
  45 import java.util.stream.Stream;
  46
  47 import org.openecomp.policy.common.im.AdministrativeStateException;
  48 import org.openecomp.policy.common.im.IntegrityMonitor;
  49 import org.openecomp.policy.common.im.StandbyStatusException;
  50
  51 import org.apache.log4j.Logger;
  52 import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE;
  53
  54 /**
  55  * Parse log files and store the information in a H2 database.
  56  *
  57  *
  58  */
  59 public class ParseLog {
  60
  61         private static final Logger logger = Logger.getLogger(ParseLog.class.getName());
  62
  63         private static String system;
  64         private static int lastNumberRead = 0;
  65         private static String type;
  66         private static long startFileSize;
  67         private static String systemLogFile;
  68         private static String logFile;
  69
  70         private static String JDBC_URL;
  71         private static String JDBC_USER;
  72         private static String JDBC_PASSWORD = "";
  73         private static String JDBC_DRIVER;
  74         private static int maxLength = 255;   //Max length that is allowed in the DB table
  75         private static String resourceName;
  76         private static long sleepTimer = 50000;
  77         static IntegrityMonitor im;
  78
  79         public static void main(String[] args) throws Exception {
  80
  81                 Properties logProperties = getPropertiesValue("parserlog.properties");
  82                 Path filePath = Paths.get(logFile);
  83                 File file = new File(logFile);
  84                 File fileLog = new File(systemLogFile);
  85                 startFileSize = file.length();
  86
  87                 im = IntegrityMonitor.getInstance(resourceName,logProperties );
  88
  89                 logger.info("System: " + system );
  90                 logger.info("System type: " + type );
  91                 logger.info("Logging File: " + systemLogFile );
  92                 logger.info("log file: " + logFile);
  93                 logger.info("JDBC_URL: " + JDBC_URL);
  94                 logger.info("JDBC_DRIVER: " + JDBC_DRIVER);
  95
  96                 String filesRead = PullLastLineRead(fileLog);
  97                 if (filesRead!= null){
  98                         filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
  99                         lastNumberRead= Integer.parseInt(filesRead.trim());
 100                 }else{
 101                         lastNumberRead = 0;
 102                 }
 103                 startFileSize =  countLines(logFile);
 104                 logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
 105                 if (startFileSize < lastNumberRead ){
 106                         logger.error("Filed Rolled: set Last number read to 0");
 107                         lastNumberRead = 0;
 108                 }
 109                 Runnable  runnable = new Runnable (){
 110                 public void run(){
 111                         while (true){
 112
 113                                 if (file.isFile()){
 114                                         try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> logger.info("Last line Read: " + lastNumberRead)).skip(lastNumberRead)) {
 115
 116                                                 lines.forEachOrdered(line -> process(line, type));
 117
 118                                         } catch (IOException e) {
 119                                                 logger.error("Error processing line in log file: " + e);
 120                                         }
 121                                 }
 122                                 try {
 123                                         Thread.sleep(sleepTimer);
 124                                         startFileSize =  countLines(logFile);
 125                                 } catch (InterruptedException | IOException e) {
 126                                         logger.error("Error: " + e);
 127                                 }
 128
 129                                 logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
 130                                 if (startFileSize < lastNumberRead ){
 131                                         logger.info("Failed Rolled: set Last number read to 0");
 132                                         lastNumberRead = 0;
 133                                 }
 134                         }
 135                 }
 136                 };
 137
 138                 Thread thread = new Thread(runnable);
 139                 thread.start();
 140
 141         }
 142
 143         public static int countLines(String filename) throws IOException {
 144             LineNumberReader reader  = new LineNumberReader(new FileReader(filename));
 145             int cnt = 0;
 146             String lineRead = "";
 147             while ((lineRead = reader.readLine()) != null) {}
 148
 149             cnt = reader.getLineNumber();
 150             reader.close();
 151             return cnt;
 152         }
 153
 154         public static String PullLastLineRead(File file) throws IOException {
 155                 if(!file.exists()){
 156                         file.createNewFile();
 157                         return null;
 158                 }
 159                 RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r");
 160         int lines = 0;
 161         StringBuilder builder = new StringBuilder();
 162         long length = file.length();
 163         length--;
 164         randomAccessFile.seek(length);
 165         for(long seek = length; seek >= 0; --seek){
 166             randomAccessFile.seek(seek);
 167             char c = (char)randomAccessFile.read();
 168             builder.append(c);
 169             if(c == '\n'){
 170                 builder = builder.reverse();
 171                 if (builder.toString().contains("Last line Read:")){
 172                         String[] parseString = builder.toString().split("Last line Read:");
 173                         String returnValue = parseString[1].replace("\r", "");
 174                         return returnValue.trim();
 175                         //return parseString[2].replace("\r", "");
 176                 }
 177                 lines++;
 178                 builder = null;
 179                 builder = new StringBuilder();
 180              }
 181
 182         }
 183                 return null;
 184         }
 185
 186         public static LogEntryObject pullOutLogValues(String line, String type){
 187                 Date date;
 188                 LogEntryObject logEntry = new LogEntryObject();
 189                 logEntry.setSystemType(type);
 190                 String description = null;
 191
 192                 logEntry.setSystem(system);
 193
 194                 //Values for PDP/PAP log file
 195                 if(line.contains("||INFO||") || line.contains("||ERROR||")){
 196                         String[] splitString = line.split("[||]");
 197                         String dateString = splitString[0].substring(0, 19);
 198                         logEntry.setDescription(splitString[splitString.length-1]);
 199
 200                         //parse out date
 201                         date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
 202                         logEntry.setDate(date);
 203
 204                         logEntry.setRemote(parseRemoteSystem(line));
 205                         if (line.contains("||INFO||")){
 206                                 logEntry.setLogType(LOGTYPE.INFO);
 207                         }else{
 208                                 logEntry.setLogType(LOGTYPE.ERROR);
 209                         }
 210                 }else if (line.contains("INFO") && line.contains(")-")){
 211                         //parse out description
 212                         logEntry.setDescription(line.substring(line.indexOf(")-")+3));
 213
 214                         date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
 215                         logEntry.setDate(date);
 216
 217                         logEntry.setRemote(parseRemoteSystem(line));
 218                         logEntry.setLogType(LOGTYPE.INFO);
 219                 } else if (line.contains("INFO") && line.contains("--- [")){
 220                         //parse out description
 221                         String temp = line.substring(line.indexOf("---")+1);
 222                         String[] split = temp.split(":");
 223
 224                         logEntry.setDescription(split[1]);
 225
 226                         //parse out date
 227                         date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
 228                         logEntry.setDate(date);
 229
 230                         //remote system
 231                         logEntry.setRemote(parseRemoteSystem(line));
 232                         logEntry.setLogType(LOGTYPE.INFO);
 233                 }else if (line.contains("SEVERE") && line.contains("[main]")){
 234                         String[] splitString = line.split(" ");
 235
 236                         for (int i = 5; i < splitString.length; i++){
 237                                 description = description +  " " + splitString[i];
 238                         }
 239
 240                         logEntry.setDescription(description);
 241                         //parse out date
 242                         date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
 243                         logEntry.setDate(date);
 244                         logEntry.setLogType(LOGTYPE.SEVERE);
 245                 } else if (line.contains("WARN") && line.contains(")-")){
 246                         //parse out description
 247
 248                         logEntry.setDescription(line.substring(line.indexOf(")-")+3));
 249
 250                         //parse out date
 251                         date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
 252                         logEntry.setDate(date);
 253
 254                         //remote system
 255                         logEntry.setRemote(parseRemoteSystem(line));
 256                         logEntry.setLogType(LOGTYPE.WARN);
 257                 }else if (line.contains("WARNING") && type =="PyPDP"){
 258                         String[] splitString = line.split(" ");
 259                         for (int i = 5; i < splitString.length; i++){
 260                                 description = description +  " " + splitString[i];
 261                         }
 262
 263                         //parse out date
 264                         date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
 265                         logEntry.setDate(date);
 266                         logEntry.setLogType(LOGTYPE.WARN);
 267                 }else if (line.contains("ERROR") && line.contains(")-")){
 268                         //parse out description
 269                         description = line.substring(line.indexOf(")-")+3);
 270
 271                         //parse out date
 272                         date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
 273                         logEntry.setDate(date);
 274                         //remote system
 275                         logEntry.setRemote(parseRemoteSystem(line));
 276                         logEntry.setLogType(LOGTYPE.ERROR);
 277                 }else {
 278                         return null;
 279                 }
 280
 281
 282                 return logEntry;
 283         }
 284
 285         private static void DBClose(Connection conn) {
 286                 try {
 287                         conn.close();
 288                 } catch (SQLException e) {
 289                         logger.error("Error closing DB Connection: " + e);
 290
 291                 }
 292         }
 293
 294         public static void process(String line, String type)  {
 295                 LogEntryObject returnLogValue = null;
 296                 if (im!=null){
 297                         try {
 298                                 im.startTransaction();
 299                         } catch (AdministrativeStateException e) {
 300                                 logger.error("Error received" + e);
 301
 302                         } catch (StandbyStatusException e) {
 303                                 logger.error("Error received" + e);
 304                         }
 305                 }
 306                 returnLogValue = pullOutLogValues(line, type);
 307                 lastNumberRead++;
 308                 if (returnLogValue!=null){
 309                         writeDB(returnLogValue);
 310                 }
 311                 if (im!=null){
 312                         im.endTransaction();
 313                 }
 314         }
 315
 316         private static void writeDB(LogEntryObject returnLogValue) {
 317                 Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
 318                 DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
 319                                                 returnLogValue.getDate(), returnLogValue.getRemote(),
 320                                                 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
 321                 DBClose(conn);
 322         }
 323
 324         private static Connection DBConnection(String driver, String jdbc, String user, String pass){
 325
 326         try {
 327                 Class.forName(driver);
 328                         Connection conn = DriverManager.getConnection(jdbc, user, pass);
 329                         return conn;
 330                 } catch ( Exception e) {
 331                         logger.error("Error connecting to DB: " + e);
 332                 }
 333                 return null;
 334         }
 335         private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType)  {
 336
 337                 String sdate = null;
 338
 339                 if (date!=null){
 340                         Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 341                         sdate = formatter.format(date);
 342                 }
 343
 344                 //ensure the length of description is less than the maximumm db char length
 345                 if (description.length() > maxLength) {
 346                         description = description.substring(0, maxLength);
 347                 }
 348
 349                 try {
 350                         PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?,  ?,  ?, ?);");
 351                         prep.setString(1, system);
 352                         prep.setString(2, description);
 353                         prep.setString(3, remote);
 354                         prep.setString(4, type);
 355                         prep.setString(5, sdate);
 356                         prep.setString(6, logType);
 357
 358                         prep.executeUpdate();
 359                         prep.close();
 360
 361                 } catch (SQLException e1) {
 362                         logger.error("Error trying to excute SQL Statment: " + e1);
 363                 }
 364         }
 365
 366         public static Date parseDate(String dateline, String pattern, boolean singleSplit)  {
 367
 368                 Date returnDate;
 369                 String[] splitString = dateline.split(" ");
 370                 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
 371                 if (singleSplit){
 372                         try {
 373                                 returnDate = formatter.parse(splitString[0]);
 374                         } catch (ParseException e) {
 375                                 logger.error("Unable to parse date for line: " + dateline);
 376                                 returnDate = null;
 377                         }
 378                 }else{
 379                         String tmpString = splitString[0] + " " + splitString[1];
 380                         try {
 381                                 returnDate = formatter.parse(tmpString);
 382                         } catch (ParseException e) {
 383                                 logger.error("Unable to parse date for line: " + dateline);
 384                                 returnDate = null;
 385                         }
 386                 }
 387
 388                 return returnDate;
 389         }
 390
 391
 392         public static String parseRemoteSystem(String line) {
 393
 394                 if (line.contains("http") && !(line.contains("www.w3.org"))){
 395
 396                         Pattern pattern = Pattern.compile("://(.+?)/");
 397                         Matcher remote = pattern.matcher(line);
 398                         if (remote.find())
 399                         {
 400                                 return remote.group(1);
 401                         }
 402                 }
 403                 return null;
 404         }
 405
 406         public static Properties getPropertiesValue(String fileName) {
 407                 Properties config = new Properties();
 408                 Path file = Paths.get(fileName);
 409                 if (Files.notExists(file)) {
 410                         logger.info("File doesn't exist in the specified Path " + file.toString());
 411                 }else{
 412                         if (file.toString().endsWith(".properties")) {
 413                                 InputStream in;
 414                                 try {
 415                                         in = new FileInputStream(file.toFile());
 416                                         config.load(in);
 417
 418                                         resourceName = config.getProperty("RESOURCE_NAME");
 419                                         system = config.getProperty("SERVER");
 420                                         type = config.getProperty("LOGTYPE");
 421                                         systemLogFile = config.getProperty("PARSERLOGPATH");
 422                                         logFile = config.getProperty("LOGPATH");
 423                                         JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
 424                                         JDBC_USER = config.getProperty("JDBC_USER");
 425                                         JDBC_DRIVER =  config.getProperty("JDBC_DRIVER");
 426                                         JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
 427                                         return config;
 428
 429                                 } catch (IOException e) {
 430                                         logger.info("Error porcessing Cofnig file will be unable to create Health Check");
 431                                 }
 432
 433                         }
 434                 }
 435                 return null;
 436         }
 437 }