LogParser/src/main/java/org/openecomp/xacml/parser/ParseLog.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * LogParser
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.xacml.parser;
  22
  23 import java.io.File;
  24 import java.io.FileInputStream;
  25 import java.io.FileReader;
  26 import java.io.IOException;
  27 import java.io.InputStream;
  28 import java.io.LineNumberReader;
  29 import java.io.RandomAccessFile;
  30 import java.nio.charset.Charset;
  31 import java.nio.file.Files;
  32 import java.nio.file.Path;
  33 import java.nio.file.Paths;
  34 import java.sql.Connection;
  35 import java.sql.DriverManager;
  36 import java.sql.PreparedStatement;
  37 import java.sql.SQLException;
  38 import java.text.Format;
  39 import java.text.ParseException;
  40 import java.text.SimpleDateFormat;
  41 import java.util.Date;
  42 import java.util.Properties;
  43 import java.util.regex.Matcher;
  44 import java.util.regex.Pattern;
  45 import java.util.stream.Stream;
  46
  47 import org.apache.log4j.Logger;
  48 import org.openecomp.policy.common.im.AdministrativeStateException;
  49 import org.openecomp.policy.common.im.IntegrityMonitor;
  50 import org.openecomp.policy.common.im.StandbyStatusException;
  51 import org.openecomp.xacml.parser.LogEntryObject.LOGTYPE;
  52
  53 /**
  54  * Parse log files and store the information in a H2 database.
  55  *
  56  *
  57  */
  58 public class ParseLog {
  59
  60         private static final Logger logger = Logger.getLogger(ParseLog.class.getName());
  61
  62         private static String system;
  63         private static int lastNumberRead = 0;
  64         private static String type;
  65         private static long startFileSize;
  66         private static String systemLogFile;
  67         private static String logFile;
  68
  69         private static String JDBC_URL;
  70         private static String JDBC_USER;
  71         private static String JDBC_PASSWORD = "";
  72         private static String JDBC_DRIVER;
  73         private static int maxLength = 255;   //Max length that is allowed in the DB table
  74         private static String resourceName;
  75         private static long sleepTimer = 50000;
  76         static IntegrityMonitor im;
  77
  78         private static RandomAccessFile randomAccessFile;
  79
  80         public static void main(String[] args) throws Exception {
  81
  82                 Properties logProperties = getPropertiesValue("parserlog.properties");
  83                 Path filePath = Paths.get(logFile);
  84                 File file = new File(logFile);
  85                 File fileLog = new File(systemLogFile);
  86                 startFileSize = file.length();
  87
  88                 im = IntegrityMonitor.getInstance(resourceName,logProperties );
  89
  90                 logger.info("System: " + system );
  91                 logger.info("System type: " + type );
  92                 logger.info("Logging File: " + systemLogFile );
  93                 logger.info("log file: " + logFile);
  94                 logger.info("JDBC_URL: " + JDBC_URL);
  95                 logger.info("JDBC_DRIVER: " + JDBC_DRIVER);
  96
  97                 String filesRead = PullLastLineRead(fileLog);
  98                 if (filesRead!= null){
  99                         filesRead = filesRead.replaceAll("(\\r\\n|\\n)", "<br />");
 100                         lastNumberRead= Integer.parseInt(filesRead.trim());
 101                 }else{
 102                         lastNumberRead = 0;
 103                 }
 104                 startFileSize =  countLines(logFile);
 105                 logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
 106                 if (startFileSize < lastNumberRead ){
 107                         logger.error("Filed Rolled: set Last number read to 0");
 108                         lastNumberRead = 0;
 109                 }
 110                 Runnable  runnable = new Runnable (){
 111                 public void run(){
 112                         while (true){
 113
 114                                 if (file.isFile()){
 115                                         try (Stream<String> lines = Files.lines(filePath, Charset.defaultCharset()).onClose(() -> logger.info("Last line Read: " + lastNumberRead)).skip(lastNumberRead)) {
 116
 117                                                 lines.forEachOrdered(line -> process(line, type));
 118
 119                                         } catch (IOException e) {
 120                                                 logger.error("Error processing line in log file: " + e);
 121                                         }
 122                                 }
 123                                 try {
 124                                         Thread.sleep(sleepTimer);
 125                                         startFileSize =  countLines(logFile);
 126                                 } catch (InterruptedException | IOException e) {
 127                                         logger.error("Error: " + e);
 128                                 }
 129
 130                                 logger.info("File Line Count: " + startFileSize + " value read in: " + lastNumberRead);
 131                                 if (startFileSize < lastNumberRead ){
 132                                         logger.info("Failed Rolled: set Last number read to 0");
 133                                         lastNumberRead = 0;
 134                                 }
 135                         }
 136                 }
 137                 };
 138
 139                 Thread thread = new Thread(runnable);
 140                 thread.start();
 141
 142         }
 143
 144         public static int countLines(String filename) throws IOException {
 145             LineNumberReader reader  = new LineNumberReader(new FileReader(filename));
 146             int cnt = 0;
 147             while ((reader.readLine()) != null);
 148             cnt = reader.getLineNumber();
 149             reader.close();
 150             return cnt;
 151         }
 152
 153         public static String PullLastLineRead(File file) throws IOException {
 154                 if(!file.exists()){
 155                         file.createNewFile();
 156                         return null;
 157                 }
 158                 randomAccessFile = new RandomAccessFile(file, "r");
 159         StringBuilder builder = new StringBuilder();
 160         long length = file.length();
 161         length--;
 162         randomAccessFile.seek(length);
 163         for(long seek = length; seek >= 0; --seek){
 164             randomAccessFile.seek(seek);
 165             char c = (char)randomAccessFile.read();
 166             builder.append(c);
 167             if(c == '\n'){
 168                 builder = builder.reverse();
 169                 if (builder.toString().contains("Last line Read:")){
 170                         String[] parseString = builder.toString().split("Last line Read:");
 171                         String returnValue = parseString[1].replace("\r", "");
 172                         return returnValue.trim();
 173                 }
 174                 builder = null;
 175                 builder = new StringBuilder();
 176              }
 177
 178         }
 179                 return null;
 180         }
 181
 182         public static LogEntryObject pullOutLogValues(String line, String type){
 183                 Date date;
 184                 LogEntryObject logEntry = new LogEntryObject();
 185                 logEntry.setSystemType(type);
 186                 String description = null;
 187
 188                 logEntry.setSystem(system);
 189
 190                 //Values for PDP/PAP log file
 191                 if(line.contains("||INFO||") || line.contains("||ERROR||")){
 192                         String[] splitString = line.split("[||]");
 193                         String dateString = splitString[0].substring(0, 19);
 194                         logEntry.setDescription(splitString[splitString.length-1]);
 195
 196                         //parse out date
 197                         date = parseDate(dateString.replace("T", " "), "yyyy-MM-dd HH:mm:ss", false);
 198                         logEntry.setDate(date);
 199
 200                         logEntry.setRemote(parseRemoteSystem(line));
 201                         if (line.contains("||INFO||")){
 202                                 logEntry.setLogType(LOGTYPE.INFO);
 203                         }else{
 204                                 logEntry.setLogType(LOGTYPE.ERROR);
 205                         }
 206                 }else if (line.contains("INFO") && line.contains(")-")){
 207                         //parse out description
 208                         logEntry.setDescription(line.substring(line.indexOf(")-")+3));
 209
 210                         date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
 211                         logEntry.setDate(date);
 212
 213                         logEntry.setRemote(parseRemoteSystem(line));
 214                         logEntry.setLogType(LOGTYPE.INFO);
 215                 } else if (line.contains("INFO") && line.contains("--- [")){
 216                         //parse out description
 217                         String temp = line.substring(line.indexOf("---")+1);
 218                         String[] split = temp.split(":");
 219
 220                         logEntry.setDescription(split[1]);
 221
 222                         //parse out date
 223                         date = parseDate(line, "yyyy-MM-dd HH:mm:ss", false);
 224                         logEntry.setDate(date);
 225
 226                         //remote system
 227                         logEntry.setRemote(parseRemoteSystem(line));
 228                         logEntry.setLogType(LOGTYPE.INFO);
 229                 }else if (line.contains("SEVERE") && line.contains("[main]")){
 230                         String[] splitString = line.split(" ");
 231
 232                         for (int i = 5; i < splitString.length; i++){
 233                                 description = description +  " " + splitString[i];
 234                         }
 235
 236                         logEntry.setDescription(description);
 237                         //parse out date
 238                         date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
 239                         logEntry.setDate(date);
 240                         logEntry.setLogType(LOGTYPE.SEVERE);
 241                 } else if (line.contains("WARN") && line.contains(")-")){
 242                         //parse out description
 243
 244                         logEntry.setDescription(line.substring(line.indexOf(")-")+3));
 245
 246                         //parse out date
 247                         date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
 248                         logEntry.setDate(date);
 249
 250                         //remote system
 251                         logEntry.setRemote(parseRemoteSystem(line));
 252                         logEntry.setLogType(LOGTYPE.WARN);
 253                 }else if (line.contains("WARNING") && type =="PyPDP"){
 254                         String[] splitString = line.split(" ");
 255                         for (int i = 5; i < splitString.length; i++){
 256                                 description = description +  " " + splitString[i];
 257                         }
 258
 259                         //parse out date
 260                         date = parseDate(line, "dd-MMM-yyyy HH:mm:ss", false);
 261                         logEntry.setDate(date);
 262                         logEntry.setLogType(LOGTYPE.WARN);
 263                 }else if (line.contains("ERROR") && line.contains(")-")){
 264                         //parse out description
 265                         description = line.substring(line.indexOf(")-")+3);
 266
 267                         //parse out date
 268                         date = parseDate(line, "yy_MM_dd_HH_mm_ss", true);
 269                         logEntry.setDate(date);
 270                         //remote system
 271                         logEntry.setRemote(parseRemoteSystem(line));
 272                         logEntry.setLogType(LOGTYPE.ERROR);
 273                 }else {
 274                         return null;
 275                 }
 276
 277
 278                 return logEntry;
 279         }
 280
 281         private static void DBClose(Connection conn) {
 282                 try {
 283                         conn.close();
 284                 } catch (SQLException e) {
 285                         logger.error("Error closing DB Connection: " + e);
 286
 287                 }
 288         }
 289
 290         public static void process(String line, String type)  {
 291                 LogEntryObject returnLogValue = null;
 292                 if (im!=null){
 293                         try {
 294                                 im.startTransaction();
 295                         } catch (AdministrativeStateException e) {
 296                                 logger.error("Error received" + e);
 297
 298                         } catch (StandbyStatusException e) {
 299                                 logger.error("Error received" + e);
 300                         }
 301                 }
 302                 returnLogValue = pullOutLogValues(line, type);
 303                 lastNumberRead++;
 304                 if (returnLogValue!=null){
 305                         writeDB(returnLogValue);
 306                 }
 307                 if (im!=null){
 308                         im.endTransaction();
 309                 }
 310         }
 311
 312         private static void writeDB(LogEntryObject returnLogValue) {
 313                 Connection conn = DBConnection(JDBC_DRIVER, JDBC_URL, JDBC_USER,JDBC_PASSWORD);
 314                 DBAccesss(conn, returnLogValue.getSystem(), returnLogValue.getDescription(),
 315                                                 returnLogValue.getDate(), returnLogValue.getRemote(),
 316                                                 returnLogValue.getSystemType(), returnLogValue.getLogType().toString());
 317                 DBClose(conn);
 318         }
 319
 320         private static Connection DBConnection(String driver, String jdbc, String user, String pass){
 321
 322         try {
 323                 Class.forName(driver);
 324                         Connection conn = DriverManager.getConnection(jdbc, user, pass);
 325                         return conn;
 326                 } catch ( Exception e) {
 327                         logger.error("Error connecting to DB: " + e);
 328                 }
 329                 return null;
 330         }
 331         private static void DBAccesss(Connection conn, String system, String description, Date date, String remote, String type, String logType)  {
 332
 333                 String sdate = null;
 334
 335                 if (date!=null){
 336                         Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 337                         sdate = formatter.format(date);
 338                 }
 339
 340                 //ensure the length of description is less than the maximumm db char length
 341                 if (description.length() > maxLength) {
 342                         description = description.substring(0, maxLength);
 343                 }
 344
 345                 try {
 346                         PreparedStatement prep = conn.prepareStatement("insert into SYSTEMLOGDB values (NULL, ?, ?, ?,  ?,  ?, ?);");
 347                         prep.setString(1, system);
 348                         prep.setString(2, description);
 349                         prep.setString(3, remote);
 350                         prep.setString(4, type);
 351                         prep.setString(5, sdate);
 352                         prep.setString(6, logType);
 353
 354                         prep.executeUpdate();
 355                         prep.close();
 356
 357                 } catch (SQLException e1) {
 358                         logger.error("Error trying to excute SQL Statment: " + e1);
 359                 }
 360         }
 361
 362         public static Date parseDate(String dateline, String pattern, boolean singleSplit)  {
 363
 364                 Date returnDate;
 365                 String[] splitString = dateline.split(" ");
 366                 SimpleDateFormat formatter = new SimpleDateFormat(pattern);
 367                 if (singleSplit){
 368                         try {
 369                                 returnDate = formatter.parse(splitString[0]);
 370                         } catch (ParseException e) {
 371                                 logger.error("Unable to parse date for line: " + dateline);
 372                                 returnDate = null;
 373                         }
 374                 }else{
 375                         String tmpString = splitString[0] + " " + splitString[1];
 376                         try {
 377                                 returnDate = formatter.parse(tmpString);
 378                         } catch (ParseException e) {
 379                                 logger.error("Unable to parse date for line: " + dateline);
 380                                 returnDate = null;
 381                         }
 382                 }
 383
 384                 return returnDate;
 385         }
 386
 387
 388         public static String parseRemoteSystem(String line) {
 389
 390                 if (line.contains("http") && !(line.contains("www.w3.org"))){
 391
 392                         Pattern pattern = Pattern.compile("://(.+?)/");
 393                         Matcher remote = pattern.matcher(line);
 394                         if (remote.find())
 395                         {
 396                                 return remote.group(1);
 397                         }
 398                 }
 399                 return null;
 400         }
 401
 402         public static Properties getPropertiesValue(String fileName) {
 403                 Properties config = new Properties();
 404                 Path file = Paths.get(fileName);
 405                 if (Files.notExists(file)) {
 406                         logger.info("File doesn't exist in the specified Path " + file.toString());
 407                 }else{
 408                         if (file.toString().endsWith(".properties")) {
 409                                 InputStream in;
 410                                 try {
 411                                         in = new FileInputStream(file.toFile());
 412                                         config.load(in);
 413
 414                                         resourceName = config.getProperty("RESOURCE_NAME");
 415                                         system = config.getProperty("SERVER");
 416                                         type = config.getProperty("LOGTYPE");
 417                                         systemLogFile = config.getProperty("PARSERLOGPATH");
 418                                         logFile = config.getProperty("LOGPATH");
 419                                         JDBC_URL = config.getProperty("JDBC_URL").replace("'", "");
 420                                         JDBC_USER = config.getProperty("JDBC_USER");
 421                                         JDBC_DRIVER =  config.getProperty("JDBC_DRIVER");
 422                                         JDBC_PASSWORD = config.getProperty("JDBC_PASSWORD");
 423                                         return config;
 424
 425                                 } catch (IOException e) {
 426                                         logger.info("Error porcessing Cofnig file will be unable to create Health Check");
 427                                 }
 428
 429                         }
 430                 }
 431                 return null;
 432         }
 433 }