auth/auth-core/src/main/java/org/onap/aaf/auth/rserv/CachingFileAccess.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.rserv;
  23
  24
  25 import java.io.File;
  26 import java.io.FileInputStream;
  27 import java.io.FileNotFoundException;
  28 import java.io.FileOutputStream;
  29 import java.io.FileReader;
  30 import java.io.IOException;
  31 import java.io.OutputStream;
  32 import java.io.OutputStreamWriter;
  33 import java.io.Writer;
  34 import java.nio.ByteBuffer;
  35 import java.nio.channels.FileChannel;
  36 import java.util.ArrayList;
  37 import java.util.Arrays;
  38 import java.util.Collections;
  39 import java.util.Comparator;
  40 import java.util.Date;
  41 import java.util.HashSet;
  42 import java.util.Map;
  43 import java.util.Map.Entry;
  44 import java.util.NavigableMap;
  45 import java.util.Set;
  46 import java.util.Timer;
  47 import java.util.TimerTask;
  48 import java.util.TreeMap;
  49 import java.util.concurrent.ConcurrentSkipListMap;
  50 import java.util.regex.Matcher;
  51 import java.util.regex.Pattern;
  52
  53 import javax.servlet.http.HttpServletRequest;
  54 import javax.servlet.http.HttpServletResponse;
  55
  56 import org.onap.aaf.misc.env.Env;
  57 import org.onap.aaf.misc.env.EnvJAXB;
  58 import org.onap.aaf.misc.env.LogTarget;
  59 import org.onap.aaf.misc.env.Store;
  60 import org.onap.aaf.misc.env.TimeTaken;
  61 import org.onap.aaf.misc.env.Trans;
  62 /*
  63  * CachingFileAccess
  64  *
  65  * Author: Jonathan Gathman, Gathsys 2010
  66  *
  67  */
  68 public class CachingFileAccess<TRANS extends Trans> extends HttpCode<TRANS, Void> {
  69         public static void setEnv(Store store, String[] args) {
  70                 for(int i=0;i<args.length-1;i+=2) { // cover two parms required for each
  71                         if(CFA_WEB_PATH.equals(args[i])) {
  72                                 store.put(store.staticSlot(CFA_WEB_PATH), args[i+1]);
  73                         } else if(CFA_CACHE_CHECK_INTERVAL.equals(args[i])) {
  74                                 store.put(store.staticSlot(CFA_CACHE_CHECK_INTERVAL), Long.parseLong(args[i+1]));
  75                         } else if(CFA_MAX_SIZE.equals(args[i])) {
  76                                 store.put(store.staticSlot(CFA_MAX_SIZE), Integer.parseInt(args[i+1]));
  77                         }
  78                 }
  79         }
  80
  81         private static String MAX_AGE = "max-age=3600"; // 1 hour Caching
  82         private final Map<String,String> typeMap;
  83         private final NavigableMap<String,Content> content;
  84         private final Set<String> attachOnly;
  85         public final static String CFA_WEB_PATH = "aaf_cfa_web_path";
  86         // when to re-validate from file
  87         // Re validating means comparing the Timestamp on the disk, and seeing it has changed.  Cache is not marked
  88         // dirty unless file has changed, but it still makes File IO, which for some kinds of cached data, i.e.
  89         // deployed GUI elements is unnecessary, and wastes time.
  90         // This parameter exists to cover the cases where data can be more volatile, so the user can choose how often the
  91         // File IO will be accessed, based on probability of change.  "0", of course, means, check every time.
  92         private final static String CFA_CACHE_CHECK_INTERVAL = "aaf_cfa_cache_check_interval";
  93         private final static String CFA_MAX_SIZE = "aaf_cfa_max_size"; // Cache size limit
  94         private final static String CFA_CLEAR_COMMAND = "aaf_cfa_clear_command";
  95
  96         // Note: can be null without a problem, but included
  97         // to tie in with existing Logging.
  98         public LogTarget logT = null;
  99         public long checkInterval; // = 600000L; // only check if not hit in 10 mins by default
 100         public int maxItemSize; // = 512000; // max file 500k
 101         private Timer timer;
 102         private String web_path;
 103         // A command key is set in the Properties, preferably changed on deployment.
 104         // it is compared at the beginning of the path, and if so, it is assumed to issue certain commands
 105         // It's purpose is to protect, to some degree the command, even though it is HTTP, allowing
 106         // local batch files to, for instance, clear caches on resetting of files.
 107         private String clear_command;
 108
 109         public CachingFileAccess(EnvJAXB env, String ... args) throws IOException {
 110                 super(null,"Caching File Access");
 111                 setEnv(env,args);
 112                 content = new ConcurrentSkipListMap<String,Content>(); // multi-thread changes possible
 113
 114                 attachOnly = new HashSet<String>();     // short, unchanged
 115
 116                 typeMap = new TreeMap<String,String>(); // Structure unchanged after Construction
 117                 typeMap.put("ico","image/icon");
 118                 typeMap.put("html","text/html");
 119                 typeMap.put("css","text/css");
 120                 typeMap.put("js","text/javascript");
 121                 typeMap.put("txt","text/plain");
 122                 typeMap.put("xml","text/xml");
 123                 typeMap.put("xsd","text/xml");
 124                 attachOnly.add("xsd");
 125                 typeMap.put("crl", "application/x-pkcs7-crl");
 126                 typeMap.put("appcache","text/cache-manifest");
 127
 128                 typeMap.put("json","text/json");
 129                 typeMap.put("ogg", "audio/ogg");
 130                 typeMap.put("jpg","image/jpeg");
 131                 typeMap.put("gif","image/gif");
 132                 typeMap.put("png","image/png");
 133                 typeMap.put("svg","image/svg+xml");
 134                 typeMap.put("jar","application/x-java-applet");
 135                 typeMap.put("jnlp", "application/x-java-jnlp-file");
 136                 typeMap.put("class", "application/java");
 137                 typeMap.put("props", "text/plain");
 138                 typeMap.put("jks", "application/octet-stream");
 139
 140                 timer = new Timer("Caching Cleanup",true);
 141                 timer.schedule(new Cleanup(content,500),60000,60000);
 142
 143                 // Property params
 144                 web_path = env.get(env.staticSlot(CFA_WEB_PATH));
 145                 env.init().log("CachingFileAccess path: " + new File(web_path).getCanonicalPath());
 146                 Object obj;
 147                 obj = env.get(env.staticSlot(CFA_CACHE_CHECK_INTERVAL),600000L);  // Default is 10 mins
 148                 if(obj instanceof Long) {checkInterval=(Long)obj;
 149                 } else {checkInterval=Long.parseLong((String)obj);}
 150
 151                 obj = env.get(env.staticSlot(CFA_MAX_SIZE), 512000);    // Default is max file 500k
 152                 if(obj instanceof Integer) {maxItemSize=(Integer)obj;
 153                 } else {maxItemSize =Integer.parseInt((String)obj);}
 154
 155                 clear_command = env.getProperty(CFA_CLEAR_COMMAND,null);
 156         }
 157
 158
 159
 160         @Override
 161         public void handle(TRANS trans, HttpServletRequest req, HttpServletResponse resp) throws IOException {
 162                 String key = pathParam(req, ":key");
 163                 String cmd = pathParam(req,":cmd");
 164                 System.out.print(key + clear_command);
 165                 if(key.equals(clear_command)) {
 166                         resp.setHeader("Content-Type",typeMap.get("txt"));
 167                         if("clear".equals(cmd)) {
 168                                 content.clear();
 169                                 resp.setStatus(200/*HttpStatus.OK_200*/);
 170                         } else {
 171                                 resp.setStatus(400/*HttpStatus.BAD_REQUEST_400 */);
 172                         }
 173                         return;
 174                 }
 175                 Content c = load(logT , web_path,cmd!=null && cmd.length()>0?key+'/'+cmd:key, null, checkInterval);
 176                 if(c.attachmentOnly) {
 177                         resp.setHeader("Content-disposition", "attachment");
 178                 }
 179                 c.setHeader(resp);
 180                 c.write(resp.getOutputStream());
 181                 trans.checkpoint(req.getPathInfo());
 182         }
 183
 184
 185         public String webPath() {
 186                 return web_path;
 187         }
 188
 189         /**
 190          * Reset the Cleanup size and interval
 191          *
 192          * The size and interval when started are 500 items (memory size unknown) checked every minute in a background thread.
 193          *
 194          * @param size
 195          * @param interval
 196          */
 197         public void cleanupParams(int size, long interval) {
 198                 timer.cancel();
 199                 timer = new Timer();
 200                 timer.schedule(new Cleanup(content,size), interval, interval);
 201         }
 202
 203
 204
 205         /**
 206          * Load a file, first checking cache
 207          *
 208          *
 209          * @param logTarget - logTarget can be null (won't log)
 210          * @param dataRoot - data root storage directory
 211          * @param key - relative File Path
 212          * @param mediaType - what kind of file is it.  If null, will check via file extension
 213          * @param timeCheck - "-1" will take system default - Otherwise, will compare "now" + timeCheck(Millis) before looking at File mod
 214          * @return
 215          * @throws IOException
 216          */
 217         public Content load(LogTarget logTarget, String dataRoot, String key, String mediaType, long _timeCheck) throws IOException {
 218             long timeCheck = _timeCheck;
 219                 if(timeCheck<0) {
 220                         timeCheck=checkInterval; // if time < 0, then use default
 221                 }
 222                 boolean isRoot;
 223                 String fileName;
 224                 if("-".equals(key)) {
 225                         fileName = dataRoot;
 226                         isRoot = true;
 227                 } else {
 228                         fileName=dataRoot + '/' + key;
 229                         isRoot = false;
 230                 }
 231                 Content c = content.get(key);
 232                 long systime = System.currentTimeMillis();
 233                 File f=null;
 234                 if(c!=null) {
 235                         // Don't check every hit... only after certain time value
 236                         if(c.date < systime + timeCheck) {
 237                                 f = new File(fileName);
 238                                 if(f.lastModified()>c.date) {
 239                                         c=null;
 240                                 }
 241                         }
 242                 }
 243                 if(c==null) {
 244                         if(logTarget!=null) {
 245                                 logTarget.log("File Read: ",key);
 246                         }
 247
 248                         if(f==null){
 249                                 f = new File(fileName);
 250                         }
 251                         boolean cacheMe;
 252                         if(f.exists()) {
 253                                 if(f.isDirectory()) {
 254                                         cacheMe = false;
 255                                         c = new DirectoryContent(f,isRoot);
 256                                 } else {
 257                                         if(f.length() > maxItemSize) {
 258                                                 c = new DirectFileContent(f);
 259                                                 cacheMe = false;
 260                                         } else {
 261                                                 c = new CachedContent(f);
 262                                                 cacheMe = checkInterval>0;
 263                                         }
 264
 265                                         if(mediaType==null) { // determine from file Ending
 266                                                 int idx = key.lastIndexOf('.');
 267                                                 String subkey = key.substring(++idx);
 268                                                 if((c.contentType = idx<0?null:typeMap.get(subkey))==null) {
 269                                                         // if nothing else, just set to default type...
 270                                                         c.contentType = "application/octet-stream";
 271                                                 }
 272                                                 c.attachmentOnly = attachOnly.contains(subkey);
 273                                         } else {
 274                                                 c.contentType=mediaType;
 275                                                 c.attachmentOnly = false;
 276                                         }
 277
 278                                         c.date = f.lastModified();
 279
 280                                         if(cacheMe) {
 281                                                 content.put(key, c);
 282                                         }
 283                                 }
 284                         } else {
 285                                 c=NULL;
 286                         }
 287                 } else {
 288                         if(logTarget!=null)logTarget.log("Cache Read: ",key);
 289                 }
 290
 291                 // refresh hit time
 292                 c.access = systime;
 293                 return c;
 294         }
 295
 296         public Content loadOrDefault(Trans trans, String targetDir, String targetFileName, String sourcePath, String mediaType) throws IOException {
 297                 try {
 298                         return load(trans.info(),targetDir,targetFileName,mediaType,0);
 299                 } catch(FileNotFoundException e) {
 300                         String targetPath = targetDir + '/' + targetFileName;
 301                         TimeTaken tt = trans.start("File doesn't exist; copy " + sourcePath + " to " + targetPath, Env.SUB);
 302                         try {
 303                                 FileInputStream sourceFIS = new FileInputStream(sourcePath);
 304                                 FileChannel sourceFC = sourceFIS.getChannel();
 305                                 File targetFile = new File(targetPath);
 306                                 targetFile.getParentFile().mkdirs(); // ensure directory exists
 307                                 FileOutputStream targetFOS = new FileOutputStream(targetFile);
 308                                 try {
 309                                         ByteBuffer bb = ByteBuffer.allocate((int)sourceFC.size());
 310                                         sourceFC.read(bb);
 311                                         bb.flip();  // ready for reading
 312                                         targetFOS.getChannel().write(bb);
 313                                 } finally {
 314                                         sourceFIS.close();
 315                                         targetFOS.close();
 316                                 }
 317                         } finally {
 318                                 tt.done();
 319                         }
 320                         return load(trans.info(),targetDir,targetFileName,mediaType,0);
 321                 }
 322         }
 323
 324         public void invalidate(String key) {
 325                 content.remove(key);
 326         }
 327
 328         private static final Content NULL=new Content() {
 329
 330                 @Override
 331                 public void setHeader(HttpServletResponse resp) {
 332                         resp.setStatus(404/*NOT_FOUND_404*/);
 333                         resp.setHeader("Content-type","text/plain");
 334                 }
 335
 336                 @Override
 337                 public void write(Writer writer) throws IOException {
 338                 }
 339
 340                 @Override
 341                 public void write(OutputStream os) throws IOException {
 342                 }
 343
 344         };
 345
 346         private static abstract class Content {
 347                 private long date;   // date of the actual artifact (i.e. File modified date)
 348                 private long access; // last accessed
 349
 350                 protected String  contentType;
 351                 protected boolean attachmentOnly;
 352
 353                 public void setHeader(HttpServletResponse resp) {
 354                         resp.setStatus(200/*OK_200*/);
 355                         resp.setHeader("Content-Type",contentType);
 356                         resp.setHeader("Cache-Control", MAX_AGE);
 357                 }
 358
 359                 public abstract void write(Writer writer) throws IOException;
 360                 public abstract void write(OutputStream os) throws IOException;
 361
 362         }
 363
 364         private static class DirectFileContent extends Content {
 365                 private File file;
 366                 public DirectFileContent(File f) {
 367                         file = f;
 368                 }
 369
 370                 public String toString() {
 371                         return file.getName();
 372                 }
 373
 374                 public void write(Writer writer) throws IOException {
 375                         FileReader fr = new FileReader(file);
 376                         char[] buff = new char[1024];
 377                         try {
 378                                 int read;
 379                                 while((read = fr.read(buff,0,1024))>=0) {
 380                                         writer.write(buff,0,read);
 381                                 }
 382                         } finally {
 383                                 fr.close();
 384                         }
 385                 }
 386
 387                 public void write(OutputStream os) throws IOException {
 388                         FileInputStream fis = new FileInputStream(file);
 389                         byte[] buff = new byte[1024];
 390                         try {
 391                                 int read;
 392                                 while((read = fis.read(buff,0,1024))>=0) {
 393                                         os.write(buff,0,read);
 394                                 }
 395                         } finally {
 396                                 fis.close();
 397                         }
 398                 }
 399
 400         }
 401         private static class DirectoryContent extends Content {
 402                 private static final Pattern A_NUMBER = Pattern.compile("\\d");
 403                 private static final String H1 = "<html><head><title>AAF Fileserver</title></head><body><h1>AAF Fileserver</h1><h2>";
 404                 private static final String H2 = "</h2><ul>\n";
 405                 private static final String F = "\n</ul></body></html>";
 406                 private File[] files;
 407                 private String name;
 408                 private boolean notRoot;
 409
 410                 public DirectoryContent(File directory, boolean isRoot) {
 411                         notRoot = !isRoot;
 412
 413                         files = directory.listFiles();
 414                         Arrays.sort(files,new Comparator<File>() {
 415                                 @Override
 416                                 public int compare(File f1, File f2) {
 417                                         // See if there are Numbers in the name
 418                                         Matcher m1 = A_NUMBER.matcher(f1.getName());
 419                                         Matcher m2 = A_NUMBER.matcher(f2.getName());
 420                                         if(m1.find() && m2.find()) {
 421                                                 // if numbers, are the numbers in the same start position
 422                                                 int i1 = m1.start();
 423                                                 int i2 = m2.start();
 424
 425                                                 // If same start position and the text is the same, then reverse sort
 426                                                 if(i1==i2 && f1.getName().startsWith(f2.getName().substring(0,i1))) {
 427                                                         // reverse sort files that start similarly, but have numbers in them
 428                                                         return f2.compareTo(f1);
 429                                                 }
 430                                         }
 431                                         return f1.compareTo(f2);
 432                                 }
 433
 434                         });
 435                         name = directory.getName();
 436                         attachmentOnly = false;
 437                         contentType = "text/html";
 438                 }
 439
 440
 441                 @Override
 442                 public void write(Writer w) throws IOException {
 443                         w.append(H1);
 444                         w.append(name);
 445                         w.append(H2);
 446                         for (File f : files) {
 447                                 w.append("<li><a href=\"");
 448                                 if(notRoot) {
 449                                         w.append(name);
 450                                         w.append('/');
 451                                 }
 452                                 w.append(f.getName());
 453                                 w.append("\">");
 454                                 w.append(f.getName());
 455                                 w.append("</a></li>\n");
 456                         }
 457                         w.append(F);
 458                         w.flush();
 459                 }
 460
 461                 @Override
 462                 public void write(OutputStream os) throws IOException {
 463                         write(new OutputStreamWriter(os));
 464                 }
 465
 466         }
 467
 468         private static class CachedContent extends Content {
 469                 private byte[] data;
 470                 private int end;
 471                 private char[] cdata;
 472
 473                 public CachedContent(File f) throws IOException {
 474                         // Read and Cache
 475                         ByteBuffer bb = ByteBuffer.allocate((int)f.length());
 476                         FileInputStream fis = new FileInputStream(f);
 477                         try {
 478                                 fis.getChannel().read(bb);
 479                         } finally {
 480                                 fis.close();
 481                         }
 482
 483                         data = bb.array();
 484                         end = bb.position();
 485                         cdata=null;
 486                 }
 487
 488                 public String toString() {
 489                         return data.toString();
 490                 }
 491
 492                 public void write(Writer writer) throws IOException {
 493                         synchronized(this) {
 494                                 // do the String Transformation once, and only if actually used
 495                                 if(cdata==null) {
 496                                         cdata = new char[end];
 497                                         new String(data).getChars(0, end, cdata, 0);
 498                                 }
 499                         }
 500                         writer.write(cdata,0,end);
 501                 }
 502                 public void write(OutputStream os) throws IOException {
 503                         os.write(data,0,end);
 504                 }
 505
 506         }
 507
 508         public void setEnv(LogTarget env) {
 509                 logT = env;
 510         }
 511
 512         /**
 513          * Cleanup thread to remove older items if max Cache is reached.
 514          * @author Jonathan
 515          *
 516          */
 517         private static class Cleanup extends TimerTask {
 518                 private int maxSize;
 519                 private NavigableMap<String, Content> content;
 520
 521                 public Cleanup(NavigableMap<String, Content> content, int size) {
 522                         maxSize = size;
 523                         this.content = content;
 524                 }
 525
 526                 private class Comp implements Comparable<Comp> {
 527                         public Map.Entry<String, Content> entry;
 528
 529                         public Comp(Map.Entry<String, Content> en) {
 530                                 entry = en;
 531                         }
 532
 533                         @Override
 534                         public int compareTo(Comp o) {
 535                                 return (int)(entry.getValue().access-o.entry.getValue().access);
 536                         }
 537
 538                 }
 539                 @SuppressWarnings("unchecked")
 540                 @Override
 541                 public void run() {
 542                         int size = content.size();
 543                         if(size>maxSize) {
 544                                 ArrayList<Comp> scont = new ArrayList<Comp>(size);
 545                                 Object[] entries = content.entrySet().toArray();
 546                                 for(int i=0;i<size;++i) {
 547                                         scont.add(i, new Comp((Map.Entry<String,Content>)entries[i]));
 548                                 }
 549                                 Collections.sort(scont);
 550                                 int end = size - ((maxSize/4)*3); // reduce to 3/4 of max size
 551                                 System.out.println("------ Cleanup Cycle ------ " + new Date().toString() + " -------");
 552                                 for(int i=0;i<end;++i) {
 553                                         Entry<String, Content> entry = scont.get(i).entry;
 554                                         content.remove(entry.getKey());
 555                                         System.out.println("removed Cache Item " + entry.getKey() + "/" + new Date(entry.getValue().access).toString());
 556                                 }
 557                                 for(int i=end;i<size;++i) {
 558                                         Entry<String, Content> entry = scont.get(i).entry;
 559                                         System.out.println("remaining Cache Item " + entry.getKey() + "/" + new Date(entry.getValue().access).toString());
 560                                 }
 561                         }
 562                 }
 563         }
 564 }