cadi/aaf/src/main/java/org/onap/aaf/cadi/persist/Persist.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.cadi.persist;
  23
  24 import java.io.IOException;
  25 import java.nio.file.FileVisitResult;
  26 import java.nio.file.FileVisitor;
  27 import java.nio.file.Files;
  28 import java.nio.file.Path;
  29 import java.nio.file.attribute.BasicFileAttributes;
  30 import java.util.Date;
  31 import java.util.Map;
  32 import java.util.Map.Entry;
  33 import java.util.Queue;
  34 import java.util.Timer;
  35 import java.util.TimerTask;
  36 import java.util.concurrent.ConcurrentHashMap;
  37 import java.util.concurrent.ConcurrentLinkedQueue;
  38
  39 import org.onap.aaf.cadi.Access;
  40 import org.onap.aaf.cadi.CadiException;
  41 import org.onap.aaf.cadi.LocatorException;
  42 import org.onap.aaf.cadi.Access.Level;
  43 import org.onap.aaf.cadi.util.Holder;
  44 import org.onap.aaf.cadi.client.Result;
  45 import org.onap.aaf.misc.env.APIException;
  46 import org.onap.aaf.misc.env.util.Chrono;
  47 import org.onap.aaf.misc.rosetta.env.RosettaDF;
  48 import org.onap.aaf.misc.rosetta.env.RosettaEnv;
  49
  50 public abstract class Persist<T,CT extends Persistable<T>> extends PersistFile {
  51     private static final long ONE_DAY = 86400000L;
  52     private static final long CLEAN_CHECK = 2*60*1000L; // check every 2 mins
  53     private static Timer clean;
  54
  55     // store all the directories to review
  56     // No Concurrent HashSet, or at least, it is all implemented with HashMap in older versions
  57     private static Queue<Persist<?,?>> allPersists = new ConcurrentLinkedQueue<Persist<?,?>>();
  58
  59     private Map<String,CT> tmap;
  60     protected RosettaEnv env;
  61     private RosettaDF<T> df;
  62
  63
  64     public Persist(Access access, RosettaEnv env, Class<T> cls, String sub_dir) throws CadiException, APIException {
  65         super(access, sub_dir);
  66         this.env = env;
  67         df = env.newDataFactory(cls);
  68         tmap = new ConcurrentHashMap<>();
  69         synchronized(Persist.class) {
  70             if (clean==null) {
  71                 clean = new Timer(true);
  72                 clean.schedule(new Clean(access), 20000, CLEAN_CHECK);
  73             }
  74         }
  75         allPersists.add(this);
  76     }
  77
  78     public void close() {
  79         allPersists.remove(this);
  80     }
  81
  82     protected abstract CT newCacheable(T t, long expires_secsFrom1970, byte[] hash, Path path) throws APIException, IOException;
  83
  84     public RosettaDF<T> getDF() {
  85         return df;
  86     }
  87     public Result<CT> get(final String key, final byte[] hash, Loader<CT> rl) throws CadiException, APIException, LocatorException {
  88         if (key==null) {
  89             return null;
  90         }
  91         Holder<Path> hp = new Holder<Path>(null);
  92         CT ct = tmap.get(key);
  93         // Make sure cached Item is synced with Disk, but only even Minute to save Disk hits
  94         if (ct!=null && ct.checkSyncTime()) { // check File Time only every SYNC Period (2 min)
  95             if (ct.hasBeenTouched()) {
  96                 tmap.remove(key);
  97                 ct = null;
  98                 access.log(Level.DEBUG,"File for",key,"has been touched, removing memory entry");
  99             }
 100         }
 101
 102         // If not currently in memory, check with Disk (which might have been updated by other processes)
 103         if (ct==null) {
 104             Holder<Long> hl = new Holder<Long>(0L);
 105             T t;
 106             if ((t = readDisk(df, hash, key, hp, hl))!=null) {
 107                 try {
 108                     if ((ct = newCacheable(t,hl.get(),hash,hp.get()))!=null) {
 109                         tmap.put(key, ct);
 110                     }
 111                     access.log(Level.DEBUG,"Read Token from",key);
 112                 } catch (IOException e) {
 113                     access.log(e,"Reading Token from",key);
 114                 }
 115             } // if not read, then ct still==null
 116
 117             // If not in memory, or on disk, get from Remote... IF reloadable (meaning, isn't hitting too often, etc).
 118             if (ct==null || ct.checkReloadable()) {
 119                 // Load from external (if makes sense)
 120                 Result<CT> rtp = rl.load(key);
 121                 if (rtp.isOK()) {
 122                     ct = rtp.value;
 123                     try {
 124                         Path p = getPath(key);
 125                         writeDisk(df, ct.get(),ct.getHash(),p,ct.expires());
 126                         access.log(Level.DEBUG, "Writing token",key);
 127                     } catch (CadiException e) {
 128                         throw e;
 129                     } catch (Exception e) {
 130                         throw new CadiException(e);
 131                     }
 132                 } else {
 133                     return Result.err(rtp);
 134                 }
 135             }
 136
 137             if (ct!=null) {
 138                 tmap.put(key, ct);
 139             }
 140         } else {
 141             access.log(Level.DEBUG,"Found token in memory",key);
 142         }
 143         // ct can only be not-null here
 144         ct.inc();
 145         return Result.ok(200,ct);
 146     }
 147
 148     public void put(String key, CT ct) throws CadiException {
 149         writeDisk(df, ct.get(), ct.getHash(), key, ct.expires());
 150         tmap.put(key,ct);
 151     }
 152
 153     public void delete(String key) {
 154         tmap.remove(key);
 155         deleteFromDisk(key);
 156     }
 157
 158     public interface Loader<CT> {
 159         Result<CT> load(String key) throws APIException, CadiException, LocatorException;
 160     }
 161
 162     /**
 163      * Clean will examine resources, and remove those that have expired.
 164      *
 165      * If "highs" have been exceeded, then we'll expire 10% more the next time.  This will adjust after each run
 166      * without checking contents more than once, making a good average "high" in the minimum speed.
 167      *
 168      * @author Jonathan
 169      *
 170      */
 171     private static final class Clean extends TimerTask {
 172         private final Access access;
 173         private long hourly;
 174
 175         public Clean(Access access) {
 176             this.access = access;
 177             hourly=0;
 178         }
 179
 180         private static class Metrics {
 181             public int mexists = 0, dexists=0;
 182             public int mremoved = 0, dremoved=0;
 183         }
 184
 185         public void run() {
 186             final long now = System.currentTimeMillis();
 187             final long dayFromNow = now + ONE_DAY;
 188             final Metrics metrics = new Metrics();
 189             for (final Persist<?,?> persist : allPersists) {
 190                 // Clear memory
 191                 if (access.willLog(Level.DEBUG)) {
 192                     access.log(Level.DEBUG, "Persist: Cleaning memory cache for",persist.tokenPath.toAbsolutePath());
 193                 }
 194                 for (Entry<String, ?> es : persist.tmap.entrySet()) {
 195                     ++metrics.mexists;
 196                     Persistable<?> p = (Persistable<?>)es.getValue();
 197                     if (p.checkSyncTime()) {
 198                         if (p.count()==0) {
 199                             ++metrics.mremoved;
 200                             persist.tmap.remove(es.getKey());
 201                             access.printf(Level.DEBUG, "Persist: removed cached item %s from memory\n", es.getKey());
 202                         } else {
 203                             p.clearCount();
 204                         }
 205                     } else if (Files.exists(p.path())) {
 206
 207                     }
 208                 }
 209                 // Clear disk
 210                 try {
 211                     final StringBuilder sb = new StringBuilder();
 212                     Files.walkFileTree(persist.tokenPath, new FileVisitor<Path>() {
 213                         @Override
 214                         public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 215                             sb.setLength(0);
 216                             sb.append("Persist: Cleaning files from ");
 217                             sb.append(dir.toAbsolutePath());
 218                             return FileVisitResult.CONTINUE;
 219                         }
 220
 221                         @Override
 222                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 223                             if (attrs.isRegularFile()) {
 224                                 ++metrics.dexists;
 225                                 try {
 226
 227                                     long exp = persist.readExpiration(file)*1000; // readExpiration is seconds from 1970
 228                                     if (now > exp) {  // cover for bad token
 229                                         sb.append("\n\tFile ");
 230                                         sb.append(file.getFileName());
 231                                         sb.append(" expired ");
 232                                         sb.append(Chrono.dateTime(new Date(exp)));
 233                                         persist.deleteFromDisk(file);
 234                                         ++metrics.dremoved;
 235                                     } else if (exp > dayFromNow) {
 236                                         sb.append("\n\tFile ");
 237                                         sb.append(file.toString());
 238                                         sb.append(" data corrupted.");
 239                                         persist.deleteFromDisk(file);
 240                                         ++metrics.dremoved;
 241                                     }
 242                                 } catch (CadiException e) {
 243                                     sb.append("\n\tError reading File ");
 244                                     sb.append(file.toString());
 245                                     sb.append(". ");
 246                                     sb.append(e.getMessage());
 247                                     ++metrics.dremoved;
 248                                 }
 249
 250                             }
 251                             return FileVisitResult.CONTINUE;
 252                         }
 253
 254                         @Override
 255                         public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
 256                             access.log(Level.ERROR,"Error visiting file %s (%s)\n",file.toString(),exc.getMessage());
 257                             return FileVisitResult.CONTINUE;
 258                         }
 259
 260                         @Override
 261                         public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
 262                             access.log(Level.DEBUG, sb);
 263                             return FileVisitResult.CONTINUE;
 264                         }
 265
 266                     });
 267                 } catch (IOException e) {
 268                     access.log(e, "Exception while cleaning Persistance");
 269                 }
 270
 271             }
 272
 273             // We want to print some activity of Persistence Check at least hourly, even if no activity has occurred, but not litter the log if nothing is happening
 274             boolean go=false;
 275             Level level=Level.WARN;
 276             if (access.willLog(Level.INFO)) {
 277                 go = true;
 278                 level=Level.INFO;
 279             } else if (access.willLog(Level.WARN)) {
 280                 go = metrics.mremoved>0 || metrics.dremoved>0 || --hourly <= 0;
 281             }
 282
 283             if (go) {
 284                 access.printf(level, "Persist Cache: removed %d of %d items from memory and %d of %d from disk",
 285                     metrics.mremoved, metrics.mexists, metrics.dremoved, metrics.dexists);
 286                 hourly = 3600000/CLEAN_CHECK;
 287             }
 288         }
 289     }
 290
 291     /* (non-Javadoc)
 292      * @see java.lang.Object#finalize()
 293      */
 294     @Override
 295     protected void finalize() throws Throwable {
 296         close(); // can call twice.
 297     }
 298
 299
 300
 301 }