cadi/aaf/src/main/java/org/onap/aaf/cadi/persist/Persist.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.cadi.persist;
  23
  24 import java.io.IOException;
  25 import java.nio.file.FileVisitResult;
  26 import java.nio.file.FileVisitor;
  27 import java.nio.file.Files;
  28 import java.nio.file.Path;
  29 import java.nio.file.attribute.BasicFileAttributes;
  30 import java.util.Date;
  31 import java.util.Map;
  32 import java.util.Map.Entry;
  33 import java.util.Queue;
  34 import java.util.Timer;
  35 import java.util.TimerTask;
  36 import java.util.concurrent.ConcurrentHashMap;
  37 import java.util.concurrent.ConcurrentLinkedQueue;
  38
  39 import org.onap.aaf.cadi.Access;
  40 import org.onap.aaf.cadi.CadiException;
  41 import org.onap.aaf.cadi.LocatorException;
  42 import org.onap.aaf.cadi.Access.Level;
  43 import org.onap.aaf.cadi.client.Holder;
  44 import org.onap.aaf.cadi.client.Result;
  45 import org.onap.aaf.misc.env.APIException;
  46 import org.onap.aaf.misc.env.util.Chrono;
  47 import org.onap.aaf.misc.rosetta.env.RosettaDF;
  48 import org.onap.aaf.misc.rosetta.env.RosettaEnv;
  49
  50 public abstract class Persist<T,CT extends Persistable<T>> extends PersistFile {
  51         private static final long ONE_DAY = 86400000L;
  52         private static final long CLEAN_CHECK = 2*60*1000L; // check every 2 mins
  53         private static Timer clean;
  54
  55         // store all the directories to review
  56         // No Concurrent HashSet, or at least, it is all implemented with HashMap in older versions
  57         private static Queue<Persist<?,?>> allPersists = new ConcurrentLinkedQueue<Persist<?,?>>();
  58
  59         private Map<String,CT> tmap;
  60         protected RosettaEnv env;
  61         private RosettaDF<T> df;
  62
  63
  64         public Persist(Access access, RosettaEnv env, Class<T> cls, String sub_dir) throws CadiException, APIException {
  65                 super(access, sub_dir);
  66                 this.env = env;
  67                 df = env.newDataFactory(cls);
  68                 tmap = new ConcurrentHashMap<>();
  69                 synchronized(Persist.class) {
  70                         if(clean==null) {
  71                                 clean = new Timer(true);
  72                                 clean.schedule(new Clean(access), 20000, CLEAN_CHECK);
  73                         }
  74                 }
  75                 allPersists.add(this);
  76         }
  77
  78         public void close() {
  79                 allPersists.remove(this);
  80         }
  81
  82         protected abstract CT newCacheable(T t, long expires_secsFrom1970, byte[] hash, Path path) throws APIException, IOException;
  83
  84         public RosettaDF<T> getDF() {
  85                 return df;
  86         }
  87         public Result<CT> get(final String key, final byte[] hash, Loader<CT> rl) throws CadiException, APIException, LocatorException {
  88                 if(key==null) {
  89                         return null;
  90                 }
  91                 Holder<Path> hp = new Holder<Path>(null);
  92                 CT ct = tmap.get(key);
  93                 // Make sure cached Item is synced with Disk, but only even Minute to save Disk hits
  94                 if(ct!=null && ct.checkSyncTime()) { // check File Time only every SYNC Period (2 min)
  95                         if(ct.hasBeenTouched()) {
  96                                 tmap.remove(key);
  97                                 ct = null;
  98                                 access.log(Level.DEBUG,"File for",key,"has been touched, removing memory entry");
  99                         }
 100                 }
 101
 102                 // If not currently in memory, check with Disk (which might have been updated by other processes)
 103                 if(ct==null) {
 104                         Holder<Long> hl = new Holder<Long>(0L);
 105                         T t;
 106                         if((t = readDisk(df, hash, key, hp, hl))!=null) {
 107                                 try {
 108                                         if((ct = newCacheable(t,hl.get(),hash,hp.get()))!=null) {
 109                                                 tmap.put(key, ct);
 110                                         }
 111                                         access.log(Level.DEBUG,"Read Token from",key);
 112                                 } catch (IOException e) {
 113                                         access.log(e,"Reading Token from",key);
 114                                 }
 115                         } // if not read, then ct still==null
 116
 117                         // If not in memory, or on disk, get from Remote... IF reloadable (meaning, isn't hitting too often, etc).
 118                         if(ct==null || ct.checkReloadable()) {
 119                                 // Load from external (if makes sense)
 120                                 Result<CT> rtp = rl.load(key);
 121                                 if(rtp.isOK()) {
 122                                         ct = rtp.value;
 123                                         try {
 124                                                 Path p = getPath(key);
 125                                                 writeDisk(df, ct.get(),ct.getHash(),p,ct.expires());
 126                                                 access.log(Level.DEBUG, "Writing token",key);
 127                                         } catch(CadiException e) {
 128                                                 throw e;
 129                                         } catch (Exception e) {
 130                                                 throw new CadiException(e);
 131                                         }
 132                                 } else {
 133                                         return Result.err(rtp);
 134                                 }
 135                         }
 136
 137                         if(ct!=null) {
 138                                 tmap.put(key, ct);
 139                         }
 140                 } else {
 141                         access.log(Level.DEBUG,"Found token in memory",key);
 142                 }
 143                 // ct can only be not-null here
 144                 ct.inc();
 145                 return Result.ok(200,ct);
 146         }
 147
 148         public void put(String key, CT ct) throws CadiException {
 149                 writeDisk(df, ct.get(), ct.getHash(), key, ct.expires());
 150                 tmap.put(key,ct);
 151         }
 152
 153         public void delete(String key) {
 154                 tmap.remove(key);
 155                 deleteFromDisk(key);
 156         }
 157
 158         public interface Loader<CT> {
 159                 Result<CT> load(String key) throws APIException, CadiException, LocatorException;
 160         }
 161
 162         /**
 163          * Clean will examine resources, and remove those that have expired.
 164          *
 165          * If "highs" have been exceeded, then we'll expire 10% more the next time.  This will adjust after each run
 166          * without checking contents more than once, making a good average "high" in the minimum speed.
 167          *
 168          * @author Jonathan
 169          *
 170          */
 171         private static final class Clean extends TimerTask {
 172                 private final Access access;
 173                 private long hourly;
 174
 175                 public Clean(Access access) {
 176                         this.access = access;
 177                         hourly=0;
 178                 }
 179
 180                 private static class Metrics {
 181                         public int mexists = 0, dexists=0;
 182                         public int mremoved = 0, dremoved=0;
 183                 }
 184
 185                 public void run() {
 186                         final long now = System.currentTimeMillis();
 187                         final long dayFromNow = now + ONE_DAY;
 188                         final Metrics metrics = new Metrics();
 189                         for(final Persist<?,?> persist : allPersists) {
 190                                 // Clear memory
 191                                 if(access.willLog(Level.DEBUG)) {
 192                                         access.log(Level.DEBUG, "Persist: Cleaning memory cache for",persist.tokenPath.toAbsolutePath());
 193                                 }
 194                                 for(Entry<String, ?> es : persist.tmap.entrySet()) {
 195                                         ++metrics.mexists;
 196                                         Persistable<?> p = (Persistable<?>)es.getValue();
 197                                         if(p.checkSyncTime()) {
 198                                                 if(p.count()==0) {
 199                                                         ++metrics.mremoved;
 200                                                         persist.tmap.remove(es.getKey());
 201                                                         access.printf(Level.DEBUG, "Persist: removed cached item %s from memory\n", es.getKey());
 202                                                 } else {
 203                                                         p.clearCount();
 204                                                 }
 205                                         } else if(Files.exists(p.path())) {
 206
 207                                         }
 208                                 }
 209                                 // Clear disk
 210                                 try {
 211                                         final StringBuilder sb = new StringBuilder();
 212                                         Files.walkFileTree(persist.tokenPath, new FileVisitor<Path>() {
 213                                                 @Override
 214                                                 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 215                                                         sb.setLength(0);
 216                                                         sb.append("Persist: Cleaning files from ");
 217                                                         sb.append(dir.toAbsolutePath());
 218                                                         return FileVisitResult.CONTINUE;
 219                                                 }
 220
 221                                                 @Override
 222                                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 223                                                         if(attrs.isRegularFile()) {
 224                                                                 ++metrics.dexists;
 225                                                                 try {
 226
 227                                                                         long exp = persist.readExpiration(file)*1000; // readExpiration is seconds from 1970
 228                                                                         if(now > exp) {  // cover for bad token
 229                                                                                 sb.append("\n\tFile ");
 230                                                                                 sb.append(file.getFileName());
 231                                                                                 sb.append(" expired ");
 232                                                                                 sb.append(Chrono.dateTime(new Date(exp)));
 233                                                                                 persist.deleteFromDisk(file);
 234                                                                                 ++metrics.dremoved;
 235                                                                         } else if(exp > dayFromNow) {
 236                                                                                 sb.append("\n\tFile ");
 237                                                                                 sb.append(file.toString());
 238                                                                                 sb.append(" data corrupted.");
 239                                                                                 persist.deleteFromDisk(file);
 240                                                                                 ++metrics.dremoved;
 241                                                                         }
 242                                                                 } catch (CadiException e) {
 243                                                                         sb.append("\n\tError reading File ");
 244                                                                         sb.append(file.toString());
 245                                                                         sb.append(". ");
 246                                                                         sb.append(e.getMessage());
 247                                                                         ++metrics.dremoved;
 248                                                                 }
 249
 250                                                         }
 251                                                         return FileVisitResult.CONTINUE;
 252                                                 }
 253
 254                                                 @Override
 255                                                 public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
 256                                                         access.log(Level.ERROR,"Error visiting file %s (%s)\n",file.toString(),exc.getMessage());
 257                                                         return FileVisitResult.CONTINUE;
 258                                                 }
 259
 260                                                 @Override
 261                                                 public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
 262                                                         access.log(Level.DEBUG, sb);
 263                                                         return FileVisitResult.CONTINUE;
 264                                                 }
 265
 266                                         });
 267                                 } catch (IOException e) {
 268                                         access.log(e, "Exception while cleaning Persistance");
 269                                 }
 270
 271                         }
 272
 273                         // We want to print some activity of Persistence Check at least hourly, even if no activity has occurred, but not litter the log if nothing is happening
 274                         boolean go=false;
 275                         Level level=Level.WARN;
 276                         if(access.willLog(Level.INFO)) {
 277                                 go = true;
 278                                 level=Level.INFO;
 279                         } else if(access.willLog(Level.WARN)) {
 280                                 go = metrics.mremoved>0 || metrics.dremoved>0 || --hourly <= 0;
 281                         }
 282
 283                         if(go) {
 284                                 access.printf(level, "Persist Cache: removed %d of %d items from memory and %d of %d from disk",
 285                                         metrics.mremoved, metrics.mexists, metrics.dremoved, metrics.dexists);
 286                                 hourly = 3600000/CLEAN_CHECK;
 287                         }
 288                 }
 289         }
 290
 291         /* (non-Javadoc)
 292          * @see java.lang.Object#finalize()
 293          */
 294         @Override
 295         protected void finalize() throws Throwable {
 296                 close(); // can call twice.
 297         }
 298
 299
 300
 301 }