2 * ============LICENSE_START====================================================
4 * ===========================================================================
5 * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
6 * ===========================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END====================================================
22 package org.onap.aaf.cadi.persist;
24 import java.io.IOException;
25 import java.nio.file.FileVisitResult;
26 import java.nio.file.FileVisitor;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.nio.file.attribute.BasicFileAttributes;
30 import java.util.Date;
32 import java.util.Map.Entry;
33 import java.util.Queue;
34 import java.util.Timer;
35 import java.util.TimerTask;
36 import java.util.concurrent.ConcurrentHashMap;
37 import java.util.concurrent.ConcurrentLinkedQueue;
39 import org.onap.aaf.cadi.Access;
40 import org.onap.aaf.cadi.CadiException;
41 import org.onap.aaf.cadi.LocatorException;
42 import org.onap.aaf.cadi.Access.Level;
43 import org.onap.aaf.cadi.util.Holder;
44 import org.onap.aaf.cadi.client.Result;
45 import org.onap.aaf.misc.env.APIException;
46 import org.onap.aaf.misc.env.util.Chrono;
47 import org.onap.aaf.misc.rosetta.env.RosettaDF;
48 import org.onap.aaf.misc.rosetta.env.RosettaEnv;
50 public abstract class Persist<T,CT extends Persistable<T>> extends PersistFile {
51 private static final long ONE_DAY = 86400000L;
52 private static final long CLEAN_CHECK = 2*60*1000L; // check every 2 mins
53 private static Timer clean;
55 // store all the directories to review
56 // No Concurrent HashSet, or at least, it is all implemented with HashMap in older versions
57 private static Queue<Persist<?,?>> allPersists = new ConcurrentLinkedQueue<Persist<?,?>>();
59 private Map<String,CT> tmap;
60 protected RosettaEnv env;
61 private RosettaDF<T> df;
64 public Persist(Access access, RosettaEnv env, Class<T> cls, String sub_dir) throws CadiException, APIException {
65 super(access, sub_dir);
67 df = env.newDataFactory(cls);
68 tmap = new ConcurrentHashMap<>();
69 synchronized(Persist.class) {
71 clean = new Timer(true);
72 clean.schedule(new Clean(access), 20000, CLEAN_CHECK);
75 allPersists.add(this);
79 allPersists.remove(this);
82 protected abstract CT newCacheable(T t, long expires_secsFrom1970, byte[] hash, Path path) throws APIException, IOException;
84 public RosettaDF<T> getDF() {
87 public Result<CT> get(final String key, final byte[] hash, Loader<CT> rl) throws CadiException, APIException, LocatorException {
91 Holder<Path> hp = new Holder<Path>(null);
92 CT ct = tmap.get(key);
93 // Make sure cached Item is synced with Disk, but only even Minute to save Disk hits
94 if (ct!=null && ct.checkSyncTime()) { // check File Time only every SYNC Period (2 min)
95 if (ct.hasBeenTouched()) {
98 access.log(Level.DEBUG,"File for",key,"has been touched, removing memory entry");
102 // If not currently in memory, check with Disk (which might have been updated by other processes)
104 Holder<Long> hl = new Holder<Long>(0L);
106 if ((t = readDisk(df, hash, key, hp, hl))!=null) {
108 if ((ct = newCacheable(t,hl.get(),hash,hp.get()))!=null) {
111 access.log(Level.DEBUG,"Read Token from",key);
112 } catch (IOException e) {
113 access.log(e,"Reading Token from",key);
115 } // if not read, then ct still==null
117 // If not in memory, or on disk, get from Remote... IF reloadable (meaning, isn't hitting too often, etc).
118 if (ct==null || ct.checkReloadable()) {
119 // Load from external (if makes sense)
120 Result<CT> rtp = rl.load(key);
124 Path p = getPath(key);
125 writeDisk(df, ct.get(),ct.getHash(),p,ct.expires());
126 access.log(Level.DEBUG, "Writing token",key);
127 } catch (CadiException e) {
129 } catch (Exception e) {
130 throw new CadiException(e);
133 return Result.err(rtp);
141 access.log(Level.DEBUG,"Found token in memory",key);
143 // ct can only be not-null here
145 return Result.ok(200,ct);
148 public void put(String key, CT ct) throws CadiException {
149 writeDisk(df, ct.get(), ct.getHash(), key, ct.expires());
153 public void delete(String key) {
158 public interface Loader<CT> {
159 Result<CT> load(String key) throws APIException, CadiException, LocatorException;
163 * Clean will examine resources, and remove those that have expired.
165 * If "highs" have been exceeded, then we'll expire 10% more the next time. This will adjust after each run
166 * without checking contents more than once, making a good average "high" in the minimum speed.
171 private static final class Clean extends TimerTask {
172 private final Access access;
175 public Clean(Access access) {
176 this.access = access;
180 private static class Metrics {
181 public int mexists = 0, dexists=0;
182 public int mremoved = 0, dremoved=0;
186 final long now = System.currentTimeMillis();
187 final long dayFromNow = now + ONE_DAY;
188 final Metrics metrics = new Metrics();
189 for (final Persist<?,?> persist : allPersists) {
191 if (access.willLog(Level.DEBUG)) {
192 access.log(Level.DEBUG, "Persist: Cleaning memory cache for",persist.tokenPath.toAbsolutePath());
194 for (Entry<String, ?> es : persist.tmap.entrySet()) {
196 Persistable<?> p = (Persistable<?>)es.getValue();
197 if (p.checkSyncTime()) {
200 persist.tmap.remove(es.getKey());
201 access.printf(Level.DEBUG, "Persist: removed cached item %s from memory\n", es.getKey());
205 } else if (Files.exists(p.path())) {
211 final StringBuilder sb = new StringBuilder();
212 Files.walkFileTree(persist.tokenPath, new FileVisitor<Path>() {
214 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
216 sb.append("Persist: Cleaning files from ");
217 sb.append(dir.toAbsolutePath());
218 return FileVisitResult.CONTINUE;
222 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
223 if (attrs.isRegularFile()) {
227 long exp = persist.readExpiration(file)*1000; // readExpiration is seconds from 1970
228 if (now > exp) { // cover for bad token
229 sb.append("\n\tFile ");
230 sb.append(file.getFileName());
231 sb.append(" expired ");
232 sb.append(Chrono.dateTime(new Date(exp)));
233 persist.deleteFromDisk(file);
235 } else if (exp > dayFromNow) {
236 sb.append("\n\tFile ");
237 sb.append(file.toString());
238 sb.append(" data corrupted.");
239 persist.deleteFromDisk(file);
242 } catch (CadiException e) {
243 sb.append("\n\tError reading File ");
244 sb.append(file.toString());
246 sb.append(e.getMessage());
251 return FileVisitResult.CONTINUE;
255 public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
256 access.log(Level.ERROR,"Error visiting file %s (%s)\n",file.toString(),exc.getMessage());
257 return FileVisitResult.CONTINUE;
261 public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
262 access.log(Level.DEBUG, sb);
263 return FileVisitResult.CONTINUE;
267 } catch (IOException e) {
268 access.log(e, "Exception while cleaning Persistance");
273 // We want to print some activity of Persistence Check at least hourly, even if no activity has occurred, but not litter the log if nothing is happening
275 Level level=Level.WARN;
276 if (access.willLog(Level.INFO)) {
279 } else if (access.willLog(Level.WARN)) {
280 go = metrics.mremoved>0 || metrics.dremoved>0 || --hourly <= 0;
284 access.printf(level, "Persist Cache: removed %d of %d items from memory and %d of %d from disk",
285 metrics.mremoved, metrics.mexists, metrics.dremoved, metrics.dexists);
286 hourly = 3600000/CLEAN_CHECK;
292 * @see java.lang.Object#finalize()
295 protected void finalize() throws Throwable {
296 close(); // can call twice.