auth/auth-core/src/main/java/org/onap/aaf/auth/local/TextIndex.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.local;
  23
  24 import java.io.File;
  25 import java.io.IOException;
  26 import java.io.RandomAccessFile;
  27 import java.nio.ByteBuffer;
  28 import java.nio.IntBuffer;
  29 import java.nio.channels.FileChannel;
  30 import java.util.ArrayList;
  31 import java.util.Collections;
  32 import java.util.LinkedList;
  33 import java.util.List;
  34
  35 import org.onap.aaf.auth.local.DataFile.Token;
  36 import org.onap.aaf.auth.local.DataFile.Token.Field;
  37 import org.onap.aaf.misc.env.Env;
  38 import org.onap.aaf.misc.env.TimeTaken;
  39 import org.onap.aaf.misc.env.Trans;
  40
  41 public class TextIndex {
  42         private static final int REC_SIZE=8;
  43
  44         private File file;
  45         private DataFile dataFile=null;
  46
  47         public TextIndex(File theFile) {
  48                 file = theFile;
  49         }
  50
  51         public void open() throws IOException {
  52                 dataFile = new DataFile(file,"r");
  53                 dataFile.open();
  54         }
  55
  56         public void close() throws IOException {
  57                 if(dataFile!=null) {
  58                         dataFile.close();
  59                         dataFile=null;
  60                 }
  61         }
  62
  63         public int find(Object key, AbsData.Reuse reuse, int offset) throws IOException {
  64                 return find(key,reuse.tokenData,reuse.getFieldData(),offset);
  65         }
  66
  67         public int find(Object key, DataFile.Token dtok, Field df, int offset) throws IOException {
  68                 if(dataFile==null) {
  69                         throw new IOException("File not opened");
  70                 }
  71                 long hash = hashToLong(key.hashCode());
  72                 int min=0, max = (int)(dataFile.size()/REC_SIZE);
  73                 Token ttok = dataFile.new Token(REC_SIZE);
  74                 IntBuffer tib = ttok.getIntBuffer();
  75                 long lhash;
  76                 int curr;
  77                 while((max-min)>100) {
  78                         ttok.pos((curr=(min+(max-min)/2))*REC_SIZE);
  79                         tib.rewind();
  80                         lhash = hashToLong(tib.get());
  81                         if(lhash<hash) {
  82                                 min=curr+1;
  83                         } else if(lhash>hash) {
  84                                 max=curr-1;
  85                         } else {
  86                                 min=curr-40;
  87                                 max=curr+40;
  88                                 break;
  89                         }
  90                 }
  91
  92                 List<Integer> entries = new ArrayList<>();
  93                 for(int i=min;i<=max;++i) {
  94                         ttok.pos(i*REC_SIZE);
  95                         tib.rewind();
  96                         lhash = hashToLong(tib.get());
  97                         if(lhash==hash) {
  98                                 entries.add(tib.get());
  99                         } else if(lhash>hash) {
 100                                 break;
 101                         }
 102                 }
 103
 104                 for(Integer i : entries) {
 105                         dtok.pos(i);
 106                         if(df.at(offset).equals(key)) {
 107                                 return i;
 108                         }
 109                 }
 110                 return -1;
 111         }
 112
 113
 114         /*
 115          * Have to change Bytes into a Long, to avoid the inevitable signs in the Hash
 116          */
 117         private static long hashToLong(int hash) {
 118                 long rv;
 119                 if(hash<0) {
 120                         rv = 0xFFFFFFFFL & hash;
 121                 } else {
 122                         rv = hash;
 123                 }
 124                 return rv;
 125         }
 126
 127         public void create(final Trans trans,final DataFile data, int maxLine, char delim, int fieldOffset, int skipLines) throws IOException {
 128                 FileChannel fos;
 129
 130                 List<Idx> list = new LinkedList<>(); // Some hashcodes will double... DO NOT make a set
 131                 TimeTaken tt2 = trans.start("Open Files", Env.SUB);
 132                 RandomAccessFile raf=null;
 133                 try {
 134                         try {
 135                                 raf = new RandomAccessFile(file,"rw");
 136                                 raf.setLength(0L);
 137                                 fos = raf.getChannel();
 138                         } finally {
 139                                 tt2.done();
 140                         }
 141
 142                         try {
 143
 144                                 Token t = data.new Token(maxLine);
 145                                 Field f = t.new Field(delim);
 146
 147                                 int count = 0;
 148                                 if(skipLines>0) {
 149                                         trans.info().log("Skipping",skipLines,"line"+(skipLines==1?" in":"s in"),data.file().getName());
 150                                 }
 151                                 for(int i=0;i<skipLines;++i) {
 152                                         t.nextLine();
 153                                 }
 154                                 tt2 = trans.start("Read", Env.SUB);
 155                                 try {
 156                                         while(t.nextLine()) {
 157                                                 list.add(new Idx(f.at(fieldOffset),t.pos()));
 158                                                 ++count;
 159                                         }
 160                                 } finally {
 161                                         tt2.done();
 162                                 }
 163                                 trans.checkpoint("    Read " + count + " records");
 164                                 tt2 = trans.start("Sort List", Env.SUB);
 165                                 Collections.sort(list);
 166                                 tt2.done();
 167                                 tt2 = trans.start("Write Idx", Env.SUB);
 168                                 try {
 169                                         ByteBuffer bb = ByteBuffer.allocate(8*1024);
 170                                         IntBuffer ib = bb.asIntBuffer();
 171                                         for(Idx idx : list) {
 172                                                 if(!ib.hasRemaining()) {
 173                                                         fos.write(bb);
 174                                                         ib.clear();
 175                                                         bb.rewind();
 176                                                 }
 177                                                 ib.put(idx.hash);
 178                                                 ib.put(idx.pos);
 179                                         }
 180                                         bb.limit(4*ib.position());
 181                                         fos.write(bb);
 182                                 } finally {
 183                                         tt2.done();
 184                                 }
 185                         } finally {
 186                                 fos.close();
 187                         }
 188                 } finally {
 189                         if(raf!=null) {
 190                                 raf.close(); // closed by fos
 191                         }
 192                 }
 193         }
 194
 195         public class Iter {
 196                 private int idx;
 197                 private Token t;
 198                 private long end;
 199                 private IntBuffer ib;
 200
 201
 202                 public Iter() {
 203                         try {
 204                                 idx = 0;
 205                                 end = dataFile.size();
 206                                 t  = dataFile.new Token(REC_SIZE);
 207                                 ib = t.getIntBuffer();
 208
 209                         } catch (IOException e) {
 210                                 end = -1L;
 211                         }
 212                 }
 213
 214                 public int next() {
 215                         t.pos(idx);
 216                         ib.clear();
 217                         ib.get();
 218                         int rec = ib.get();
 219                         idx += REC_SIZE;
 220                         return rec;
 221                 }
 222
 223                 public boolean hasNext() {
 224                         return idx<end;
 225                 }
 226         }
 227
 228         private static class Idx implements Comparable<Idx> {
 229                 public int hash, pos;
 230                 public Idx(Object obj, int pos) {
 231                         hash = obj.hashCode();
 232                         this.pos = pos;
 233                 }
 234
 235                 @Override
 236                 public int compareTo(Idx ib) {
 237                         long a = hashToLong(hash);
 238                         long b = hashToLong(ib.hash);
 239                         return a>b?1:a<b?-1:0;
 240                 }
 241
 242                 /* (non-Javadoc)
 243                  * @see java.lang.Object#equals(java.lang.Object)
 244                  */
 245                 @Override
 246                 public boolean equals(Object o) {
 247                         if(o!=null && o instanceof Idx) {
 248                                 return hash == ((Idx)o).hash;
 249                         }
 250                         return false;
 251                 }
 252
 253                 /* (non-Javadoc)
 254                  * @see java.lang.Object#hashCode()
 255                  */
 256                 @Override
 257                 public int hashCode() {
 258                         return hash;
 259                 }
 260         }
 261 }