auth/auth-core/src/main/java/org/onap/aaf/auth/local/TextIndex.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.local;
  23
  24 import java.io.File;
  25 import java.io.IOException;
  26 import java.io.RandomAccessFile;
  27 import java.nio.ByteBuffer;
  28 import java.nio.IntBuffer;
  29 import java.nio.channels.FileChannel;
  30 import java.util.ArrayList;
  31 import java.util.Collections;
  32 import java.util.LinkedList;
  33 import java.util.List;
  34
  35 import org.onap.aaf.auth.local.DataFile.Token;
  36 import org.onap.aaf.auth.local.DataFile.Token.Field;
  37 import org.onap.aaf.misc.env.Env;
  38 import org.onap.aaf.misc.env.TimeTaken;
  39 import org.onap.aaf.misc.env.Trans;
  40
  41 public class TextIndex {
  42         private static final int REC_SIZE=8;
  43
  44         private File file;
  45         private DataFile dataFile=null;
  46
  47         public TextIndex(File theFile) {
  48                 file = theFile;
  49         }
  50
  51         public void open() throws IOException {
  52                 dataFile = new DataFile(file,"r");
  53                 dataFile.open();
  54         }
  55
  56         public void close() throws IOException {
  57                 if(dataFile!=null) {
  58                         dataFile.close();
  59                         dataFile=null;
  60                 }
  61         }
  62
  63         public int find(Object key, AbsData.Reuse reuse, int offset) throws IOException {
  64                 return find(key,reuse.tokenData,reuse.getFieldData(),offset);
  65         }
  66
  67         public int find(Object key, DataFile.Token dtok, Field df, int offset) throws IOException {
  68                 if(dataFile==null) {
  69                         throw new IOException("File not opened");
  70                 }
  71                 long hash = hashToLong(key.hashCode());
  72                 int min=0, max = (int)(dataFile.size()/REC_SIZE);
  73                 Token ttok = dataFile.new Token(REC_SIZE);
  74                 IntBuffer tib = ttok.getIntBuffer();
  75                 long lhash;
  76                 int curr;
  77                 while((max-min)>100) {
  78                         ttok.pos((curr=(min+(max-min)/2))*REC_SIZE);
  79                         tib.rewind();
  80                         lhash = hashToLong(tib.get());
  81                         if(lhash<hash) {
  82                                 min=curr+1;
  83                         } else if(lhash>hash) {
  84                                 max=curr-1;
  85                         } else {
  86                                 min=curr-40;
  87                                 max=curr+40;
  88                                 break;
  89                         }
  90                 }
  91
  92                 List<Integer> entries = new ArrayList<Integer>();
  93                 for(int i=min;i<=max;++i) {
  94                         ttok.pos(i*REC_SIZE);
  95                         tib.rewind();
  96                         lhash = hashToLong(tib.get());
  97                         if(lhash==hash) {
  98                                 entries.add(tib.get());
  99                         } else if(lhash>hash) {
 100                                 break;
 101                         }
 102                 }
 103
 104                 for(Integer i : entries) {
 105                         dtok.pos(i);
 106                         if(df.at(offset).equals(key)) {
 107                                 return i;
 108                         }
 109                 }
 110                 return -1;
 111         }
 112
 113
 114         /*
 115          * Have to change Bytes into a Long, to avoid the inevitable signs in the Hash
 116          */
 117         private static long hashToLong(int hash) {
 118                 long rv;
 119                 if(hash<0) {
 120                         rv = 0xFFFFFFFFL & hash;
 121                 } else {
 122                         rv = hash;
 123                 }
 124                 return rv;
 125         }
 126
 127         public void create(final Trans trans,final DataFile data, int maxLine, char delim, int fieldOffset, int skipLines) throws IOException {
 128                 RandomAccessFile raf;
 129                 FileChannel fos;
 130
 131                 List<Idx> list = new LinkedList<Idx>(); // Some hashcodes will double... DO NOT make a set
 132                 TimeTaken tt2 = trans.start("Open Files", Env.SUB);
 133                 try {
 134                         raf = new RandomAccessFile(file,"rw");
 135                         raf.setLength(0L);
 136                         fos = raf.getChannel();
 137                 } finally {
 138                         tt2.done();
 139                 }
 140
 141                 try {
 142
 143                         Token t = data.new Token(maxLine);
 144                         Field f = t.new Field(delim);
 145
 146                         int count = 0;
 147                         if(skipLines>0) {
 148                                 trans.info().log("Skipping",skipLines,"line"+(skipLines==1?" in":"s in"),data.file().getName());
 149                         }
 150                         for(int i=0;i<skipLines;++i) {
 151                                 t.nextLine();
 152                         }
 153                         tt2 = trans.start("Read", Env.SUB);
 154                         try {
 155                                 while(t.nextLine()) {
 156                                         list.add(new Idx(f.at(fieldOffset),t.pos()));
 157                                         ++count;
 158                                 }
 159                         } finally {
 160                                 tt2.done();
 161                         }
 162                         trans.checkpoint("    Read " + count + " records");
 163                         tt2 = trans.start("Sort List", Env.SUB);
 164                         Collections.sort(list);
 165                         tt2.done();
 166                         tt2 = trans.start("Write Idx", Env.SUB);
 167                         try {
 168                                 ByteBuffer bb = ByteBuffer.allocate(8*1024);
 169                                 IntBuffer ib = bb.asIntBuffer();
 170                                 for(Idx idx : list) {
 171                                         if(!ib.hasRemaining()) {
 172                                                 fos.write(bb);
 173                                                 ib.clear();
 174                                                 bb.rewind();
 175                                         }
 176                                         ib.put(idx.hash);
 177                                         ib.put(idx.pos);
 178                                 }
 179                                 bb.limit(4*ib.position());
 180                                 fos.write(bb);
 181                         } finally {
 182                                 tt2.done();
 183                         }
 184                 } finally {
 185                         fos.close();
 186                         raf.close();
 187                 }
 188         }
 189
 190         public class Iter {
 191                 private int idx;
 192                 private Token t;
 193                 private long end;
 194                 private IntBuffer ib;
 195
 196
 197                 public Iter() {
 198                         try {
 199                                 idx = 0;
 200                                 end = dataFile.size();
 201                                 t  = dataFile.new Token(REC_SIZE);
 202                                 ib = t.getIntBuffer();
 203
 204                         } catch (IOException e) {
 205                                 end = -1L;
 206                         }
 207                 }
 208
 209                 public int next() {
 210                         t.pos(idx);
 211                         ib.clear();
 212                         ib.get();
 213                         int rec = ib.get();
 214                         idx += REC_SIZE;
 215                         return rec;
 216                 }
 217
 218                 public boolean hasNext() {
 219                         return idx<end;
 220                 }
 221         }
 222
 223         private static class Idx implements Comparable<Idx> {
 224                 public int hash, pos;
 225                 public Idx(Object obj, int pos) {
 226                         hash = obj.hashCode();
 227                         this.pos = pos;
 228                 }
 229
 230                 @Override
 231                 public int compareTo(Idx ib) {
 232                         long a = hashToLong(hash);
 233                         long b = hashToLong(ib.hash);
 234                         return a>b?1:a<b?-1:0;
 235                 }
 236
 237                 /* (non-Javadoc)
 238                  * @see java.lang.Object#equals(java.lang.Object)
 239                  */
 240                 @Override
 241                 public boolean equals(Object o) {
 242                         if(o!=null && o instanceof Idx) {
 243                                 return hash == ((Idx)o).hash;
 244                         }
 245                         return false;
 246                 }
 247
 248                 /* (non-Javadoc)
 249                  * @see java.lang.Object#hashCode()
 250                  */
 251                 @Override
 252                 public int hashCode() {
 253                         return hash;
 254                 }
 255         }
 256 }