authz-core/src/main/java/com/att/authz/local/TextIndex.java

   1 /*******************************************************************************\r
   2  * ============LICENSE_START====================================================\r
   3  * * org.onap.aaf\r
   4  * * ===========================================================================\r
   5  * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.\r
   6  * * ===========================================================================\r
   7  * * Licensed under the Apache License, Version 2.0 (the "License");\r
   8  * * you may not use this file except in compliance with the License.\r
   9  * * You may obtain a copy of the License at\r
  10  * * \r
  11  *  *      http://www.apache.org/licenses/LICENSE-2.0\r
  12  * * \r
  13  *  * Unless required by applicable law or agreed to in writing, software\r
  14  * * distributed under the License is distributed on an "AS IS" BASIS,\r
  15  * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
  16  * * See the License for the specific language governing permissions and\r
  17  * * limitations under the License.\r
  18  * * ============LICENSE_END====================================================\r
  19  * *\r
  20  * * ECOMP is a trademark and service mark of AT&T Intellectual Property.\r
  21  * *\r
  22  ******************************************************************************/\r
  23 package com.att.authz.local;\r
  24 \r
  25 import java.io.File;\r
  26 import java.io.IOException;\r
  27 import java.io.RandomAccessFile;\r
  28 import java.nio.ByteBuffer;\r
  29 import java.nio.IntBuffer;\r
  30 import java.nio.channels.FileChannel;\r
  31 import java.util.ArrayList;\r
  32 import java.util.Collections;\r
  33 import java.util.LinkedList;\r
  34 import java.util.List;\r
  35 \r
  36 import com.att.authz.local.DataFile.Token;\r
  37 import com.att.authz.local.DataFile.Token.Field;\r
  38 import com.att.inno.env.Env;\r
  39 import com.att.inno.env.TimeTaken;\r
  40 import com.att.inno.env.Trans;\r
  41 \r
  42 public class TextIndex {\r
  43         private static final int REC_SIZE=8;\r
  44         \r
  45         private File file;\r
  46         private DataFile dataFile=null;\r
  47         \r
  48         public TextIndex(File theFile) {\r
  49                 file = theFile;\r
  50         }\r
  51         \r
  52         public void open() throws IOException {\r
  53                 dataFile = new DataFile(file,"r");\r
  54                 dataFile.open();\r
  55         }\r
  56         \r
  57         public void close() throws IOException {\r
  58                 if(dataFile!=null) {dataFile.close();}\r
  59         }\r
  60 \r
  61         public int find(Object key, AbsData.Reuse reuse, int offset) throws IOException {\r
  62                 return find(key,reuse.getTokenData(),reuse.getFieldData(),offset);\r
  63         }\r
  64         \r
  65         public int find(Object key, DataFile.Token dtok, Field df, int offset) throws IOException {\r
  66                 if(dataFile==null) {throw new IOException("File not opened");}\r
  67                 long hash = hashToLong(key.hashCode());\r
  68                 int min=0, max = (int)(dataFile.size()/REC_SIZE);\r
  69                 Token ttok = dataFile.new Token(REC_SIZE);\r
  70                 IntBuffer tib = ttok.getIntBuffer();\r
  71                 long lhash;\r
  72                 int curr;\r
  73                 while((max-min)>100) {\r
  74                         ttok.pos((curr=(min+(max-min)/2))*REC_SIZE);\r
  75                         tib.rewind();\r
  76                         lhash = hashToLong(tib.get());\r
  77                         if(lhash<hash) {\r
  78                                 min=curr+1;\r
  79                         } else if(lhash>hash) {\r
  80                                 max=curr-1;\r
  81                         } else {\r
  82                                 min=curr-40;\r
  83                                 max=curr+40;\r
  84                                 break;\r
  85                         }\r
  86                 }\r
  87                 \r
  88                 List<Integer> entries = new ArrayList<Integer>();\r
  89                 for(int i=min;i<=max;++i) {\r
  90                         ttok.pos(i*REC_SIZE);\r
  91                         tib.rewind();\r
  92                         lhash = hashToLong(tib.get());\r
  93                         if(lhash==hash) {\r
  94                                 entries.add(tib.get());\r
  95                         } else if(lhash>hash) {\r
  96                                 break;\r
  97                         }\r
  98                 }\r
  99                 \r
 100                 for(Integer i : entries) {\r
 101                         dtok.pos(i);\r
 102                         if(df.at(offset).equals(key)) {\r
 103                                 return i;\r
 104                         }\r
 105                 }\r
 106                 return -1;\r
 107         }\r
 108         \r
 109 \r
 110         /*\r
 111          * Have to change Bytes into a Long, to avoid the inevitable signs in the Hash\r
 112          */\r
 113         private static long hashToLong(int hash) {\r
 114                 long rv;\r
 115                 if(hash<0) {\r
 116                         rv = 0xFFFFFFFFL & hash;\r
 117                 } else {\r
 118                         rv = hash;\r
 119                 }\r
 120                 return rv;\r
 121         }\r
 122         \r
 123         public void create(final Trans trans,final DataFile data, int maxLine, char delim, int fieldOffset, int skipLines) throws IOException {\r
 124                 RandomAccessFile raf;\r
 125                 FileChannel fos;\r
 126                 \r
 127                 List<Idx> list = new LinkedList<Idx>(); // Some hashcodes will double... DO NOT make a set\r
 128                 TimeTaken tt2 = trans.start("Open Files", Env.SUB);\r
 129                 try {\r
 130                         raf = new RandomAccessFile(file,"rw");\r
 131                         raf.setLength(0L);\r
 132                         fos = raf.getChannel();\r
 133                 } finally {\r
 134                         tt2.done();\r
 135                 }\r
 136                 \r
 137                 try {\r
 138                         \r
 139                         Token t = data.new Token(maxLine);  \r
 140                         Field f = t.new Field(delim);\r
 141                         \r
 142                         int count = 0;\r
 143                         if(skipLines>0) {\r
 144                                 trans.info().log("Skipping",skipLines,"line"+(skipLines==1?" in":"s in"),data.file().getName());\r
 145                         }\r
 146                         for(int i=0;i<skipLines;++i) {\r
 147                                 t.nextLine();\r
 148                         }\r
 149                         tt2 = trans.start("Read", Env.SUB);\r
 150                         try {\r
 151                                 while(t.nextLine()) {\r
 152                                         list.add(new Idx(f.at(fieldOffset),t.pos()));\r
 153                                         ++count;\r
 154                                 }\r
 155                         } finally {\r
 156                                 tt2.done();\r
 157                         }\r
 158                         trans.checkpoint("    Read " + count + " records");\r
 159                         tt2 = trans.start("Sort List", Env.SUB);\r
 160                         Collections.sort(list);\r
 161                         tt2.done();\r
 162                         tt2 = trans.start("Write Idx", Env.SUB);\r
 163                         try {\r
 164                                 ByteBuffer bb = ByteBuffer.allocate(8*1024);\r
 165                                 IntBuffer ib = bb.asIntBuffer();\r
 166                                 for(Idx idx : list) {\r
 167                                         if(!ib.hasRemaining()) {\r
 168                                                 fos.write(bb);\r
 169                                                 ib.clear();\r
 170                                                 bb.rewind();\r
 171                                         }\r
 172                                         ib.put(idx.hash);\r
 173                                         ib.put(idx.pos);\r
 174                                 }\r
 175                                 bb.limit(4*ib.position());\r
 176                                 fos.write(bb);\r
 177                         } finally {\r
 178                                 tt2.done();\r
 179                         }\r
 180                 } finally {\r
 181                         fos.close();\r
 182                         raf.close();\r
 183                 }\r
 184         }\r
 185         \r
 186         public class Iter {\r
 187                 private int idx;\r
 188                 private Token t;\r
 189                 private long end;\r
 190                 private IntBuffer ib;\r
 191 \r
 192 \r
 193                 public Iter() {\r
 194                         try {\r
 195                                 idx = 0;\r
 196                                 end = dataFile.size();\r
 197                                 t  = dataFile.new Token(REC_SIZE);\r
 198                                 ib = t.getIntBuffer();\r
 199 \r
 200                         } catch (IOException e) {\r
 201                                 end = -1L;\r
 202                         }\r
 203                 }\r
 204                 \r
 205                 public int next() {\r
 206                         t.pos(idx);\r
 207                         ib.clear();\r
 208                         ib.get();\r
 209                         int rec = ib.get();\r
 210                         idx += REC_SIZE;\r
 211                         return rec;\r
 212                 }\r
 213 \r
 214                 public boolean hasNext() {\r
 215                         return idx<end;\r
 216                 }\r
 217         }\r
 218         \r
 219         private static class Idx implements Comparable<Idx> {\r
 220                 public int hash, pos;\r
 221                 public Idx(Object obj, int pos) {\r
 222                         hash = obj.hashCode();\r
 223                         this.pos = pos;\r
 224                 }\r
 225                 \r
 226                 @Override\r
 227                 public int compareTo(Idx ib) {\r
 228                         long a = hashToLong(hash);\r
 229                         long b = hashToLong(ib.hash);\r
 230                         return a>b?1:a<b?-1:0;\r
 231                 }\r
 232 \r
 233                 /* (non-Javadoc)\r
 234                  * @see java.lang.Object#equals(java.lang.Object)\r
 235                  */\r
 236                 @Override\r
 237                 public boolean equals(Object o) {\r
 238                         if(o!=null && o instanceof Idx) {\r
 239                                 return hash == ((Idx)o).hash;\r
 240                         }\r
 241                         return false;\r
 242                 }\r
 243 \r
 244                 /* (non-Javadoc)\r
 245                  * @see java.lang.Object#hashCode()\r
 246                  */\r
 247                 @Override\r
 248                 public int hashCode() {\r
 249                         return hash;\r
 250                 }\r
 251         }\r
 252 }\r