authz-core/src/main/java/com/att/authz/local/TextIndex.java

   1 /*******************************************************************************\r
   2  * ============LICENSE_START====================================================\r
   3  * * org.onap.aai\r
   4  * * ===========================================================================\r
   5  * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.\r
   6  * * Copyright © 2017 Amdocs\r
   7  * * ===========================================================================\r
   8  * * Licensed under the Apache License, Version 2.0 (the "License");\r
   9  * * you may not use this file except in compliance with the License.\r
  10  * * You may obtain a copy of the License at\r
  11  * * \r
  12  *  *      http://www.apache.org/licenses/LICENSE-2.0\r
  13  * * \r
  14  *  * Unless required by applicable law or agreed to in writing, software\r
  15  * * distributed under the License is distributed on an "AS IS" BASIS,\r
  16  * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r
  17  * * See the License for the specific language governing permissions and\r
  18  * * limitations under the License.\r
  19  * * ============LICENSE_END====================================================\r
  20  * *\r
  21  * * ECOMP is a trademark and service mark of AT&T Intellectual Property.\r
  22  * *\r
  23  ******************************************************************************/\r
  24 package com.att.authz.local;\r
  25 \r
  26 import java.io.File;\r
  27 import java.io.IOException;\r
  28 import java.io.RandomAccessFile;\r
  29 import java.nio.ByteBuffer;\r
  30 import java.nio.IntBuffer;\r
  31 import java.nio.channels.FileChannel;\r
  32 import java.util.ArrayList;\r
  33 import java.util.Collections;\r
  34 import java.util.LinkedList;\r
  35 import java.util.List;\r
  36 \r
  37 import com.att.authz.local.DataFile.Token;\r
  38 import com.att.authz.local.DataFile.Token.Field;\r
  39 import com.att.inno.env.Env;\r
  40 import com.att.inno.env.TimeTaken;\r
  41 import com.att.inno.env.Trans;\r
  42 \r
  43 public class TextIndex {\r
  44         private static final int REC_SIZE=8;\r
  45         \r
  46         private File file;\r
  47         private DataFile dataFile=null;\r
  48         \r
  49         public TextIndex(File theFile) {\r
  50                 file = theFile;\r
  51         }\r
  52         \r
  53         public void open() throws IOException {\r
  54                 dataFile = new DataFile(file,"r");\r
  55                 dataFile.open();\r
  56         }\r
  57         \r
  58         public void close() throws IOException {\r
  59                 if(dataFile!=null) {dataFile.close();}\r
  60         }\r
  61 \r
  62         public int find(Object key, AbsData.Reuse reuse, int offset) throws IOException {\r
  63                 return find(key,reuse.getTokenData(),reuse.getFieldData(),offset);\r
  64         }\r
  65         \r
  66         public int find(Object key, DataFile.Token dtok, Field df, int offset) throws IOException {\r
  67                 if(dataFile==null) {throw new IOException("File not opened");}\r
  68                 long hash = hashToLong(key.hashCode());\r
  69                 int min=0, max = (int)(dataFile.size()/REC_SIZE);\r
  70                 Token ttok = dataFile.new Token(REC_SIZE);\r
  71                 IntBuffer tib = ttok.getIntBuffer();\r
  72                 long lhash;\r
  73                 int curr;\r
  74                 while((max-min)>100) {\r
  75                         ttok.pos((curr=(min+(max-min)/2))*REC_SIZE);\r
  76                         tib.rewind();\r
  77                         lhash = hashToLong(tib.get());\r
  78                         if(lhash<hash) {\r
  79                                 min=curr+1;\r
  80                         } else if(lhash>hash) {\r
  81                                 max=curr-1;\r
  82                         } else {\r
  83                                 min=curr-40;\r
  84                                 max=curr+40;\r
  85                                 break;\r
  86                         }\r
  87                 }\r
  88                 \r
  89                 List<Integer> entries = new ArrayList<Integer>();\r
  90                 for(int i=min;i<=max;++i) {\r
  91                         ttok.pos(i*REC_SIZE);\r
  92                         tib.rewind();\r
  93                         lhash = hashToLong(tib.get());\r
  94                         if(lhash==hash) {\r
  95                                 entries.add(tib.get());\r
  96                         } else if(lhash>hash) {\r
  97                                 break;\r
  98                         }\r
  99                 }\r
 100                 \r
 101                 for(Integer i : entries) {\r
 102                         dtok.pos(i);\r
 103                         if(df.at(offset).equals(key)) {\r
 104                                 return i;\r
 105                         }\r
 106                 }\r
 107                 return -1;\r
 108         }\r
 109         \r
 110 \r
 111         /*\r
 112          * Have to change Bytes into a Long, to avoid the inevitable signs in the Hash\r
 113          */\r
 114         private static long hashToLong(int hash) {\r
 115                 long rv;\r
 116                 if(hash<0) {\r
 117                         rv = 0xFFFFFFFFL & hash;\r
 118                 } else {\r
 119                         rv = hash;\r
 120                 }\r
 121                 return rv;\r
 122         }\r
 123         \r
 124         public void create(final Trans trans,final DataFile data, int maxLine, char delim, int fieldOffset, int skipLines) throws IOException {\r
 125                 RandomAccessFile raf;\r
 126                 FileChannel fos;\r
 127                 \r
 128                 List<Idx> list = new LinkedList<Idx>(); // Some hashcodes will double... DO NOT make a set\r
 129                 TimeTaken tt2 = trans.start("Open Files", Env.SUB);\r
 130                 try {\r
 131                         raf = new RandomAccessFile(file,"rw");\r
 132                         raf.setLength(0L);\r
 133                         fos = raf.getChannel();\r
 134                 } finally {\r
 135                         tt2.done();\r
 136                 }\r
 137                 \r
 138                 try {\r
 139                         \r
 140                         Token t = data.new Token(maxLine);  \r
 141                         Field f = t.new Field(delim);\r
 142                         \r
 143                         int count = 0;\r
 144                         if(skipLines>0) {\r
 145                                 trans.info().log("Skipping",skipLines,"line"+(skipLines==1?" in":"s in"),data.file().getName());\r
 146                         }\r
 147                         for(int i=0;i<skipLines;++i) {\r
 148                                 t.nextLine();\r
 149                         }\r
 150                         tt2 = trans.start("Read", Env.SUB);\r
 151                         try {\r
 152                                 while(t.nextLine()) {\r
 153                                         list.add(new Idx(f.at(fieldOffset),t.pos()));\r
 154                                         ++count;\r
 155                                 }\r
 156                         } finally {\r
 157                                 tt2.done();\r
 158                         }\r
 159                         trans.checkpoint("    Read " + count + " records");\r
 160                         tt2 = trans.start("Sort List", Env.SUB);\r
 161                         Collections.sort(list);\r
 162                         tt2.done();\r
 163                         tt2 = trans.start("Write Idx", Env.SUB);\r
 164                         try {\r
 165                                 ByteBuffer bb = ByteBuffer.allocate(8*1024);\r
 166                                 IntBuffer ib = bb.asIntBuffer();\r
 167                                 for(Idx idx : list) {\r
 168                                         if(!ib.hasRemaining()) {\r
 169                                                 fos.write(bb);\r
 170                                                 ib.clear();\r
 171                                                 bb.rewind();\r
 172                                         }\r
 173                                         ib.put(idx.hash);\r
 174                                         ib.put(idx.pos);\r
 175                                 }\r
 176                                 bb.limit(4*ib.position());\r
 177                                 fos.write(bb);\r
 178                         } finally {\r
 179                                 tt2.done();\r
 180                         }\r
 181                 } finally {\r
 182                         fos.close();\r
 183                         raf.close();\r
 184                 }\r
 185         }\r
 186         \r
 187         public class Iter {\r
 188                 private int idx;\r
 189                 private Token t;\r
 190                 private long end;\r
 191                 private IntBuffer ib;\r
 192 \r
 193 \r
 194                 public Iter() {\r
 195                         try {\r
 196                                 idx = 0;\r
 197                                 end = dataFile.size();\r
 198                                 t  = dataFile.new Token(REC_SIZE);\r
 199                                 ib = t.getIntBuffer();\r
 200 \r
 201                         } catch (IOException e) {\r
 202                                 end = -1L;\r
 203                         }\r
 204                 }\r
 205                 \r
 206                 public int next() {\r
 207                         t.pos(idx);\r
 208                         ib.clear();\r
 209                         ib.get();\r
 210                         int rec = ib.get();\r
 211                         idx += REC_SIZE;\r
 212                         return rec;\r
 213                 }\r
 214 \r
 215                 public boolean hasNext() {\r
 216                         return idx<end;\r
 217                 }\r
 218         }\r
 219         \r
 220         private static class Idx implements Comparable<Idx> {\r
 221                 public int hash, pos;\r
 222                 public Idx(Object obj, int pos) {\r
 223                         hash = obj.hashCode();\r
 224                         this.pos = pos;\r
 225                 }\r
 226                 \r
 227                 @Override\r
 228                 public int compareTo(Idx ib) {\r
 229                         long a = hashToLong(hash);\r
 230                         long b = hashToLong(ib.hash);\r
 231                         return a>b?1:a<b?-1:0;\r
 232                 }\r
 233 \r
 234                 /* (non-Javadoc)\r
 235                  * @see java.lang.Object#equals(java.lang.Object)\r
 236                  */\r
 237                 @Override\r
 238                 public boolean equals(Object o) {\r
 239                         if(o!=null && o instanceof Idx) {\r
 240                                 return hash == ((Idx)o).hash;\r
 241                         }\r
 242                         return false;\r
 243                 }\r
 244 \r
 245                 /* (non-Javadoc)\r
 246                  * @see java.lang.Object#hashCode()\r
 247                  */\r
 248                 @Override\r
 249                 public int hashCode() {\r
 250                         return hash;\r
 251                 }\r
 252         }\r
 253 }\r