auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Upload.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  */
  20
  21 package org.onap.aaf.auth.batch.update;
  22
  23 import java.io.BufferedReader;
  24 import java.io.File;
  25 import java.io.FileReader;
  26 import java.io.IOException;
  27 import java.util.ArrayList;
  28 import java.util.List;
  29
  30 import org.onap.aaf.auth.batch.Batch;
  31 import org.onap.aaf.auth.env.AuthzTrans;
  32 import org.onap.aaf.auth.org.OrganizationException;
  33 import org.onap.aaf.misc.env.APIException;
  34 import org.onap.aaf.misc.env.Env;
  35 import org.onap.aaf.misc.env.TimeTaken;
  36
  37 import com.datastax.driver.core.ResultSet;
  38
  39 public class Upload extends Batch {
  40         public Upload(AuthzTrans trans) throws APIException, IOException, OrganizationException {
  41                 super(trans.env());
  42             trans.info().log("Starting Connection Process");
  43
  44             TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
  45             try {
  46                         TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
  47                         try {
  48                                 session = cluster.connect();
  49                         } finally {
  50                                 tt.done();
  51                         }
  52
  53             } finally {
  54                 tt0.done();
  55             }
  56         }
  57
  58         private static final int BATCH_LENGTH = 100;
  59
  60         int count,batchCnt;
  61
  62         @Override
  63         protected void run(AuthzTrans trans) {
  64                 String line;
  65                 StringBuilder sb = new StringBuilder();
  66                 StringBuilder query = new StringBuilder();
  67                 List<String> array = new ArrayList<String>();
  68                 for(String feed : args()) {
  69                         File file = new File(feed + ".dat");
  70                         TimeTaken tt = trans.start(file.getAbsolutePath(), Env.SUB);
  71                         System.out.println("#### Running " + feed + ".dat Feed ####");
  72                     try {
  73
  74                                 if(file.exists()) {
  75                                         count=batchCnt=0;
  76                                         boolean justOne = false;
  77                                         try {
  78                                                 BufferedReader br = new BufferedReader(new FileReader(file));
  79                                                 try {
  80                                                         while((line=br.readLine())!=null) {
  81                                                                 if(line.length()>5000) {
  82                                                                         if(query.length()>0) {
  83                                                                                 applyBatch(query);
  84                                                                                 justOne=true;
  85                                                                         }
  86                                                                 }
  87                                                                 if(query.length()==0) {
  88                                                                         query.append("BEGIN BATCH\n");
  89                                                                 }
  90                                                                 // Split into fields, first turning Escaped values into something we can convert back from
  91                                                                 char c=0;
  92                                                                 boolean inQuote = false;
  93                                                                 int fldcnt = 0;
  94
  95                                                                 for(int i=0;i<line.length();++i) {
  96                                                                         switch(c=line.charAt(i)) {
  97                                                                                 case '"':
  98                                                                                         inQuote = !inQuote;
  99                                                                                         break;
 100                                                                                 case '|':
 101                                                                                         if(inQuote) {
 102                                                                                                 sb.append(c);
 103                                                                                         } else {
 104                                                                                                 addField(feed,fldcnt++,array,sb);
 105                                                                                         }
 106                                                                                         break;
 107                                                                                 default:
 108                                                                                         sb.append(c);
 109                                                                         }
 110                                                                 }
 111                                                                 addField(feed,fldcnt,array,sb);
 112                                                                 query.append(build(feed, array));
 113
 114                                                                 if((++count % BATCH_LENGTH)==0 || justOne) {
 115                                                                         applyBatch(query);
 116                                                                         justOne=false;
 117                                                                 }
 118                                                         }
 119                                                         if(query.length()>0) {
 120                                                                 applyBatch(query);
 121                                                         }
 122
 123                                                 } finally {
 124                                                         br.close();
 125                                                         sb.setLength(0);
 126                                                         query.setLength(0);
 127                                                 }
 128
 129                                         } catch (IOException e) {
 130                                                 trans.error().log(e);
 131                                                 e.printStackTrace();
 132                                         }
 133
 134                                 } else {
 135                                         trans.error().log("No file found: ", file.getAbsolutePath());
 136                                 }
 137                         } finally {
 138                                 tt.done();
 139                                 System.err.flush();
 140                                 System.out.printf("\n%d applied in %d batches\n",count,batchCnt);
 141                         }
 142
 143                 }
 144
 145         }
 146
 147         // APPROVALS
 148         private static final String APPR_INS_FMT="  INSERT INTO authz.approval "
 149                         + "(id,approver,last_notified,memo,operation,status,ticket,type,user) "
 150                         + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
 151         private static final Boolean[] APPR_QUOTES = new Boolean[]{false,true,true,true,true,true,false,true,true};
 152
 153         // ARTIFACTS
 154         private static final String ARTI_INS_FMT="  INSERT INTO authz.artifact "
 155                         + "(mechid,machine,ca,dir,expires,notify,ns,os_user,renewdays,sans,sponsor,type) "
 156                         + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
 157         private static final Boolean[] ARTI_QUOTES = new Boolean[]
 158                         {true,true,true,true,true,true,true,true,false,false,true,false};
 159
 160         // CREDS
 161         private static final String CRED_INS_FMT="  INSERT INTO authz.cred "
 162                         + "(id,type,expires,cred,notes,ns,other,prev) "
 163                         + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s);\n";
 164         private static final Boolean[] CRED_QUOTES = new Boolean[]
 165                         {true,false,true,false,true,true,false,false};
 166
 167         // NS
 168         private static final String NS_INS_FMT="  INSERT INTO authz.ns "
 169                         + "(name,description,parent,scope,type) "
 170                         + "VALUES (%s,%s,%s,%s,%s);\n";
 171         private static final Boolean[] NS_QUOTES = new Boolean[]
 172                         {true,true,true,false,false};
 173
 174         // x509
 175         private static final String X509_INS_FMT="  INSERT INTO authz.x509 "
 176                         + "(ca,serial,id,x500,x509) "
 177                         + "VALUES (%s,%s,%s,%s,%s);\n";
 178         private static final Boolean[] X509_QUOTES = new Boolean[]
 179                         {true,false,true,true,true};
 180
 181         // ROLE
 182         private static final String ROLE_INS_FMT="  INSERT INTO authz.role "
 183                         + "(ns,name,description,perms) "
 184                         + "VALUES (%s,%s,%s,%s);\n";
 185         private static final Boolean[] ROLE_QUOTES = new Boolean[]
 186                         {true,true,true,false};
 187         // ROLE
 188         private static final String PERM_INS_FMT="  INSERT INTO authz.perm "
 189                         + "(ns,type,instance,action,description,roles) "
 190                         + "VALUES (%s,%s,%s,%s,%s,%s);\n";
 191         private static final Boolean[] PERM_QUOTES = new Boolean[]
 192                         {true,true,true,true,true,false};
 193
 194
 195         private String build(String feed, List<String> array) {
 196                 String rv;
 197                 switch(feed) {
 198                         case "approval":
 199                                 rv = String.format(APPR_INS_FMT,array.toArray());
 200                                 break;
 201                         case "artifact":
 202                                 rv = String.format(ARTI_INS_FMT,array.toArray());
 203                                 break;
 204                         case "cred":
 205                                 rv = String.format(CRED_INS_FMT,array.toArray());
 206                                 break;
 207                         case "ns":
 208                                 rv = String.format(NS_INS_FMT,array.toArray());
 209                                 break;
 210                         case "role":
 211                                 rv = String.format(ROLE_INS_FMT,array.toArray());
 212                                 break;
 213                         case "perm":
 214                                 rv = String.format(PERM_INS_FMT,array.toArray());
 215                                 break;
 216                         case "x509":
 217                                 rv = String.format(X509_INS_FMT,array.toArray());
 218                                 break;
 219                         default:
 220                                 rv = "";
 221                 }
 222                 array.clear();
 223                 return rv;
 224         }
 225
 226         private void addField(String feed, int fldcnt, List<String> array, StringBuilder sb) {
 227                 Boolean[] ba;
 228                 switch(feed) {
 229                         case "approval":
 230                                 ba = APPR_QUOTES;
 231                                 break;
 232                         case "artifact":
 233                                 ba = ARTI_QUOTES;
 234                                 break;
 235                         case "cred":
 236                                 ba = CRED_QUOTES;
 237                                 break;
 238                         case "ns":
 239                                 ba = NS_QUOTES;
 240                                 break;
 241                         case "role":
 242                                 ba = ROLE_QUOTES;
 243                                 break;
 244                         case "perm":
 245                                 ba = PERM_QUOTES;
 246                                 break;
 247                         case "x509":
 248                                 ba = X509_QUOTES;
 249                                 break;
 250                         default:
 251                                 ba = null;
 252                 }
 253                 if(ba!=null) {
 254                         if(sb.toString().length()==0) {
 255                                 array.add("null");
 256                         } else {
 257                                 if(ba[fldcnt]) {
 258                                         String s = null;
 259                                         if(sb.indexOf("'")>=0) {
 260                                                 s = sb.toString().replace("'","''");
 261                                         }
 262                                         if(sb.indexOf("\\n")>=0) {
 263                                                 if(s==null) {
 264                                                         s = sb.toString().replace("\\n","\n");
 265                                                 } else {
 266                                                         s = s.replace("\\n","\n");
 267                                                 }
 268                                         }
 269                                         if(sb.indexOf("\\t")>=0) {
 270                                                 if(s==null) {
 271                                                         s = sb.toString().replace("\\t","\t");
 272                                                 } else {
 273                                                         s = s.replace("\\t","\t");
 274                                                 }
 275                                         }
 276                                         if(s==null) {
 277                                                 array.add("'" + sb + '\'');
 278                                         } else {
 279                                                 array.add("'" + s + '\'');
 280                                         }
 281                                 } else {
 282                                         array.add(sb.toString());
 283                                 }
 284                         }
 285                         sb.setLength(0);
 286                 }
 287         }
 288
 289         private void applyBatch(StringBuilder query) {
 290                 try {
 291                         query.append("APPLY BATCH;");
 292                         ResultSet rv = session.execute(query.toString());
 293                         if(rv.wasApplied()) {
 294                                 System.out.print('.');
 295                                 if((++batchCnt % 60)==0) {
 296                                         System.out.println();
 297                                 }
 298                         } else {
 299                                 System.out.print("Data NOT APPLIED");
 300                         }
 301                 } finally {
 302                         query.setLength(0);
 303                 }
 304         }
 305
 306
 307         @Override
 308         protected void _close(AuthzTrans trans) {
 309         session.close();
 310         }
 311
 312 }
 313