auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Upload.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  *
   7  * Modifications Copyright (C) 2018 IBM.
   8  * ===========================================================================
   9  * Licensed under the Apache License, Version 2.0 (the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  *      http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  * ============LICENSE_END====================================================
  21  */
  22
  23 package org.onap.aaf.auth.batch.update;
  24
  25 import java.io.BufferedReader;
  26 import java.io.File;
  27 import java.io.FileReader;
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.List;
  31
  32 import org.onap.aaf.auth.batch.Batch;
  33 import org.onap.aaf.auth.env.AuthzTrans;
  34 import org.onap.aaf.auth.org.OrganizationException;
  35 import org.onap.aaf.misc.env.APIException;
  36 import org.onap.aaf.misc.env.Env;
  37 import org.onap.aaf.misc.env.TimeTaken;
  38
  39 import com.datastax.driver.core.ResultSet;
  40
  41 public class Upload extends Batch {
  42
  43         private static final int BATCH_LENGTH = 100;
  44
  45         int count;
  46         int batchCnt;
  47
  48         // APPROVALS
  49         private static final String APPR_INS_FMT="  INSERT INTO authz.approval "
  50                         + "(id,approver,last_notified,memo,operation,status,ticket,type,user) "
  51                         + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
  52         private static final Boolean[] APPR_QUOTES = new Boolean[]{false,true,true,true,true,true,false,true,true};
  53
  54         // ARTIFACTS
  55         private static final String ARTI_INS_FMT="  INSERT INTO authz.artifact "
  56                         + "(mechid,machine,ca,dir,expires,notify,ns,os_user,renewdays,sans,sponsor,type) "
  57                         + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);\n";
  58         private static final Boolean[] ARTI_QUOTES = new Boolean[]
  59                         {true,true,true,true,true,true,true,true,false,false,true,false};
  60
  61         // CREDS
  62         private static final String CRED_INS_FMT="  INSERT INTO authz.cred "
  63                         + "(id,type,expires,cred,notes,ns,other,prev) "
  64                         + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s);\n";
  65         private static final Boolean[] CRED_QUOTES = new Boolean[]
  66                         {true,false,true,false,true,true,false,false};
  67
  68         // NS
  69         private static final String NS_INS_FMT="  INSERT INTO authz.ns "
  70                         + "(name,description,parent,scope,type) "
  71                         + "VALUES (%s,%s,%s,%s,%s);\n";
  72         private static final Boolean[] NS_QUOTES = new Boolean[]
  73                         {true,true,true,false,false};
  74
  75         // x509
  76         private static final String X509_INS_FMT="  INSERT INTO authz.x509 "
  77                         + "(ca,serial,id,x500,x509) "
  78                         + "VALUES (%s,%s,%s,%s,%s);\n";
  79         private static final Boolean[] X509_QUOTES = new Boolean[]
  80                         {true,false,true,true,true};
  81
  82         // ROLE
  83         private static final String ROLE_INS_FMT="  INSERT INTO authz.role "
  84                         + "(ns,name,description,perms) "
  85                         + "VALUES (%s,%s,%s,%s);\n";
  86         private static final Boolean[] ROLE_QUOTES = new Boolean[]
  87                         {true,true,true,false};
  88         // ROLE
  89         private static final String PERM_INS_FMT="  INSERT INTO authz.perm "
  90                         + "(ns,type,instance,action,description,roles) "
  91                         + "VALUES (%s,%s,%s,%s,%s,%s);\n";
  92         private static final Boolean[] PERM_QUOTES = new Boolean[]
  93                         {true,true,true,true,true,false};
  94
  95         public Upload(AuthzTrans trans) throws APIException, IOException, OrganizationException {
  96                 super(trans.env());
  97             trans.info().log("Starting Connection Process");
  98
  99             TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
 100             try {
 101                         TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
 102                         try {
 103                                 session = cluster.connect();
 104                         } finally {
 105                                 tt.done();
 106                         }
 107
 108             } finally {
 109                 tt0.done();
 110             }
 111         }
 112
 113         @Override
 114         protected void run(AuthzTrans trans) {
 115                 String line;
 116                 StringBuilder sb = new StringBuilder();
 117                 StringBuilder query = new StringBuilder();
 118                 List<String> array = new ArrayList<String>();
 119                 for(String feed : args()) {
 120                         File file = new File(feed + ".dat");
 121                         TimeTaken tt = trans.start(file.getAbsolutePath(), Env.SUB);
 122                         System.out.println("#### Running " + feed + ".dat Feed ####");
 123                     try {
 124
 125                                 if(file.exists()) {
 126                                         count=batchCnt=0;
 127                                         boolean justOne = false;
 128                                         try {
 129                                                 BufferedReader br = new BufferedReader(new FileReader(file));
 130                                                 try {
 131                                                         while((line=br.readLine())!=null) {
 132                                                                 if(line.length()>5000) {
 133                                                                         if(query.length()>0) {
 134                                                                                 applyBatch(query);
 135                                                                                 justOne=true;
 136                                                                         }
 137                                                                 }
 138                                                                 if(query.length()==0) {
 139                                                                         query.append("BEGIN BATCH\n");
 140                                                                 }
 141                                                                 // Split into fields, first turning Escaped values into something we can convert back from
 142                                                                 char c=0;
 143                                                                 boolean inQuote = false;
 144                                                                 int fldcnt = 0;
 145
 146                                                                 for(int i=0;i<line.length();++i) {
 147                                                                         switch(c=line.charAt(i)) {
 148                                                                                 case '"':
 149                                                                                         inQuote = !inQuote;
 150                                                                                         break;
 151                                                                                 case '|':
 152                                                                                         if(inQuote) {
 153                                                                                                 sb.append(c);
 154                                                                                         } else {
 155                                                                                                 addField(feed,fldcnt++,array,sb);
 156                                                                                         }
 157                                                                                         break;
 158                                                                                 default:
 159                                                                                         sb.append(c);
 160                                                                         }
 161                                                                 }
 162                                                                 addField(feed,fldcnt,array,sb);
 163                                                                 query.append(build(feed, array));
 164
 165                                                                 if((++count % BATCH_LENGTH)==0 || justOne) {
 166                                                                         applyBatch(query);
 167                                                                         justOne=false;
 168                                                                 }
 169                                                         }
 170                                                         if(query.length()>0) {
 171                                                                 applyBatch(query);
 172                                                         }
 173
 174                                                 } finally {
 175                                                         br.close();
 176                                                         sb.setLength(0);
 177                                                         query.setLength(0);
 178                                                 }
 179
 180                                         } catch (IOException e) {
 181                                                 trans.error().log(e);
 182                                                 e.printStackTrace();
 183                                         }
 184
 185                                 } else {
 186                                         trans.error().log("No file found: ", file.getAbsolutePath());
 187                                 }
 188                         } finally {
 189                                 tt.done();
 190                                 System.err.flush();
 191                                 System.out.printf("\n%d applied in %d batches\n",count,batchCnt);
 192                         }
 193
 194                 }
 195
 196         }
 197
 198
 199         private String build(String feed, List<String> array) {
 200                 String rv;
 201                 switch(feed) {
 202                         case "approval":
 203                                 rv = String.format(APPR_INS_FMT,array.toArray());
 204                                 break;
 205                         case "artifact":
 206                                 rv = String.format(ARTI_INS_FMT,array.toArray());
 207                                 break;
 208                         case "cred":
 209                                 rv = String.format(CRED_INS_FMT,array.toArray());
 210                                 break;
 211                         case "ns":
 212                                 rv = String.format(NS_INS_FMT,array.toArray());
 213                                 break;
 214                         case "role":
 215                                 rv = String.format(ROLE_INS_FMT,array.toArray());
 216                                 break;
 217                         case "perm":
 218                                 rv = String.format(PERM_INS_FMT,array.toArray());
 219                                 break;
 220                         case "x509":
 221                                 rv = String.format(X509_INS_FMT,array.toArray());
 222                                 break;
 223                         default:
 224                                 rv = "";
 225                 }
 226                 array.clear();
 227                 return rv;
 228         }
 229
 230         private void addField(String feed, int fldcnt, List<String> array, StringBuilder sb) {
 231                 Boolean[] ba;
 232                 switch(feed) {
 233                         case "approval":
 234                                 ba = APPR_QUOTES;
 235                                 break;
 236                         case "artifact":
 237                                 ba = ARTI_QUOTES;
 238                                 break;
 239                         case "cred":
 240                                 ba = CRED_QUOTES;
 241                                 break;
 242                         case "ns":
 243                                 ba = NS_QUOTES;
 244                                 break;
 245                         case "role":
 246                                 ba = ROLE_QUOTES;
 247                                 break;
 248                         case "perm":
 249                                 ba = PERM_QUOTES;
 250                                 break;
 251                         case "x509":
 252                                 ba = X509_QUOTES;
 253                                 break;
 254                         default:
 255                                 ba = null;
 256                 }
 257                 if(ba!=null) {
 258                         if(sb.toString().length()==0) {
 259                                 array.add("null");
 260                         } else {
 261                                 if(ba[fldcnt]) {
 262                                         String s = null;
 263                                         if(sb.indexOf("'")>=0) {
 264                                                 s = sb.toString().replace("'","''");
 265                                         }
 266                                         if(sb.indexOf("\\n")>=0) {
 267                                                 if(s==null) {
 268                                                         s = sb.toString().replace("\\n","\n");
 269                                                 } else {
 270                                                         s = s.replace("\\n","\n");
 271                                                 }
 272                                         }
 273                                         if(sb.indexOf("\\t")>=0) {
 274                                                 if(s==null) {
 275                                                         s = sb.toString().replace("\\t","\t");
 276                                                 } else {
 277                                                         s = s.replace("\\t","\t");
 278                                                 }
 279                                         }
 280                                         if(s==null) {
 281                                                 array.add("'" + sb + '\'');
 282                                         } else {
 283                                                 array.add("'" + s + '\'');
 284                                         }
 285                                 } else {
 286                                         array.add(sb.toString());
 287                                 }
 288                         }
 289                         sb.setLength(0);
 290                 }
 291         }
 292
 293         private void applyBatch(StringBuilder query) {
 294                 try {
 295                         query.append("APPLY BATCH;");
 296                         ResultSet rv = session.execute(query.toString());
 297                         if(rv.wasApplied()) {
 298                                 System.out.print('.');
 299                                 if((++batchCnt % 60)==0) {
 300                                         System.out.println();
 301                                 }
 302                         } else {
 303                                 System.out.print("Data NOT APPLIED");
 304                         }
 305                 } finally {
 306                         query.setLength(0);
 307                 }
 308         }
 309
 310
 311         @Override
 312         protected void _close(AuthzTrans trans) {
 313         session.close();
 314         }
 315
 316 }
 317