auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Upload.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  *
   7  * Modifications Copyright (C) 2018 IBM.
   8  * ===========================================================================
   9  * Licensed under the Apache License, Version 2.0 (the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  *      http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  * ============LICENSE_END====================================================
  21  */
  22
  23 package org.onap.aaf.auth.batch.update;
  24
  25 import java.io.File;
  26 import java.io.IOException;
  27 import java.util.ArrayList;
  28 import java.util.HashMap;
  29 import java.util.List;
  30 import java.util.Map;
  31
  32 import org.onap.aaf.auth.batch.Batch;
  33 import org.onap.aaf.auth.batch.helpers.CQLBatch;
  34 import org.onap.aaf.auth.batch.helpers.CQLBatchLoop;
  35 import org.onap.aaf.auth.env.AuthzTrans;
  36 import org.onap.aaf.auth.org.OrganizationException;
  37 import org.onap.aaf.cadi.util.CSV;
  38 import org.onap.aaf.misc.env.APIException;
  39 import org.onap.aaf.misc.env.Env;
  40 import org.onap.aaf.misc.env.LogTarget;
  41 import org.onap.aaf.misc.env.TimeTaken;
  42 import org.onap.aaf.misc.env.util.Split;
  43
  44 public class Upload extends Batch {
  45
  46         private static final String DAT = ".dat";
  47
  48         private CQLBatch cqlBatch;
  49
  50         private Map<String,Feed> feeds;
  51
  52
  53         public Upload(AuthzTrans trans) throws APIException, IOException, OrganizationException {
  54                 super(trans.env());
  55             trans.info().log("Starting Connection Process");
  56
  57             TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
  58             try {
  59                         TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
  60                         try {
  61                                 session = cluster.connect();
  62                         } finally {
  63                                 tt.done();
  64                         }
  65
  66                         cqlBatch = new CQLBatch(LogTarget.NULL,session);
  67
  68                         feeds=new HashMap<>();
  69                         new Feed(feeds,"ns",1,"name,description,parent,scope=int,type=int",300);
  70                         new Feed(feeds,"notified",3,"user,target,key,last",300);
  71                         new Feed(feeds,"approval",1,"id=UUID,approver,last_notified,memo,operation,status,ticket=UUID,type,user",200);
  72                         new Feed(feeds,"artifact",2,"mechid,machine,ca,dir,expires,notify,ns,os_user,renewdays=int,sans=set,sponsor,type=set",200);
  73                         new Feed(feeds,"cred",1,"id,type=int,expires,cred=blob,notes,ns,other=int,prev=blob,tag",200);
  74                         new Feed(feeds,"x509",2,"ca,serial=blob,id,x500,x509=C/R",200);
  75                         new Feed(feeds,"role",2,"ns,name,description,perms=set",200);
  76                         new Feed(feeds,"perm",4,"ns,type,instance,action,description,roles=set",200);
  77                         new Feed(feeds,"history",1,"id=UUID,action,memo,reconstruct=blob,subject,target,user,yr_mon=int",300);
  78
  79             } finally {
  80                 tt0.done();
  81             }
  82         }
  83
  84
  85         @Override
  86         protected void run(AuthzTrans trans) {
  87                 List<File> files = new ArrayList<>();
  88                 if(args().length>0) {
  89                         File dir = new File(args()[0]);
  90                         if(dir.isDirectory()) {
  91                                 for(File f : dir.listFiles(pathname -> {
  92                                         return pathname.getName().endsWith(DAT);
  93                                 })) {
  94                                         files.add(f);
  95                                 }
  96                         } else {
  97                                 File f;
  98                                 for(String arg : args()) {
  99                                         if(arg.endsWith(DAT)) {
 100                                                 f=new File(arg);
 101                                         } else {
 102                                                 f=new File(arg+DAT);
 103                                         }
 104                                         files.add(f);
 105                                 }
 106                         }
 107                 }
 108                 for(File file : files) {
 109                         String f = file.getName();
 110                         final Feed feed = feeds.get(f.substring(0,f.length()-4));
 111                         if(feed!=null) {
 112                                 TimeTaken tt = trans.start(file.getAbsolutePath(), Env.SUB);
 113                                 String msg = String.format("#### Running %s.dat Feed ####",feed.getName());
 114                                 trans.info().log(msg);
 115                                 System.out.println(msg);
 116                                 CQLBatchLoop cbl = new CQLBatchLoop(cqlBatch,feed.batchSize,dryRun).showProgress();
 117
 118                             try {
 119                                         if(file.exists()) {
 120                                                 CSV csv = new CSV(trans.env().access(),file).setDelimiter('|');
 121                                                 csv.visit( row -> {
 122                                                         feed.insert(cbl.inc(),row);
 123                                                 });
 124                                         }
 125                                         cbl.flush();
 126                                 } catch (Throwable e) {
 127                                         e.printStackTrace();
 128                                 } finally {
 129                                         tt.done();
 130                                         System.err.flush();
 131                                         msg = String.format("\n%d applied in %d batches\n",cbl.total(), cbl.batches());
 132                                         trans.info().log(msg);
 133                                         System.out.println(msg);
 134                                 }
 135                         }
 136                 }
 137         }
 138
 139         @Override
 140         protected void _close(AuthzTrans trans) {
 141         session.close();
 142         }
 143
 144         private class Feed {
 145                 private final String name;
 146                 private final String[] flds;
 147                 private final String[] types;
 148                 private final int key;
 149                 private final int batchSize;
 150                 public Feed(Map<String, Feed> feeds, String feed, int keyLength, String fields,int batchSize) {
 151                         name=feed;
 152                         key = keyLength;
 153                         flds = Split.splitTrim(',', fields);
 154                         types = new String[flds.length];
 155                         this.batchSize = batchSize;
 156                         int equals;
 157                         for(int i=0;i<flds.length;++i) {
 158                                 if((equals = flds[i].indexOf('='))>0) {
 159                                         types[i]=flds[i].substring(equals+1);
 160                                         flds[i]=flds[i].substring(0, equals);
 161                                 }
 162                         }
 163                         feeds.put(feed,this);
 164                 }
 165
 166                 public String getName() {
 167                         return name;
 168                 }
 169
 170                 public void insert(StringBuilder sb,List<String> row) {
 171                         sb.append("INSERT INTO authz.");
 172                         sb.append(name);
 173                         sb.append(" (");
 174                         boolean first = true;
 175                         StringBuilder values = new StringBuilder(") VALUES (");
 176                         String value;
 177                         String type;
 178                         for(int idx=0;idx<row.size();++idx) {
 179                                 value = row.get(idx).trim();
 180                                 if(idx<key || !(value.isEmpty() || "null".equals(value))) {
 181                                         if(first) {
 182                                                 first = false;
 183                                         } else {
 184                                                 sb.append(',');
 185                                                 values.append(',');
 186                                         }
 187                                         sb.append(flds[idx]);
 188                                         type=types[idx];
 189                                         if(type==null) { // String is default.
 190                                                 switch(value) {
 191                                                   case "":
 192                                                         if(idx<key) {
 193                                                                 // Key value has to be something, but can't be actual null
 194                                                                 values.append("''");
 195                                                         } else {
 196                                                                 values.append("null");
 197                                                         }
 198                                                         break;
 199                                                   default:
 200                                                         values.append('\'');
 201                                                         values.append(value.replaceAll("'","''"));
 202                                                         values.append('\'');
 203                                                 }
 204                                         } else switch(type) {
 205                                                         case "C/R":
 206                                                                 values.append('\'');
 207                                                                 values.append(value.replaceAll("\\\\n", "\n"));
 208                                                                 values.append('\'');
 209                                                                 break;
 210                                                         default:
 211                                                                 values.append(value);
 212                                                                 break;
 213
 214                                         }
 215                                 }
 216                         }
 217                         sb.append(values);
 218                         sb.append(");\n");
 219                 }
 220         }
 221 }
 222