X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=auth%2Fauth-batch%2Fsrc%2Fmain%2Fjava%2Forg%2Fonap%2Faaf%2Fauth%2Fbatch%2Fupdate%2FRemove.java;h=3d26ce99bc90c344d96d16cd415e701caa97277a;hb=83b93fc82cc39590f1f3aa1e92a53f37cd191416;hp=5f6021b3bbc481d878b4c5da1e107c23d8b369f0;hpb=00d96bbfd0f6caaa0bbcbe51d1c067822aa5040f;p=aaf%2Fauthz.git diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java index 5f6021b3..3d26ce99 100644 --- a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java +++ b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Remove.java @@ -7,9 +7,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,8 +31,12 @@ import java.util.List; import org.onap.aaf.auth.batch.Batch; import org.onap.aaf.auth.batch.BatchPrincipal; +import org.onap.aaf.auth.batch.helpers.Approval; import org.onap.aaf.auth.batch.helpers.CQLBatch; +import org.onap.aaf.auth.batch.helpers.CQLBatchLoop; import org.onap.aaf.auth.batch.helpers.Cred; +import org.onap.aaf.auth.batch.helpers.Future; +import org.onap.aaf.auth.batch.helpers.LastNotified; import org.onap.aaf.auth.batch.helpers.UserRole; import org.onap.aaf.auth.batch.helpers.X509; import org.onap.aaf.auth.dao.CassAccess; @@ -48,32 +52,31 @@ import org.onap.aaf.cadi.util.CSV; import org.onap.aaf.misc.env.APIException; import org.onap.aaf.misc.env.Env; import org.onap.aaf.misc.env.TimeTaken; +import org.onap.aaf.misc.env.Trans; import org.onap.aaf.misc.env.util.Chrono; public class Remove extends Batch { private final AuthzTrans noAvg; private HistoryDAO historyDAO; - private CQLBatch cqlBatch; + private CQLBatch cqlBatch; public Remove(AuthzTrans trans) throws APIException, IOException, OrganizationException { super(trans.env()); trans.info().log("Starting Connection Process"); - + noAvg = env.newTransNoAvg(); noAvg.setUser(new BatchPrincipal("Remove")); TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB); try { - historyDAO = new HistoryDAO(trans, cluster, CassAccess.KEYSPACE); + historyDAO = new HistoryDAO(trans, cluster, CassAccess.KEYSPACE); TimeTaken tt2 = trans.start("Connect to Cluster", Env.REMOTE); try { session = historyDAO.getSession(trans); } finally { tt2.done(); } - cqlBatch = new CQLBatch(noAvg.info(),session); - - + cqlBatch = new CQLBatch(noAvg.info(),session); } finally { tt0.done(); } @@ -81,124 +84,133 @@ public class Remove extends Batch { @Override protected void run(AuthzTrans trans) { - final int maxBatch = 25; - // Create Intermediate Output - File logDir = new File(logDir()); - + // Create Intermediate Output + File logDir = logDir(); + List remove = new ArrayList<>(); if(args().length>0) { - for(int i=0;i { + String name = f.getName(); + return name.endsWith(ending) && ( + name.startsWith("Delete") || name.startsWith("Clean.")); + })) { + remove.add(file); + }; } - + for(File f : remove) { - trans.init().log("Processing File:",f.getAbsolutePath()); + trans.init().log("Processing File:",f.getAbsolutePath()); } - + final Holder ur = new Holder<>(false); final Holder cred = new Holder<>(false); final Holder x509 = new Holder<>(false); final Holder memoFmt = new Holder(""); final HistoryDAO.Data hdd = new HistoryDAO.Data(); final String orgName = trans.org().getName(); - + hdd.action="delete"; hdd.reconstruct = ByteBuffer.allocate(0); hdd.user = noAvg.user(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); hdd.yr_mon = Integer.parseInt(sdf.format(new Date())); - - try { - for(File f : remove) { - trans.info().log("Processing ",f.getAbsolutePath(),"for Deletions"); - if(f.exists()) { - CSV removeCSV = new CSV(f); - - try { - final StringBuilder sb = cqlBatch.begin(); - final Holder hi = new Holder(0); - removeCSV.visit(new CSV.Visitor() { - @Override - public void visit(List row) throws IOException, CadiException { - int i = hi.get(); - if(i>=maxBatch) { - cqlBatch.execute(dryRun); - hi.set(0); - cqlBatch.begin(); - i=0; - } - switch(row.get(0)) { - case "info": - switch(row.get(1)) { - case "Delete": - memoFmt.set("%s expired from %s on %s"); - break; - case "NotInOrgDelete": - memoFmt.set("Identity %s was removed from %s on %s"); - break; - } - break; - case "ur": - if(!ur.get()) { - ur.set(true); - } - hi.set(++i); - UserRole.batchDelete(sb,row); - hdd.target=UserRoleDAO.TABLE; - hdd.subject=UserRole.histSubject(row); - hdd.memo=UserRole.histMemo(memoFmt.get(), row); - historyDAO.createBatch(sb, hdd); - break; - case "cred": - if(!cred.get()) { - cred.set(true); - } - hi.set(++i); - Cred.batchDelete(sb,row); - hdd.target=CredDAO.TABLE; - hdd.subject=Cred.histSubject(row); - hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row); - historyDAO.createBatch(sb, hdd); - break; - case "x509": - if(!x509.get()) { - x509.set(true); - } - hi.set(++i); - X509.row(sb,row); - hdd.target=CertDAO.TABLE; - hdd.subject=X509.histSubject(row); - hdd.memo=X509.histMemo(memoFmt.get(),row); - historyDAO.createBatch(sb, hdd); - break; - } - } - }); - cqlBatch.execute(dryRun); - } catch (IOException | CadiException e) { - e.printStackTrace(); - } - } else { - trans.error().log("File",f.getAbsolutePath(),"does not exist."); - } - } + + try { + final CQLBatchLoop cbl = new CQLBatchLoop(cqlBatch,50,dryRun); + for(File f : remove) { + trans.info().log("Processing ",f.getAbsolutePath(),"for Deletions"); + if(f.exists()) { + CSV removeCSV = new CSV(env.access(),f); + try { + removeCSV.visit( row -> { + switch(row.get(0)) { + case "info": + switch(row.get(1)) { + case "Delete": + memoFmt.set("%s expired from %s on %s"); + break; + case "NotInOrgDelete": + memoFmt.set("Identity %s was removed from %s on %s"); + break; + } + break; + case "ur": + if(!ur.get()) { + ur.set(true); + } + //TODO If deleted because Role is no longer there, double check... + + UserRole.batchDelete(cbl.inc(),row); + hdd.target=UserRoleDAO.TABLE; + hdd.subject=UserRole.histSubject(row); + hdd.memo=UserRole.histMemo(memoFmt.get(), row); + historyDAO.createBatch(cbl.inc(), hdd); + break; + case "cred": + if(!cred.get()) { + cred.set(true); + } + Cred.batchDelete(cbl.inc(),row); + hdd.target=CredDAO.TABLE; + hdd.subject=Cred.histSubject(row); + hdd.memo=Cred.histMemo(memoFmt.get(), orgName,row); + historyDAO.createBatch(cbl.inc(), hdd); + break; + case "x509": + if(!x509.get()) { + x509.set(true); + } + X509.batchDelete(cbl.inc(),row); + hdd.target="x509"; + hdd.subject=X509.histSubject(row); + hdd.memo=X509.histMemo(memoFmt.get(),row); + historyDAO.createBatch(cbl.inc(), hdd); + break; + case "future": + // Not cached + Future.deleteByIDBatch(cbl.inc(),row.get(1)); + break; + case "approval": + // Not cached + Approval.deleteByIDBatch(cbl.inc(),row.get(1)); + break; + case "notified": + LastNotified.delete(cbl.inc(),row); + break; + } + }); + cbl.flush(); + } catch (IOException | CadiException e) { + e.printStackTrace(); + } + } else { + trans.error().log("File",f.getAbsolutePath(),"does not exist."); + } + } } finally { - if(ur.get()) { - cqlBatch.touch(UserRoleDAO.TABLE, 0, UserRoleDAO.CACHE_SEG, dryRun); - } - if(cred.get()) { - cqlBatch.touch(CredDAO.TABLE, 0, CredDAO.CACHE_SEG, dryRun); - } - if(x509.get()) { - cqlBatch.touch(CertDAO.TABLE, 0, CertDAO.CACHE_SEG, dryRun); - } + TimeTaken tt = trans.start("Touch UR,Cred and Cert Caches",Trans.REMOTE); + try { + if(ur.get()) { + cqlBatch.touch(UserRoleDAO.TABLE, 0, UserRoleDAO.CACHE_SEG, dryRun); + } + if(cred.get()) { + cqlBatch.touch(CredDAO.TABLE, 0, CredDAO.CACHE_SEG, dryRun); + } + if(x509.get()) { + cqlBatch.touch(CertDAO.TABLE, 0, CertDAO.CACHE_SEG, dryRun); + } + } finally { + tt.done(); + } } } - + @Override protected void _close(AuthzTrans trans) { session.close();