Refine Props, Batch Extend 51/77951/1
authorInstrumental <jonathan.gathman@att.com>
Wed, 6 Feb 2019 12:49:17 +0000 (06:49 -0600)
committerInstrumental <jonathan.gathman@att.com>
Wed, 6 Feb 2019 12:49:34 +0000 (06:49 -0600)
Issue-ID: AAF-740
Change-Id: I119c3e6573d59d6d182303dfd364342f0218b865
Signed-off-by: Instrumental <jonathan.gathman@att.com>
auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/helpers/Cred.java
auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java [new file with mode: 0644]
auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Extend.java [new file with mode: 0644]
auth/auth-batch/src/test/java/org/onap/aaf/auth/batch/helpers/test/JU_Future.java
auth/docker/aaf.sh
cadi/core/src/main/java/org/onap/aaf/cadi/config/Config.java
cadi/core/src/main/java/org/onap/aaf/cadi/config/RegistrationPropHolder.java

index e3ac9e6..c4a9b0d 100644 (file)
@@ -138,13 +138,14 @@ public class Cred  {
                     row = iter.next();
                     int type = row.getInt(1);
                     if (types.length>0) { // filter by types, if requested
-                        boolean quit = true;
+                        boolean hastype = false;
                         for (int t : types) {
                             if (t==type) {
+                               hastype=true;
                                 break;
                             }
                         }
-                        if (quit) {
+                        if (!hastype) {
                             continue;
                         }
                     }
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/PrepExtend.java
new file mode 100644 (file)
index 0000000..3e0dd01
--- /dev/null
@@ -0,0 +1,184 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+package org.onap.aaf.auth.batch.reports;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.onap.aaf.auth.batch.Batch;
+import org.onap.aaf.auth.batch.helpers.Cred;
+import org.onap.aaf.auth.batch.helpers.Cred.Instance;
+import org.onap.aaf.auth.batch.helpers.UserRole;
+import org.onap.aaf.auth.dao.cass.CredDAO;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.cadi.util.CSV.Writer;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+public class PrepExtend extends Batch {
+
+       public static final String PREP_EXTEND = "PrepExtend";
+       private static final String CSV = ".csv";
+       private static final String INFO = "info";
+
+       /**
+        * Create a list of Creds and UserRoles to extend
+        * Note: Certificates cannot be renewed in this way.
+        * 
+        * Arguments From (0 = today, -2 = 2 weeks back) and To (weeks from today)
+        * 
+        * @param trans
+        * @throws APIException
+        * @throws IOException
+        * @throws OrganizationException
+        */
+       public PrepExtend(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+               super(trans.env());
+               trans.info().log("Starting Connection Process");
+
+               TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+               try {
+                       TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+                       try {
+                               session = cluster.connect();
+                       } finally {
+                               tt.done();
+                       }
+               } finally {
+                       tt0.done();
+               }
+       }
+
+       @Override
+       protected void run(AuthzTrans trans) {
+               GregorianCalendar gc = new GregorianCalendar();
+               Date now = gc.getTime();
+               
+               int ifrom = 0;
+               int ito = 0;
+               
+               for(int i=0; i< args().length;++i) {
+                       switch(args()[i]) {
+                               case "-from":
+                                       if(args().length>i+1) {
+                                               ifrom = Integer.parseInt(args()[i++ +1]); 
+                                       }
+                                       break;
+                               case "-to":
+                                       if(args().length>i+1) {
+                                               ito = Integer.parseInt(args()[i++ +1]);
+                                       }
+                                       break;
+                       }
+               }
+               if(ifrom < -4) {
+                       System.err.println("Invalid -from param");
+                       return;
+               }
+               
+               if(ito<=0 || ito>24 || ifrom>ito) {
+                       System.err.println("Invalid -to param");
+                       return;
+               }
+               
+               // Make sure to is Zero based from today.
+               if(ifrom<0) {
+                       ito+= ifrom*-1;
+               }
+               
+               gc.add(GregorianCalendar.WEEK_OF_MONTH, ifrom);
+               Date from = gc.getTime();
+               
+               gc.add(GregorianCalendar.WEEK_OF_MONTH, ito /* with From calculated in */);
+               Date to = gc.getTime();
+               
+               try {
+                       File file = new File(logDir(), PREP_EXTEND + Chrono.dateOnlyStamp(now) + CSV);
+                       final CSV puntCSV = new CSV(env.access(),file);
+                       final Writer cw = puntCSV.writer();
+                       cw.row(INFO,PREP_EXTEND,Chrono.dateOnlyStamp(now),0);
+
+                       try {
+                               trans.info().log("Process UserRoles for Extending");
+                               /**
+                                  Run through User Roles.  
+                                  If match Date Range, write out to appropriate file.
+                               */
+                               UserRole.load(trans, session, UserRole.v2_0_11, ur -> {
+                                       if(from.before(ur.expires()) && to.after(ur.expires())) {
+                                               ur.row(cw);
+                                       }
+                               });
+                               
+                               trans.info().log("Process BasicAuth for Extending");
+                               TimeTaken tt0 = trans.start("Load Credentials", Env.REMOTE);
+                               try {
+                                       // Load only Valid Basic Auth
+                                       Cred.load(trans, session, CredDAO.BASIC_AUTH_SHA256);
+                               } finally {
+                                       tt0.done();
+                               }
+
+
+                               /**
+                                  Run through Creds.  
+                                  If match Date Range, write out to appropriate file.
+                               */
+                               Map<Integer,Instance> imap = new HashMap<>();
+                               Instance prev;
+                               for(Cred cred : Cred.data.values()) {
+                                       imap.clear();
+                                       for(Instance i : cred.instances) {
+                                               if(from.before(i.expires) && to.after(i.expires)) {
+                                                       prev = imap.get(i.other);
+                                                       // Only do LATEST instance of same cred (accounts for previously extended creds)
+                                                       if(prev==null || prev.expires.before(i.expires)) {
+                                                               imap.put(i.other,i);
+                                                       }
+                                               }
+                                       };
+                                       for(Instance i: imap.values()) {
+                                               cred.row(cw,i);
+                                       }
+                               }
+                       } finally {
+                               cw.close();
+                       }
+               } catch (FileNotFoundException e) {
+                       e.printStackTrace();
+               }
+       }
+       @Override
+       protected void _close(AuthzTrans trans) {
+               session.close();
+       }
+
+
+}
diff --git a/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Extend.java b/auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/update/Extend.java
new file mode 100644 (file)
index 0000000..0179593
--- /dev/null
@@ -0,0 +1,207 @@
+/**
+ * ============LICENSE_START====================================================
+ * org.onap.aaf
+ * ===========================================================================
+ * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+ * ===========================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END====================================================
+ *
+ */
+package org.onap.aaf.auth.batch.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.List;
+
+import org.onap.aaf.auth.batch.Batch;
+import org.onap.aaf.auth.batch.BatchPrincipal;
+import org.onap.aaf.auth.batch.helpers.CQLBatch;
+import org.onap.aaf.auth.batch.helpers.UserRole;
+import org.onap.aaf.auth.batch.reports.PrepExtend;
+import org.onap.aaf.auth.dao.CassAccess;
+import org.onap.aaf.auth.dao.cass.CredDAO;
+import org.onap.aaf.auth.dao.cass.CredDAO.Data;
+import org.onap.aaf.auth.env.AuthzTrans;
+import org.onap.aaf.auth.layer.Result;
+import org.onap.aaf.auth.org.OrganizationException;
+import org.onap.aaf.cadi.CadiException;
+import org.onap.aaf.cadi.client.Holder;
+import org.onap.aaf.cadi.util.CSV;
+import org.onap.aaf.misc.env.APIException;
+import org.onap.aaf.misc.env.Env;
+import org.onap.aaf.misc.env.TimeTaken;
+import org.onap.aaf.misc.env.util.Chrono;
+
+public class Extend extends Batch {
+       private final CQLBatch cqlBatch;
+       private final CredDAO credDAO;
+       private final AuthzTrans noAvg;
+       private List<File> extFiles;
+       private final int extendBy;
+       private int gcType;
+       
+       public Extend(AuthzTrans trans) throws APIException, IOException, OrganizationException {
+               super(trans.env());
+        trans.info().log("Starting Connection Process");
+        
+        noAvg = env.newTransNoAvg();
+        noAvg.setUser(new BatchPrincipal("Extend"));
+
+        TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
+        try {
+                       TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
+                       credDAO = new CredDAO(trans, cluster, CassAccess.KEYSPACE);
+                       try {
+                               session = credDAO.getSession(trans);
+                       } finally {
+                               tt.done();
+                       }
+            cqlBatch = new CQLBatch(noAvg.info(),session); 
+        } finally {
+            tt0.done();
+        }
+
+        gcType = GregorianCalendar.WEEK_OF_YEAR;
+        int weeks = 4;
+               
+               for(int i=0; i< args().length;++i) {
+                       if("-weeks".equals(args()[i])) {
+                               if(args().length>i+1) {
+                                       weeks = Integer.parseInt(args()[i +1]);
+                                       break;
+                               }
+                       }
+               }
+               
+               if(weeks<1 || weeks > 24) {
+                       throw new APIException("Invalid --weeks");
+               }
+               extendBy = weeks;
+
+        // Create Intermediate Output 
+        File logDir = logDir();
+        extFiles = new ArrayList<>();
+        if(args().length>0) {
+               for(int i=0;i<args().length;++i) {
+                       extFiles.add(new File(logDir, args()[i]));
+               }
+        } else {
+               extFiles.add(new File(logDir,PrepExtend.PREP_EXTEND+Chrono.dateOnlyStamp()+".csv"));
+        }
+        
+        // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
+        // to find the last one.
+       }
+
+       @Override
+       protected void run(AuthzTrans trans) {
+        final int maxBatch = 50;
+
+               // Setup Date boundaries
+        final Holder<GregorianCalendar> hgc = new Holder<>(new GregorianCalendar());
+        final GregorianCalendar now = new GregorianCalendar();
+
+        ///////////////////////////
+        trans.info().log("Bulk Extend Expiring User-Roles and Creds");
+
+               final Holder<List<String>> info = new Holder<>(null);
+               final Holder<StringBuilder> hsb = new Holder<>(null);
+
+               for(File f : extFiles) {
+                       CSV csv = new CSV(env.access(),f);
+                       try {
+                               csv.visit(new CSV.Visitor() {
+                               final Holder<Integer> hi = new Holder<>(0); 
+
+                                       @Override
+                                       public void visit(List<String> row) throws IOException, CadiException {
+                                               GregorianCalendar gc;
+                                               int i = hi.get();
+                                               StringBuilder sb = hsb.get();
+                                               if(sb==null) {
+                                                       hsb.set(sb=cqlBatch.begin());
+                                               }
+                                               switch(row.get(0)) {
+                                                       case "info":
+                                                               info.set(row);
+                                                               break;
+                                                       case "ur":
+                                                               hi.set(++i);
+                                                               gc = hgc.get();
+                                                               gc.setTime(new Date(Long.parseLong(row.get(5))));
+                                                               if(gc.before(now)) {
+                                                                       gc.setTime(now.getTime());
+                                                               }
+                                                               gc.add(gcType, extendBy);
+                                                               UserRole.batchExtend(sb,row,Chrono.dateTime(gc));
+                                                               break;
+                                                       case "cred":
+                                                               int ctype = Integer.parseInt(row.get(3));
+                                                               if(ctype == CredDAO.BASIC_AUTH_SHA256 || ctype == CredDAO.BASIC_AUTH) {
+                                                                       Result<List<Data>> result = credDAO.readID(noAvg, row.get(1));
+                                                                       if(result.isOKhasData()) {
+                                                                               for(CredDAO.Data cd : result.value) {
+                                                                                       if(cd.type == CredDAO.BASIC_AUTH_SHA256 || cd.type == CredDAO.BASIC_AUTH) {
+                                                                                               String prev;
+                                                                                               if(row.get(4).equals(prev=Chrono.dateOnlyStamp(cd.expires))) {
+                                                                                                       gc = hgc.get();
+                                                                                                       gc.setTime(new Date(Long.parseLong(row.get(5))));
+                                                                                                       if(gc.before(now)) {
+                                                                                                               gc.setTime(now.getTime());
+                                                                                                       }
+                                                                                                       gc.add(gcType, extendBy);
+                                                                                                       cd.expires = gc.getTime();
+                                                                                                       if(dryRun) {
+                                                                                                               noAvg.info().printf("Would extend %s, %d - %s to %s",cd.id,cd.type,prev, Chrono.dateOnlyStamp(cd.expires));
+                                                                                                       } else {
+                                                                                                               Result<Void> r = credDAO.update(noAvg, cd, true);
+                                                                                                               noAvg.info().printf("%s %s, %d - %s to %s",
+                                                                                                                               r.isOK()?"Extended":"Failed to Extend",
+                                                                                                                               cd.id,cd.type,prev, Chrono.dateOnlyStamp(cd.expires));
+                                                                                                       }
+                                                                                               }
+                                                                                       }
+                                                                               }
+                                                                       }
+                                                               }
+                                                               break;
+                                               }
+                                               if(i%maxBatch==0 && sb!=null) {
+                                                       cqlBatch.execute(dryRun);
+                                                       hi.set(1);
+                                                       hsb.set(sb=null);
+                                               }
+                                       }
+                               });
+                       } catch (IOException | CadiException e) {
+                               e.printStackTrace();
+                       }
+               }
+               
+               // Cleanup, if required.
+               cqlBatch.execute(dryRun);
+
+       }
+       
+       @Override
+       protected void _close(AuthzTrans trans) {
+        trans.info().log("End " + this.getClass().getSimpleName() + " processing" );
+        credDAO.close(trans);
+        session.close();
+       }
+
+}
index 91950c6..0db682a 100644 (file)
@@ -105,15 +105,7 @@ public class JU_Future {
        @Test
        public void testResetLocalData() {
                Future.resetLocalData();
-       }
-
-       @Test
-       public void testSizeForDeletion() {
                Assert.assertEquals(0, Future.sizeForDeletion());
-       }
-
-       @Test
-       public void testPendingDelete() {
                Assert.assertEquals(false, Future.pendingDelete(future));
        }
 
index 14166a3..3ed7eaf 100644 (file)
@@ -29,6 +29,7 @@ function run_it() {
     --env aaf_locator_container=docker \
     --env aaf_locator_fqdn=${HOSTNAME} \
     --env aaf_locate_url=https://aaf-locate:8095 \
+    --env aaf_locator_public_hostname=$HOSTNAME \
     --env AAF_ENV=${AAF_ENV} \
     --env LATITUDE=${LATITUDE} \
     --env LONGITUDE=${LONGITUDE} \
index f265947..fc972c8 100644 (file)
@@ -36,8 +36,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import javax.servlet.Registration;
-
 import org.onap.aaf.cadi.AbsUserCache;
 import org.onap.aaf.cadi.Access;
 import org.onap.aaf.cadi.Access.Level;
index c813964..b6cd533 100644 (file)
@@ -28,7 +28,7 @@ import org.onap.aaf.cadi.CadiException;
 import org.onap.aaf.cadi.util.Split;
 
 public class RegistrationPropHolder {
-
+       private final String PUBLIC_NAME="%NS.%N";
        private final Access access;
        public String hostname;
        private int port;
@@ -38,6 +38,7 @@ public class RegistrationPropHolder {
        public Float longitude;
        public final String default_fqdn;
        public final String default_container_ns;
+       public final String default_name;
        public final String lentries;
        public final String lcontainer;
 
@@ -74,6 +75,8 @@ public class RegistrationPropHolder {
 
                public_hostname = access.getProperty(Config.AAF_LOCATOR_PUBLIC_HOSTNAME, hostname);
                                
+               default_name = access.getProperty(Config.AAF_LOCATOR_NAME, "%CNS.%NS.%N");
+               
                latitude=null;
                String slatitude = access.getProperty(Config.CADI_LATITUDE, null);
                if(slatitude == null) {
@@ -100,7 +103,7 @@ public class RegistrationPropHolder {
                        }
                }
                
-               default_fqdn = access.getProperty(Config.AAF_LOCATOR_FQDN, public_hostname);
+               default_fqdn = access.getProperty(Config.AAF_LOCATOR_FQDN, hostname);
                default_container_ns = access.getProperty(Config.AAF_LOCATOR_CONTAINER_NS,"");
                
                if(errs.length()>0) {
@@ -120,17 +123,18 @@ public class RegistrationPropHolder {
                if(public_hostname!=null && dot_le.isEmpty()) {
                        str = public_hostname;
                } else {
-                       str = access.getProperty(Config.AAF_LOCATOR_FQDN+dot_le, null);
-                       if(str==null) {
-                               str = access.getProperty(Config.AAF_LOCATOR_FQDN, hostname);
-                       }
+                       str = access.getProperty(Config.AAF_LOCATOR_FQDN+dot_le, default_fqdn);
                }
                return replacements(str,entry,dot_le);
        }
        
        public String getEntryName(final String entry, final String dot_le) {
                String str;
-               str = access.getProperty(Config.AAF_LOCATOR_NAME+dot_le, "%NS.%N");
+               if(dot_le.isEmpty()) {
+                       str = PUBLIC_NAME;
+               } else {
+                       str = access.getProperty(Config.AAF_LOCATOR_NAME+dot_le, default_name);
+               }
                return replacements(str,entry,dot_le);
        }