eb998b047bafb2fbcf8b78651bc1d47a7a499a40
[aaf/authz.git] / auth / auth-batch / src / main / java / org / onap / aaf / auth / batch / reports / Analyze.java
1 /**
2  * ============LICENSE_START====================================================
3  * org.onap.aaf
4  * ===========================================================================
5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
6  *
7  * Modifications Copyright (C) 2019 IBM.
8  * ===========================================================================
9  * Licensed under the Apache License, Version 2.0 (the "License");
10  * you may not use this file except in compliance with the License.
11  * You may obtain a copy of the License at
12  * 
13  *      http://www.apache.org/licenses/LICENSE-2.0
14  * 
15  * Unless required by applicable law or agreed to in writing, software
16  * distributed under the License is distributed on an "AS IS" BASIS,
17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18  * See the License for the specific language governing permissions and
19  * limitations under the License.
20  * ============LICENSE_END====================================================
21  *
22  */
23
24 package org.onap.aaf.auth.batch.reports;
25
26 import java.io.File;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.security.cert.Certificate;
30 import java.security.cert.CertificateException;
31 import java.security.cert.X509Certificate;
32 import java.util.Date;
33 import java.util.GregorianCalendar;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.TreeMap;
41 import java.util.TreeSet;
42 import java.util.UUID;
43
44 import org.onap.aaf.auth.batch.Batch;
45 import org.onap.aaf.auth.batch.approvalsets.Pending;
46 import org.onap.aaf.auth.batch.approvalsets.Ticket;
47 import org.onap.aaf.auth.batch.helpers.Approval;
48 import org.onap.aaf.auth.batch.helpers.Cred;
49 import org.onap.aaf.auth.batch.helpers.Cred.Instance;
50 import org.onap.aaf.auth.batch.helpers.ExpireRange;
51 import org.onap.aaf.auth.batch.helpers.ExpireRange.Range;
52 import org.onap.aaf.auth.batch.helpers.Future;
53 import org.onap.aaf.auth.batch.helpers.LastNotified;
54 import org.onap.aaf.auth.batch.helpers.Role;
55 import org.onap.aaf.auth.batch.helpers.UserRole;
56 import org.onap.aaf.auth.batch.helpers.X509;
57 import org.onap.aaf.auth.dao.cass.CredDAO;
58 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
59 import org.onap.aaf.auth.env.AuthzTrans;
60 import org.onap.aaf.auth.org.Organization.Identity;
61 import org.onap.aaf.auth.org.OrganizationException;
62 import org.onap.aaf.cadi.configure.Factory;
63 import org.onap.aaf.cadi.util.CSV;
64 import org.onap.aaf.misc.env.APIException;
65 import org.onap.aaf.misc.env.Env;
66 import org.onap.aaf.misc.env.TimeTaken;
67 import org.onap.aaf.misc.env.Trans;
68 import org.onap.aaf.misc.env.util.Chrono;
69
70
71 public class Analyze extends Batch {
72         private static final int unknown=0;
73     private static final int owner=1;
74     private static final int supervisor=2;
75     private static final int total=0;
76     private static final int pending=1;
77     private static final int approved=2;
78     
79     
80         public static final String NEED_APPROVALS = "NeedApprovals";
81         private static final String EXTEND = "Extend";
82         private static final String EXPIRED_OWNERS = "ExpiredOwners";
83         private static final String CSV = ".csv";
84         private static final String INFO = "info";
85         private int minOwners;
86         private Map<String, CSV.Writer> writerList;
87         private ExpireRange expireRange;
88         private Date deleteDate;
89         private CSV.Writer deleteCW;
90         private CSV.Writer needApproveCW;
91         private CSV.Writer extendCW;
92         private Range futureRange;
93         private final String sdate;
94         private LastNotified ln;
95         
96         public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException {
97         super(trans.env());
98         trans.info().log("Starting Connection Process");
99         
100         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
101         try {
102             TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
103             try {
104                 session = cluster.connect();
105             } finally {
106                 tt.done();
107             }
108             
109
110             minOwners=1;
111
112             // Create Intermediate Output 
113             writerList = new HashMap<>();
114             
115             expireRange = new ExpireRange(trans.env().access());
116             sdate = Chrono.dateOnlyStamp(now);
117             for( List<Range> lr : expireRange.ranges.values()) {
118                 for(Range r : lr ) {
119                         if(writerList.get(r.name())==null) {
120                         File file = new File(logDir(),r.name() + sdate +CSV);
121                         CSV csv = new CSV(env.access(),file);
122                         CSV.Writer cw = csv.writer(false);
123                         cw.row(INFO,r.name(),sdate,r.reportingLevel());
124                         writerList.put(r.name(),cw);
125                         if("Delete".equals(r.name())) {
126                                 deleteDate = r.getEnd();
127                                 deleteCW = cw;
128                         }
129                         trans.init().log("Creating File:",file.getAbsolutePath());
130                         }
131                 }
132             }
133             
134             // Setup New Approvals file
135             futureRange = expireRange.newFutureRange();
136             File file = new File(logDir(),NEED_APPROVALS + sdate +CSV);
137             CSV approveCSV = new CSV(env.access(),file);
138             needApproveCW = approveCSV.writer();
139             needApproveCW.row(INFO,NEED_APPROVALS,sdate,1);
140             writerList.put(NEED_APPROVALS,needApproveCW);
141             
142             // Setup Extend Approvals file
143             file = new File(logDir(),EXTEND + sdate +CSV);
144             CSV extendCSV = new CSV(env.access(),file);
145             extendCW = extendCSV.writer();
146             extendCW.row(INFO,EXTEND,sdate,1);
147             writerList.put(EXTEND,extendCW);
148             
149             // Load full data of the following
150             ln = new LastNotified(session);
151
152         } finally {
153             tt0.done();
154         }
155     }
156
157     @Override
158     protected void run(AuthzTrans trans) {
159         TimeTaken tt;
160         AuthzTrans noAvg = trans.env().newTransNoAvg();
161         
162         ////////////////////
163         // Load all Notifieds, and either add to local Data, or mark for Deletion.
164         ln.loadAll(noAvg,expireRange.approveDelete,deleteCW);
165         
166                 // Hold Good Tickets to keyed User/Role for UserRole Step
167                 Map<String,Ticket> mur = new TreeMap<>();
168
169         try {
170                 Approval.load(trans, session, Approval.v2_0_17);
171         
172                         ////////////////////
173                         final Map<UUID,Ticket> goodTickets = new TreeMap<>();
174                 tt = trans.start("Analyze Expired Futures",Trans.SUB);
175                 try {
176                                 Future.load(noAvg, session, Future.withConstruct, fut -> {
177                                         List<Approval> appls = Approval.byTicket.get(fut.id());
178                                         if(!futureRange.inRange(fut.expires())) {
179                                                 deleteCW.comment("Future %s expired", fut.id());
180                                                 Future.row(deleteCW,fut);
181                                                 if(appls!=null) {
182                                                         for(Approval a : appls) {
183                                                                 Approval.row(deleteCW, a);
184                                                         }
185                                                 }
186                                         } else if(appls==null) { // Orphaned Future (no Approvals)
187                                                 deleteCW.comment("Future is Orphaned");
188                                                 Future.row(deleteCW,fut);
189                                         } else  {
190                                                 goodTickets.put(fut.fdd.id, new Ticket(fut));
191                                         }
192                                 });
193                 } finally {
194                         tt.done();
195                 }
196                         
197                 Set<String> approvers = new TreeSet<>();
198                 tt = trans.start("Connect Approvals with Futures",Trans.SUB);
199                 try {
200                                 for(Approval appr : Approval.list) {
201                                         Ticket ticket=null;
202                                         UUID ticketID = appr.getTicket();
203                                         if(ticketID!=null) {
204                                                 ticket = goodTickets.get(appr.getTicket());
205                                         }
206                                         if(ticket == null) { // Orphaned Approvals, no Futures
207                                                 deleteCW.comment("Approval is Orphaned");
208                                                 Approval.row(deleteCW, appr);
209                                         } else {
210                                                 // for users and approvers still valid
211                                                 String user = appr.getUser();
212                                                 
213                                                 if(org.isRevoked(noAvg, appr.getApprover())) {
214                                                         deleteCW.comment("Approver ID is revoked");
215                                                         Approval.row(deleteCW, appr);
216                                                 } else if(user!=null && !user.isEmpty() && org.isRevoked(noAvg, user)) {
217                                                         deleteCW.comment("USER ID is revoked");
218                                                         Approval.row(deleteCW, appr);
219                                                 } else {
220                                                         ticket.approvals.add(appr); // add to found Ticket
221                                                         approvers.add(appr.getApprover());
222                                                 }
223                                         }
224                                 }
225                 } finally {
226                         tt.done();
227                 }
228         
229                         /* Run through all Futures, and see if 
230                          * 1) they have been executed (no longer valid)
231                          * 2) The current Approvals indicate they can proceed 
232                          */
233                         Map<String,Pending> pendingApprs = new HashMap<>();
234                         Map<String,Pending> pendingTemp = new HashMap<>();
235         
236                         String approver;
237                         
238                         tt = trans.start("Analyze Good Tickets",Trans.SUB);
239                         try {
240                                 for(Ticket ticket : goodTickets.values()) {
241                                         try {
242                                                 pendingTemp.clear();
243                                                 switch(ticket.f.target()) {
244                                                         case "user_role":
245                                                                 int state[][] = new int[3][3];
246                                                                 int type;
247                                                                                 
248                                                                 for(Approval appr : ticket.approvals) {
249                                                                         switch(appr.getType()) {
250                                                                                 case "owner":
251                                                                                         type=owner;
252                                                                                         break;
253                                                                                 case "supervisor":
254                                                                                         type=supervisor;
255                                                                                         break;
256                                                                                 default:
257                                                                                         type=0;
258                                                                         }
259                                                                         ++state[type][total]; // count per type
260                                                                         switch(appr.getStatus()) {
261                                                                                 case "pending":
262                                                                                         ++state[type][pending];
263                                                                                         approver = appr.getApprover();
264                                                                                         Pending n = pendingTemp.get(approver);
265                                                                                         if(n==null) {
266                                                                                                 Date lastNotified = ln.lastNotified(approver,"pending",null);
267                                                                                                 pendingTemp.put(approver,new Pending(lastNotified));
268                                                                                         } else {
269                                                                                                 n.inc();
270                                                                                         }
271                                                                                         break;
272                                                                                 case "approved":
273                                                                                         ++state[type][approved];
274                                                                                         break;
275                                                                                 default:
276                                                                                         ++state[type][unknown];
277                                                                         }
278                                                                 }
279                                                                 
280                                                                 // To Approve:
281                                                                 // Always must have at least 1 owner
282                                                                 if((state[owner][total]>0 && state[owner][approved]>0) &&
283                                                                         // If there are no Supervisors, that's ok
284                                                                     (state[supervisor][total]==0 || 
285                                                                     // But if there is a Supervisor, they must have approved 
286                                                                     (state[supervisor][approved]>0))) {
287                                                                                 UserRoleDAO.Data urdd = new UserRoleDAO.Data();
288                                                                                 try {
289                                                                                         urdd.reconstitute(ticket.f.fdd.construct);
290                                                                                         if(urdd.expires.before(ticket.f.expires())) {
291                                                                                                 extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires());
292                                                                                         }
293                                                                                 } catch (IOException e) {
294                                                                                         trans.error().log("Could not reconstitute UserRole");
295                                                                                 }
296                                                                 } else { // Load all the Pending.
297                                                                         for(Entry<String, Pending> es : pendingTemp.entrySet()) {
298                                                                                 Pending p = pendingApprs.get(es.getKey());
299                                                                                 if(p==null) {
300                                                                                         pendingApprs.put(es.getKey(), es.getValue());
301                                                                                 } else {
302                                                                                         p.inc(es.getValue());
303                                                                                 }
304                                                                         }
305                                                                 }
306                                                                 break;
307                                                 }
308                                         } finally {
309                                                 if("user_role".equals(ticket.f.fdd.target)) {
310                                                         String key = ticket.f.fdd.target_key; 
311                                                         if(key!=null) {
312                                                                 mur.put(key, ticket);
313                                                         }
314                                                 }
315                                         }
316                                 }
317                         } finally {
318                                 tt.done();
319                         }
320                         // Good Tickets no longer needed
321                         goodTickets.clear();
322         
323                         /**
324                          * Decide to Notify about Approvals, based on activity/last Notified
325                          */
326                         tt = trans.start("Analyze Approval Reminders", Trans.SUB);
327                         try {
328                                 GregorianCalendar gc = new GregorianCalendar();
329                                 gc.add(GregorianCalendar.DAY_OF_WEEK, 5);
330                                 Date remind = gc.getTime();
331                                 
332                                 for(Entry<String, Pending> es : pendingApprs.entrySet()) {
333                                         Pending p = es.getValue();
334                                         if(p.newApprovals() 
335                                                         || p.earliest() == LastNotified.NEVER // yes, equals. 
336                                                         || p.earliest().after(remind)) {
337                                                 p.row(needApproveCW,es.getKey());
338                                         }
339                                 }
340                         } finally {
341                                 tt.done();
342                         }
343                         
344                         // clear out Approval Intermediates
345                         pendingTemp = null;
346                         pendingApprs = null;
347         } finally {
348         }
349                         
350                 /**
351                    Run through User Roles.  
352                    Owners are treated specially in next section.
353                    Regular roles are checked against Date Ranges.  If match Date Range, write out to appropriate file.
354                 */      
355         
356         try {
357                 Role.load(trans, session);
358         
359                         try {
360                                 tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB);
361                                 Set<String> specialCommented = new HashSet<>();
362                                 Map<String, Set<UserRole>> owners = new TreeMap<>();
363                                 try {
364                                         UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> {
365                                                 Identity identity;
366                                                 try {
367                                                         identity = trans.org().getIdentity(noAvg,ur.user());
368                                                         if(identity==null) {
369                                                                 // Candidate for Delete, but not Users if Special
370                                                                 String id = ur.user();
371                                                                 for(String s : specialDomains) {
372                                                                         if(id.endsWith(s)) {
373                                                                                 if(!specialCommented.contains(id)) {
374                                                                                         deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s);
375                                                                                         specialCommented.add(id);
376                                                                                 }
377                                                                                 return;
378                                                                         }
379                                                                 }
380                                                                 if(specialNames.contains(id)) {
381                                                                         if(!specialCommented.contains(id)) {
382                                                                                 deleteCW.comment("ID %s is a special ID  (UR Org Check)", id);
383                                                                                 specialCommented.add(id);
384                                                                         }
385                                                                         return;
386                                                                 }
387                                                                 ur.row(deleteCW, UserRole.UR,"Not in Organization");
388                                                                 return;
389                                                         } else if(Role.byName.get(ur.role())==null) {
390                                                                 ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role()));
391                                                                 return;
392                                                         }
393                                                         // Just let expired UserRoles sit until deleted
394                                                         if(futureRange.inRange(ur.expires())) {
395                                                                 if(!mur.containsKey(ur.user() + '|' + ur.role())) {
396                                                                         // Cannot just delete owners, unless there is at least one left. Process later
397                                                                         if ("owner".equals(ur.rname())) {
398                                                                                 Set<UserRole> urs = owners.get(ur.role());
399                                                                                 if (urs == null) {
400                                                                                         urs = new HashSet<UserRole>();
401                                                                                         owners.put(ur.role(), urs);
402                                                                                 }
403                                                                                 urs.add(ur);
404                                                                         } else {
405                                                                                 Range r = writeAnalysis(noAvg,ur);
406                                                                                 if(r!=null) {
407                                                                                         Approval existing = findApproval(ur);
408                                                                                         if(existing==null) {
409                                                                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
410                                                                                         }
411                                                                                 }
412                                                                         }
413                                                                 }
414                                                         }
415                                                 } catch (OrganizationException e) {
416                                                         noAvg.error().log(e);
417                                                 }
418                                         });
419                                 } finally {
420                                         tt.done();
421                                 }
422                                 mur.clear();
423                                 
424                                 /**
425                                   Now Process Owners, one owner Role at a time, ensuring one is left,
426                                   preferably a good one. If so, process the others as normal. 
427                                   
428                                   Otherwise, write to ExpiredOwners Report
429                                 */
430                                 tt = trans.start("Analyze Owners Separately",Trans.SUB);
431                                 try {
432                                         if (!owners.values().isEmpty()) {
433                                                 File file = new File(logDir(), EXPIRED_OWNERS + sdate + CSV);
434                                                 final CSV ownerCSV = new CSV(env.access(),file);
435                                                 CSV.Writer expOwner = ownerCSV.writer();
436                                                 expOwner.row(INFO,EXPIRED_OWNERS,sdate,2);
437         
438                                                 try {
439                                                         for (Set<UserRole> sur : owners.values()) {
440                                                                 int goodOwners = 0;
441                                                                 for (UserRole ur : sur) {
442                                                                         if (ur.expires().after(now)) {
443                                                                                 ++goodOwners;
444                                                                         }
445                                                                 }
446                 
447                                                                 for (UserRole ur : sur) {
448                                                                         if (goodOwners >= minOwners) {
449                                                                                 Range r = writeAnalysis(noAvg, ur);
450                                                                                 if(r!=null) {
451                                                                                         Approval existing = findApproval(ur);
452                                                                                         if(existing==null) {
453                                                                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
454                                                                                         }
455                                                                                 }
456                                                                         } else {
457                                                                                 expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
458                                                                                 Approval existing = findApproval(ur);
459                                                                                 if(existing==null) {
460                                                                                         ur.row(needApproveCW,UserRole.APPROVE_UR);
461                                                                                 }
462                                                                         }
463                                                                 }
464                                                         }
465                                                 } finally {
466                                                         if(expOwner!=null) {
467                                                                 expOwner.close();
468                                                         }
469                                                 }
470                                         }
471                                 } finally {
472                                         tt.done();
473                                 }
474                         } finally {
475                                 Role.clear();
476                                 UserRole.clear();
477                         }
478                         
479                         /**
480                          * Check for Expired Credentials
481                          */
482                         try {
483                         // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
484                         Cred.load(trans, session);
485         
486                                 tt = trans.start("Analyze Expired Credentials",Trans.SUB);
487                                 try {
488                                         for (Cred cred : Cred.data.values()) {
489                                         List<Instance> linst = cred.instances;
490                                         if(linst!=null) {
491                                                 Instance lastBath = null;
492                                                 for(Instance inst : linst) {
493                                                         // All Creds go through Life Cycle
494                                                         if(deleteDate!=null && inst.expires.before(deleteDate)) {
495                                                                 writeAnalysis(noAvg, cred, inst); // will go to Delete
496                                                         // Basic Auth has Pre-EOL notifications IF there is no Newer Credential
497                                                         } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
498                                                                 if(lastBath==null || lastBath.expires.before(inst.expires)) {
499                                                                         lastBath = inst;
500                                                                 }
501                                                         }
502                                                 }
503                                                 if(lastBath!=null) {
504                                                         writeAnalysis(noAvg, cred, lastBath);
505                                                 }
506                                         }
507                                         }
508                                 } finally {
509                                         tt.done();
510                                 }
511                         } finally {
512                                 Cred.clear();
513                         }
514         
515                         ////////////////////
516                         tt = trans.start("Analyze Expired X509s",Trans.SUB);
517                         try {
518                                 X509.load(noAvg, session, x509 -> {
519                                         try {
520                                                 for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
521                                                         writeAnalysis(noAvg, x509, (X509Certificate)cert);
522                                                 }
523                                         } catch (CertificateException | IOException e) {
524                                                 noAvg.error().log(e, "Error Decrypting X509");
525                                         }
526                                 });
527                         } finally {
528                                 tt.done();
529                         }
530                 } catch (FileNotFoundException e) {
531                         noAvg.info().log(e);
532                 }
533         }
534  
535         private Approval findApproval(UserRole ur) {
536                 Approval existing = null;
537                 List<Approval> apprs = Approval.byUser.get(ur.user());
538                 if(apprs!=null) {
539                         for(Approval appr : apprs) {
540                                 if(ur.role().equals(appr.getRole()) &&
541                                         appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) {
542                                                 existing = appr; 
543                                 }
544                         }
545                 }
546                 return existing;
547         }
548
549         private Range writeAnalysis(AuthzTrans noAvg, UserRole ur) {
550                 Range r = expireRange.getRange("ur", ur.expires());
551                 if(r!=null) {
552                         Date lnd = ln.lastNotified(LastNotified.newKey(ur));
553                         // Note: lnd is NEVER null
554                         Identity i;
555                         try {
556                                 i = org.getIdentity(noAvg, ur.user());
557                         } catch (OrganizationException e) {
558                                 i=null;
559                         }
560                         if(r.needsContact(lnd,i)) {                             
561                                 CSV.Writer cw = writerList.get(r.name());
562                                 if(cw!=null) {
563                                         ur.row(cw,UserRole.UR);
564                                 }
565                         }
566                 }
567                 return r;
568         }
569     
570     private void writeAnalysis(AuthzTrans noAvg, Cred cred, Instance inst) {
571         if(cred!=null && inst!=null) {
572                         Range r = expireRange.getRange("cred", inst.expires);
573                         if(r!=null) {
574                                 Date lnd = ln.lastNotified(LastNotified.newKey(cred,inst));
575                                 // Note: lnd is NEVER null
576                                 Identity i;
577                                 try {
578                                         i = org.getIdentity(noAvg, cred.id);
579                                 } catch (OrganizationException e) {
580                                         i=null;
581                                 }
582                                 if(r.needsContact(lnd,i)) {                             
583                                         CSV.Writer cw = writerList.get(r.name());
584                                         if(cw!=null) {
585                                                 cred.row(cw,inst);
586                                         }
587                                 }
588                         }
589         }
590         }
591
592     private void writeAnalysis(AuthzTrans noAvg, X509 x509, X509Certificate x509Cert) throws IOException {
593                 Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
594                 if(r!=null) {
595                         Date lnd = ln.lastNotified(LastNotified.newKey(x509,x509Cert));
596                         // Note: lnd is NEVER null
597                         Identity i;
598                         try {
599                                 i = org.getIdentity(noAvg, x509.id);
600                         } catch (OrganizationException e) {
601                                 i=null;
602                         }
603                         if(r.needsContact(lnd,i)) {
604                                 CSV.Writer cw = writerList.get(r.name());
605                                 if(cw!=null) {
606                                         x509.row(cw,x509Cert);
607                                 }
608                         }
609                 }
610         }
611     
612     @Override
613     protected void _close(AuthzTrans trans) {
614         session.close();
615         for(CSV.Writer cw : writerList.values()) {
616                 cw.close();
617         }
618     }
619
620 }