Minor changes for Organization needs
[aaf/authz.git] / auth / auth-batch / src / main / java / org / onap / aaf / auth / batch / reports / Analyze.java
1 /**
2  * ============LICENSE_START====================================================
3  * org.onap.aaf
4  * ===========================================================================
5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
6  *
7  * Modifications Copyright (C) 2019 IBM.
8  * ===========================================================================
9  * Licensed under the Apache License, Version 2.0 (the "License");
10  * you may not use this file except in compliance with the License.
11  * You may obtain a copy of the License at
12  *
13  *      http://www.apache.org/licenses/LICENSE-2.0
14  *
15  * Unless required by applicable law or agreed to in writing, software
16  * distributed under the License is distributed on an "AS IS" BASIS,
17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18  * See the License for the specific language governing permissions and
19  * limitations under the License.
20  * ============LICENSE_END====================================================
21  *
22  */
23
24 package org.onap.aaf.auth.batch.reports;
25
26 import java.io.File;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.security.cert.Certificate;
30 import java.security.cert.CertificateException;
31 import java.security.cert.X509Certificate;
32 import java.util.Date;
33 import java.util.GregorianCalendar;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.TreeMap;
41 import java.util.TreeSet;
42 import java.util.UUID;
43
44 import org.onap.aaf.auth.batch.Batch;
45 import org.onap.aaf.auth.batch.approvalsets.Pending;
46 import org.onap.aaf.auth.batch.approvalsets.Ticket;
47 import org.onap.aaf.auth.batch.helpers.Approval;
48 import org.onap.aaf.auth.batch.helpers.Cred;
49 import org.onap.aaf.auth.batch.helpers.Cred.Instance;
50 import org.onap.aaf.auth.batch.helpers.ExpireRange;
51 import org.onap.aaf.auth.batch.helpers.ExpireRange.Range;
52 import org.onap.aaf.auth.batch.helpers.Future;
53 import org.onap.aaf.auth.batch.helpers.LastNotified;
54 import org.onap.aaf.auth.batch.helpers.Role;
55 import org.onap.aaf.auth.batch.helpers.UserRole;
56 import org.onap.aaf.auth.batch.helpers.X509;
57 import org.onap.aaf.auth.dao.cass.CredDAO;
58 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
59 import org.onap.aaf.auth.env.AuthzTrans;
60 import org.onap.aaf.auth.org.Organization.Expiration;
61 import org.onap.aaf.auth.org.Organization.Identity;
62 import org.onap.aaf.auth.org.OrganizationException;
63 import org.onap.aaf.cadi.configure.Factory;
64 import org.onap.aaf.cadi.util.CSV;
65 import org.onap.aaf.misc.env.APIException;
66 import org.onap.aaf.misc.env.Env;
67 import org.onap.aaf.misc.env.TimeTaken;
68 import org.onap.aaf.misc.env.Trans;
69 import org.onap.aaf.misc.env.util.Chrono;
70
71
72 public class Analyze extends Batch {
73         private static final int unknown=0;
74     private static final int owner=1;
75     private static final int supervisor=2;
76     private static final int total=0;
77     private static final int pending=1;
78     private static final int approved=2;
79
80
81     public static final String NEED_APPROVALS = "NeedApprovals";
82     private static final String EXTEND = "Extend";
83     private static final String EXPIRED_OWNERS = "ExpiredOwners";
84     private static final String CSV = ".csv";
85     private static final String INFO = "info";
86     private static final String NOT_COMPLIANT = "NotCompliant";
87     private int minOwners;
88     private Map<String, CSV.Writer> writerList;
89     private ExpireRange expireRange;
90     private Date deleteDate;
91     private CSV.Writer deleteCW;
92     private CSV.Writer needApproveCW;
93     private CSV.Writer extendCW;
94     private CSV.Writer notCompliantCW;
95     private Range futureRange;
96     private final String sdate;
97     private LastNotified ln;
98
99     public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException {
100         super(trans.env());
101         trans.info().log("Starting Connection Process");
102
103         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
104         try {
105             TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
106             try {
107                 session = cluster.connect();
108             } finally {
109                 tt.done();
110             }
111
112
113             minOwners=1;
114
115             // Create Intermediate Output
116             writerList = new HashMap<>();
117
118             expireRange = new ExpireRange(trans.env().access());
119             sdate = Chrono.dateOnlyStamp(now);
120             for( List<Range> lr : expireRange.ranges.values()) {
121                 for(Range r : lr ) {
122                     if(writerList.get(r.name())==null) {
123                         File file = new File(logDir(),r.name() + sdate +CSV);
124                         CSV csv = new CSV(env.access(),file);
125                         CSV.Writer cw = csv.writer(false);
126                         cw.row(INFO,r.name(),sdate,r.reportingLevel());
127                         writerList.put(r.name(),cw);
128                         if("Delete".equals(r.name())) {
129                             deleteDate = r.getEnd();
130                             deleteCW = cw;
131                         }
132                         trans.init().log("Creating File:",file.getAbsolutePath());
133                     }
134                 }
135             }
136
137             // Setup New Approvals file
138             futureRange = expireRange.newFutureRange();
139             File file = new File(logDir(),NEED_APPROVALS + sdate +CSV);
140             CSV approveCSV = new CSV(env.access(),file);
141             needApproveCW = approveCSV.writer();
142             needApproveCW.row(INFO,NEED_APPROVALS,sdate,1);
143             writerList.put(NEED_APPROVALS,needApproveCW);
144
145             // Setup Extend Approvals file
146             file = new File(logDir(),EXTEND + sdate +CSV);
147             CSV extendCSV = new CSV(env.access(),file);
148             extendCW = extendCSV.writer();
149             extendCW.row(INFO,EXTEND,sdate,1);
150             writerList.put(EXTEND,extendCW);
151
152             // Setup NotCompliant Writer for Apps
153             file = new File(logDir(),NOT_COMPLIANT + sdate + CSV);
154             CSV ncCSV = new CSV(env.access(),file);
155             notCompliantCW = ncCSV.writer();
156             writerList.put(NOT_COMPLIANT, notCompliantCW);
157             
158             // Load full data of the following
159             ln = new LastNotified(session);
160
161         } finally {
162             tt0.done();
163         }
164     }
165
166     @Override
167     protected void run(AuthzTrans trans) {
168         TimeTaken tt;
169         AuthzTrans noAvg = trans.env().newTransNoAvg();
170
171         ////////////////////
172         // Load all Notifieds, and either add to local Data, or mark for Deletion.
173         ln.loadAll(noAvg,expireRange.approveDelete,deleteCW);
174
175         // Hold Good Tickets to keyed User/Role for UserRole Step
176         Map<String,Ticket> mur = new TreeMap<>();
177
178         try {
179             Approval.load(trans, session, Approval.v2_0_17);
180
181             ////////////////////
182             final Map<UUID,Ticket> goodTickets = new TreeMap<>();
183             tt = trans.start("Analyze Expired Futures",Trans.SUB);
184             try {
185                 Future.load(noAvg, session, Future.withConstruct, fut -> {
186                     List<Approval> appls = Approval.byTicket.get(fut.id());
187                     if(!futureRange.inRange(fut.expires())) {
188                         deleteCW.comment("Future %s expired", fut.id());
189                         Future.row(deleteCW,fut);
190                         if(appls!=null) {
191                             for(Approval a : appls) {
192                                 Approval.row(deleteCW, a);
193                             }
194                         }
195                     } else if(appls==null) { // Orphaned Future (no Approvals)
196                         deleteCW.comment("Future is Orphaned");
197                         Future.row(deleteCW,fut);
198                     } else  {
199                         goodTickets.put(fut.fdd.id, new Ticket(fut));
200                     }
201                 });
202             } finally {
203                 tt.done();
204             }
205
206             Set<String> approvers = new TreeSet<>();
207             tt = trans.start("Connect Approvals with Futures",Trans.SUB);
208             try {
209                 for(Approval appr : Approval.list) {
210                     Ticket ticket=null;
211                     UUID ticketID = appr.getTicket();
212                     if(ticketID!=null) {
213                         ticket = goodTickets.get(appr.getTicket());
214                     }
215                     if(ticket == null) { // Orphaned Approvals, no Futures
216                         deleteCW.comment("Approval is Orphaned");
217                         Approval.row(deleteCW, appr);
218                     } else {
219                         // for users and approvers still valid
220                         String user = appr.getUser();
221
222                         if(org.isRevoked(noAvg, appr.getApprover())) {
223                             deleteCW.comment("Approver ID is revoked");
224                             Approval.row(deleteCW, appr);
225                         } else if(user!=null && !user.isEmpty() && org.isRevoked(noAvg, user)) {
226                             deleteCW.comment("USER ID is revoked");
227                             Approval.row(deleteCW, appr);
228                         } else {
229                             ticket.approvals.add(appr); // add to found Ticket
230                             approvers.add(appr.getApprover());
231                         }
232                     }
233                 }
234             } finally {
235                 tt.done();
236             }
237
238             /* Run through all Futures, and see if
239              * 1) they have been executed (no longer valid)
240              * 2) The current Approvals indicate they can proceed
241              */
242             Map<String,Pending> pendingApprs = new HashMap<>();
243             Map<String,Pending> pendingTemp = new HashMap<>();
244
245             String approver;
246
247             tt = trans.start("Analyze Good Tickets",Trans.SUB);
248             try {
249                 for(Ticket ticket : goodTickets.values()) {
250                     try {
251                         pendingTemp.clear();
252                         switch(ticket.f.target()) {
253                             case "user_role":
254                                 int state[][] = new int[3][3];
255                                 int type;
256
257                                 for(Approval appr : ticket.approvals) {
258                                     switch(appr.getType()) {
259                                         case "owner":
260                                             type=owner;
261                                             break;
262                                         case "supervisor":
263                                             type=supervisor;
264                                             break;
265                                         default:
266                                             type=0;
267                                     }
268                                     ++state[type][total]; // count per type
269                                     switch(appr.getStatus()) {
270                                         case "pending":
271                                             ++state[type][pending];
272                                             approver = appr.getApprover();
273                                             Pending n = pendingTemp.get(approver);
274                                             if(n==null) {
275                                                 Date lastNotified = ln.lastNotified(approver,"pending",null);
276                                                 pendingTemp.put(approver,new Pending(lastNotified));
277                                             } else {
278                                                 n.inc();
279                                             }
280                                             break;
281                                         case "approved":
282                                             ++state[type][approved];
283                                             break;
284                                         default:
285                                             ++state[type][unknown];
286                                     }
287                                 }
288
289                                 // To Approve:
290                                 // Always must have at least 1 owner
291                                 if((state[owner][total]>0 && state[owner][approved]>0) &&
292                                     // If there are no Supervisors, that's ok
293                                     (state[supervisor][total]==0 ||
294                                     // But if there is a Supervisor, they must have approved
295                                     (state[supervisor][approved]>0))) {
296                                         UserRoleDAO.Data urdd = new UserRoleDAO.Data();
297                                         try {
298                                             urdd.reconstitute(ticket.f.fdd.construct);
299                                             if(urdd.expires.before(ticket.f.expires())) {
300                                                 extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires());
301                                             }
302                                         } catch (IOException e) {
303                                             trans.error().log("Could not reconstitute UserRole");
304                                         }
305                                 } else { // Load all the Pending.
306                                     for(Entry<String, Pending> es : pendingTemp.entrySet()) {
307                                         Pending p = pendingApprs.get(es.getKey());
308                                         if(p==null) {
309                                             pendingApprs.put(es.getKey(), es.getValue());
310                                         } else {
311                                             p.inc(es.getValue());
312                                         }
313                                     }
314                                 }
315                                 break;
316                         }
317                     } finally {
318                         if("user_role".equals(ticket.f.fdd.target)) {
319                             String key = ticket.f.fdd.target_key;
320                             if(key!=null) {
321                                 mur.put(key, ticket);
322                             }
323                         }
324                     }
325                 }
326             } finally {
327                 tt.done();
328             }
329             // Good Tickets no longer needed
330             goodTickets.clear();
331
332             /**
333              * Decide to Notify about Approvals, based on activity/last Notified
334              */
335             tt = trans.start("Analyze Approval Reminders", Trans.SUB);
336             try {
337                 GregorianCalendar gc = new GregorianCalendar();
338                 gc.add(GregorianCalendar.DAY_OF_WEEK, 5);
339                 Date remind = gc.getTime();
340
341                 for(Entry<String, Pending> es : pendingApprs.entrySet()) {
342                     Pending p = es.getValue();
343                     if(p.newApprovals()
344                             || p.earliest() == LastNotified.NEVER // yes, equals.
345                             || p.earliest().after(remind)) {
346                         p.row(needApproveCW,es.getKey());
347                     }
348                 }
349             } finally {
350                 tt.done();
351             }
352
353             // clear out Approval Intermediates
354             pendingTemp = null;
355             pendingApprs = null;
356         } finally {
357         }
358
359         /**
360            Run through User Roles.
361            Owners are treated specially in next section.
362            Regular roles are checked against Date Ranges.  If match Date Range, write out to appropriate file.
363         */
364
365         try {
366             Role.load(trans, session);
367
368             try {
369                 tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB);
370                 Set<String> specialCommented = new HashSet<>();
371                 Map<String, Set<UserRole>> owners = new TreeMap<>();
372                  try {
373                     UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> {
374                         Identity identity;
375                         try {
376                             identity = trans.org().getIdentity(noAvg,ur.user());
377                             if(identity==null) {
378                                 // Candidate for Delete, but not Users if Special
379                                 String id = ur.user();
380                                 for(String s : specialDomains) {
381                                     if(id.endsWith(s)) {
382                                         if(!specialCommented.contains(id)) {
383                                             deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s);
384                                             specialCommented.add(id);
385                                         }
386                                         return;
387                                     }
388                                 }
389                                 if(specialNames.contains(id)) {
390                                     if(!specialCommented.contains(id)) {
391                                         deleteCW.comment("ID %s is a special ID  (UR Org Check)", id);
392                                         specialCommented.add(id);
393                                     }
394                                     return;
395                                 }
396                                 if(org.isRevoked(trans, ur.user())) {
397                                         GregorianCalendar gc = new GregorianCalendar();
398                                         gc.setTime(ur.expires());
399                                         GregorianCalendar gracePeriodEnds = org.expiration(gc, Expiration.RevokedGracePeriodEnds, ur.user());
400                                         if(now.after(gracePeriodEnds.getTime())) {
401                                         ur.row(deleteCW, UserRole.UR,"Revoked ID, no grace period left");
402                                         } else {
403                                                 ur.row(notCompliantCW, UserRole.UR, "Revoked ID: WARNING! GracePeriod Ends " + gracePeriodEnds.toString());
404                                         }
405                                         return;
406                                 }
407                                 ur.row(deleteCW, UserRole.UR,"Not in Organization");
408                                 return;
409                             } else if(Role.byName.get(ur.role())==null) {
410                                 ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role()));
411                                 return;
412                             // Make sure owners can still be owners.
413                             } else if(ur.role().endsWith(".owner")) {
414                                 String err = identity.mayOwn(); 
415                                 if(err!=null) {
416                                         ur.row(deleteCW, UserRole.UR,String.format("%s may not be an owner: %s",ur.user(),err));
417                                         return;
418                                 }
419                             }
420                             
421                             
422                             
423                             // Just let expired UserRoles sit until deleted
424                             if(futureRange.inRange(ur.expires())&&(!mur.containsKey(ur.user() + '|' + ur.role()))) {
425                                     // Cannot just delete owners, unless there is at least one left. Process later
426                                     if ("owner".equals(ur.rname())) {
427                                         Set<UserRole> urs = owners.get(ur.role());
428                                         if (urs == null) {
429                                             urs = new HashSet<UserRole>();
430                                             owners.put(ur.role(), urs);
431                                         }
432                                         urs.add(ur);
433                                     } else {
434                                         Range r = writeAnalysis(noAvg,ur);
435                                         if(r!=null) {
436                                             Approval existing = findApproval(ur);
437                                             if(existing==null) {
438                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
439                                             }
440                                         }
441                                     }
442                              }
443                         } catch (OrganizationException e) {
444                             noAvg.error().log(e);
445                         }
446                     });
447                  } finally {
448                      tt.done();
449                  }
450                  mur.clear();
451
452                 /**
453                   Now Process Owners, one owner Role at a time, ensuring one is left,
454                   preferably a good one. If so, process the others as normal.
455
456                   Otherwise, write to ExpiredOwners Report
457                 */
458                  tt = trans.start("Analyze Owners Separately",Trans.SUB);
459                  try {
460                     if (!owners.values().isEmpty()) {
461                         File file = new File(logDir(), EXPIRED_OWNERS + sdate + CSV);
462                         final CSV ownerCSV = new CSV(env.access(),file);
463                         CSV.Writer expOwner = ownerCSV.writer();
464                         expOwner.row(INFO,EXPIRED_OWNERS,sdate,2);
465
466                         try {
467                             for (Set<UserRole> sur : owners.values()) {
468                                 int goodOwners = 0;
469                                 for (UserRole ur : sur) {
470                                     if (ur.expires().after(now)) {
471                                         ++goodOwners;
472                                     }
473                                 }
474
475                                 for (UserRole ur : sur) {
476                                     if (goodOwners >= minOwners) {
477                                         Range r = writeAnalysis(noAvg, ur);
478                                         if(r!=null) {
479                                             Approval existing = findApproval(ur);
480                                             if(existing==null) {
481                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
482                                             }
483                                         }
484                                     } else {
485                                         expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
486                                         Approval existing = findApproval(ur);
487                                         if(existing==null) {
488                                             ur.row(needApproveCW,UserRole.APPROVE_UR);
489                                         }
490                                     }
491                                 }
492                             }
493                         } finally {
494                             if(expOwner!=null) {
495                                 expOwner.close();
496                             }
497                         }
498                     }
499                  } finally {
500                      tt.done();
501                  }
502             } finally {
503                 Role.clear();
504                 UserRole.clear();
505             }
506
507             /**
508              * Check for Expired Credentials
509              */
510             try {
511                  // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
512                  Cred.load(trans, session);
513
514                 tt = trans.start("Analyze Expired Credentials",Trans.SUB);
515                 try {
516                     for (Cred cred : Cred.data.values()) {
517                         List<Instance> linst = cred.instances;
518                         if(linst!=null) {
519                             Instance lastBath = null;
520                             for(Instance inst : linst) {
521                                 // All Creds go through Life Cycle
522                                 if(deleteDate!=null && inst.expires.before(deleteDate)) {
523                                     writeAnalysis(noAvg, cred, inst); // will go to Delete
524                                 // Basic Auth has Pre-EOL notifications IF there is no Newer Credential
525                                 } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
526                                     if(lastBath==null || lastBath.expires.before(inst.expires)) {
527                                         lastBath = inst;
528                                     }
529                                 }
530                             }
531                             if(lastBath!=null) {
532                                 writeAnalysis(noAvg, cred, lastBath);
533                             }
534                         }
535                     }
536                 } finally {
537                     tt.done();
538                 }
539             } finally {
540                 Cred.clear();
541             }
542
543             ////////////////////
544             tt = trans.start("Analyze Expired X509s",Trans.SUB);
545             try {
546                 X509.load(noAvg, session, x509 -> {
547                     try {
548                         for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
549                             writeAnalysis(noAvg, x509, (X509Certificate)cert);
550                         }
551                     } catch (CertificateException | IOException e) {
552                         noAvg.error().log(e, "Error Decrypting X509");
553                     }
554                 });
555             } finally {
556                 tt.done();
557             }
558         } catch (FileNotFoundException e) {
559             noAvg.info().log(e);
560         }
561     }
562
563     private Approval findApproval(UserRole ur) {
564         Approval existing = null;
565         List<Approval> apprs = Approval.byUser.get(ur.user());
566         if(apprs!=null) {
567             for(Approval appr : apprs) {
568                 if(ur.role().equals(appr.getRole()) &&
569                     appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) {
570                         existing = appr;
571                 }
572             }
573         }
574         return existing;
575     }
576
577     private Range writeAnalysis(AuthzTrans noAvg, UserRole ur) {
578         Range r = expireRange.getRange("ur", ur.expires());
579         if(r!=null) {
580             Date lnd = ln.lastNotified(LastNotified.newKey(ur));
581             // Note: lnd is NEVER null
582             Identity i;
583             try {
584                 i = org.getIdentity(noAvg, ur.user());
585             } catch (OrganizationException e) {
586                 i=null;
587             }
588             if(r.needsContact(lnd,i)) {
589                 CSV.Writer cw = writerList.get(r.name());
590                 if(cw!=null) {
591                     ur.row(cw,UserRole.UR);
592                 }
593             }
594         }
595         return r;
596     }
597
598     private void writeAnalysis(AuthzTrans noAvg, Cred cred, Instance inst) {
599         if(cred!=null && inst!=null) {
600             Range r = expireRange.getRange("cred", inst.expires);
601             if(r!=null) {
602                 Date lnd = ln.lastNotified(LastNotified.newKey(cred,inst));
603                 // Note: lnd is NEVER null
604                 Identity i;
605                 try {
606                     i = org.getIdentity(noAvg, cred.id);
607                 } catch (OrganizationException e) {
608                     i=null;
609                 }
610                 if(r.needsContact(lnd,i)) {
611                     CSV.Writer cw = writerList.get(r.name());
612                     if(cw!=null) {
613                         cred.row(cw,inst);
614                     }
615                 }
616             }
617         }
618     }
619
620     private void writeAnalysis(AuthzTrans noAvg, X509 x509, X509Certificate x509Cert) throws IOException {
621         Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
622         if(r!=null) {
623             Date lnd = ln.lastNotified(LastNotified.newKey(x509,x509Cert));
624             // Note: lnd is NEVER null
625             Identity i;
626             try {
627                 i = org.getIdentity(noAvg, x509.id);
628             } catch (OrganizationException e) {
629                 i=null;
630             }
631             if(r.needsContact(lnd,i)) {
632                 CSV.Writer cw = writerList.get(r.name());
633                 if(cw!=null) {
634                     x509.row(cw,x509Cert);
635                 }
636             }
637         }
638     }
639
640     @Override
641     protected void _close(AuthzTrans trans) {
642         session.close();
643         for(CSV.Writer cw : writerList.values()) {
644             cw.close();
645         }
646     }
647
648 }