Update DCAE Startup Info
[aaf/authz.git] / auth / auth-batch / src / main / java / org / onap / aaf / auth / batch / reports / Analyze.java
1 /**
2  * ============LICENSE_START====================================================
3  * org.onap.aaf
4  * ===========================================================================
5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
6  *
7  * Modifications Copyright (C) 2019 IBM.
8  * ===========================================================================
9  * Licensed under the Apache License, Version 2.0 (the "License");
10  * you may not use this file except in compliance with the License.
11  * You may obtain a copy of the License at
12  *
13  *      http://www.apache.org/licenses/LICENSE-2.0
14  *
15  * Unless required by applicable law or agreed to in writing, software
16  * distributed under the License is distributed on an "AS IS" BASIS,
17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18  * See the License for the specific language governing permissions and
19  * limitations under the License.
20  * ============LICENSE_END====================================================
21  *
22  */
23
24 package org.onap.aaf.auth.batch.reports;
25
26 import java.io.File;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.security.cert.Certificate;
30 import java.security.cert.CertificateException;
31 import java.security.cert.X509Certificate;
32 import java.util.Date;
33 import java.util.GregorianCalendar;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.TreeMap;
41 import java.util.TreeSet;
42 import java.util.UUID;
43
44 import org.onap.aaf.auth.batch.Batch;
45 import org.onap.aaf.auth.batch.approvalsets.Pending;
46 import org.onap.aaf.auth.batch.approvalsets.Ticket;
47 import org.onap.aaf.auth.batch.helpers.Approval;
48 import org.onap.aaf.auth.batch.helpers.Cred;
49 import org.onap.aaf.auth.batch.helpers.Cred.Instance;
50 import org.onap.aaf.auth.batch.helpers.ExpireRange;
51 import org.onap.aaf.auth.batch.helpers.ExpireRange.Range;
52 import org.onap.aaf.auth.batch.helpers.Future;
53 import org.onap.aaf.auth.batch.helpers.LastNotified;
54 import org.onap.aaf.auth.batch.helpers.Role;
55 import org.onap.aaf.auth.batch.helpers.UserRole;
56 import org.onap.aaf.auth.batch.helpers.X509;
57 import org.onap.aaf.auth.dao.cass.CredDAO;
58 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
59 import org.onap.aaf.auth.env.AuthzTrans;
60 import org.onap.aaf.auth.org.Organization.Expiration;
61 import org.onap.aaf.auth.org.Organization.Identity;
62 import org.onap.aaf.auth.org.OrganizationException;
63 import org.onap.aaf.cadi.configure.Factory;
64 import org.onap.aaf.cadi.util.CSV;
65 import org.onap.aaf.misc.env.APIException;
66 import org.onap.aaf.misc.env.Env;
67 import org.onap.aaf.misc.env.TimeTaken;
68 import org.onap.aaf.misc.env.Trans;
69 import org.onap.aaf.misc.env.util.Chrono;
70
71
72 public class Analyze extends Batch {
73         private static final int unknown=0;
74     private static final int owner=1;
75     private static final int supervisor=2;
76     private static final int total=0;
77     private static final int pending=1;
78     private static final int approved=2;
79
80
81     public static final String NEED_APPROVALS = "NeedApprovals";
82     private static final String EXTEND = "Extend";
83     private static final String EXPIRED_OWNERS = "ExpiredOwners";
84     private static final String CSV = ".csv";
85     private static final String INFO = "info";
86     private static final String NOT_COMPLIANT = "NotCompliant";
87     private int minOwners;
88     private Map<String, CSV.Writer> writerList;
89     private ExpireRange expireRange;
90     private Date deleteDate;
91     private CSV.Writer deleteCW;
92     private CSV.Writer needApproveCW;
93     private CSV.Writer extendCW;
94     private CSV.Writer notCompliantCW;
95     private Range futureRange;
96     private final String sdate;
97     private LastNotified ln;
98
99     public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException {
100         super(trans.env());
101         trans.info().log("Starting Connection Process");
102
103         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
104         try {
105             TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
106             try {
107                 session = cluster.connect();
108             } finally {
109                 tt.done();
110             }
111
112
113             minOwners=1;
114
115             // Create Intermediate Output
116             writerList = new HashMap<>();
117
118             expireRange = new ExpireRange(trans.env().access());
119             sdate = Chrono.dateOnlyStamp(now);
120             for( List<Range> lr : expireRange.ranges.values()) {
121                 for(Range r : lr ) {
122                     if(writerList.get(r.name())==null) {
123                         File file = new File(logDir(),r.name() + sdate +CSV);
124                         CSV csv = new CSV(env.access(),file);
125                         CSV.Writer cw = csv.writer(false);
126                         cw.row(INFO,r.name(),sdate,r.reportingLevel());
127                         writerList.put(r.name(),cw);
128                         if("Delete".equals(r.name())) {
129                             deleteDate = r.getEnd();
130                             deleteCW = cw;
131                         }
132                         trans.init().log("Creating File:",file.getAbsolutePath());
133                     }
134                 }
135             }
136
137             // Setup New Approvals file
138             futureRange = expireRange.newFutureRange();
139             File file = new File(logDir(),NEED_APPROVALS + sdate +CSV);
140             CSV approveCSV = new CSV(env.access(),file);
141             needApproveCW = approveCSV.writer();
142             needApproveCW.row(INFO,NEED_APPROVALS,sdate,1);
143             writerList.put(NEED_APPROVALS,needApproveCW);
144
145             // Setup Extend Approvals file
146             file = new File(logDir(),EXTEND + sdate +CSV);
147             CSV extendCSV = new CSV(env.access(),file);
148             extendCW = extendCSV.writer();
149             extendCW.row(INFO,EXTEND,sdate,1);
150             writerList.put(EXTEND,extendCW);
151
152             // Setup NotCompliant Writer for Apps
153             file = new File(logDir(),NOT_COMPLIANT + sdate + CSV);
154             CSV ncCSV = new CSV(env.access(),file);
155             notCompliantCW = ncCSV.writer();
156             writerList.put(NOT_COMPLIANT, notCompliantCW);
157             
158             // Load full data of the following
159             ln = new LastNotified(session);
160
161         } finally {
162             tt0.done();
163         }
164     }
165
166     @Override
167     protected void run(AuthzTrans trans) {
168         TimeTaken tt;
169         AuthzTrans noAvg = trans.env().newTransNoAvg();
170
171         ////////////////////
172         // Load all Notifieds, and either add to local Data, or mark for Deletion.
173         ln.loadAll(noAvg,expireRange.approveDelete,deleteCW);
174
175         // Hold Good Tickets to keyed User/Role for UserRole Step
176         Map<String,Ticket> mur = new TreeMap<>();
177
178         try {
179             Approval.load(trans, session, Approval.v2_0_17);
180
181             ////////////////////
182             final Map<UUID,Ticket> goodTickets = new TreeMap<>();
183             tt = trans.start("Analyze Expired Futures",Trans.SUB);
184             try {
185                 Future.load(noAvg, session, Future.withConstruct, fut -> {
186                     List<Approval> appls = Approval.byTicket.get(fut.id());
187                     if(!futureRange.inRange(fut.expires())) {
188                         deleteCW.comment("Future %s expired", fut.id());
189                         Future.row(deleteCW,fut);
190                         if(appls!=null) {
191                             for(Approval a : appls) {
192                                 Approval.row(deleteCW, a);
193                             }
194                         }
195                     } else if(appls==null) { // Orphaned Future (no Approvals)
196                         deleteCW.comment("Future is Orphaned");
197                         Future.row(deleteCW,fut);
198                     } else  {
199                         goodTickets.put(fut.fdd.id, new Ticket(fut));
200                     }
201                 });
202             } finally {
203                 tt.done();
204             }
205
206             Set<String> approvers = new TreeSet<>();
207             tt = trans.start("Connect Approvals with Futures",Trans.SUB);
208             try {
209                 for(Approval appr : Approval.list) {
210                     Ticket ticket=null;
211                     UUID ticketID = appr.getTicket();
212                     if(ticketID!=null) {
213                         ticket = goodTickets.get(appr.getTicket());
214                     }
215                     if(ticket == null) { // Orphaned Approvals, no Futures
216                         deleteCW.comment("Approval is Orphaned");
217                         Approval.row(deleteCW, appr);
218                     } else {
219                         // for users and approvers still valid
220                         String user = appr.getUser();
221
222                         Date revokedAppr = org.isRevoked(noAvg, appr.getApprover());
223                         Date revokedUser = org.isRevoked(noAvg, user);
224                         if(revokedAppr!=null) {
225                             deleteCW.comment("Approver ID is revoked on " + revokedAppr);
226                             Approval.row(deleteCW, appr);
227                         } else if(user!=null && !user.isEmpty() && revokedUser!=null) {
228                             deleteCW.comment("USER ID is revoked on " + revokedUser);
229                             Approval.row(deleteCW, appr);
230                         } else {
231                             ticket.approvals.add(appr); // add to found Ticket
232                             approvers.add(appr.getApprover());
233                         }
234                     }
235                 }
236             } finally {
237                 tt.done();
238             }
239
240             /* Run through all Futures, and see if
241              * 1) they have been executed (no longer valid)
242              * 2) The current Approvals indicate they can proceed
243              */
244             Map<String,Pending> pendingApprs = new HashMap<>();
245             Map<String,Pending> pendingTemp = new HashMap<>();
246
247             String approver;
248
249             tt = trans.start("Analyze Good Tickets",Trans.SUB);
250             try {
251                 for(Ticket ticket : goodTickets.values()) {
252                     try {
253                         pendingTemp.clear();
254                         switch(ticket.f.target()) {
255                             case "user_role":
256                                 int state[][] = new int[3][3];
257                                 int type;
258
259                                 for(Approval appr : ticket.approvals) {
260                                     switch(appr.getType()) {
261                                         case "owner":
262                                             type=owner;
263                                             break;
264                                         case "supervisor":
265                                             type=supervisor;
266                                             break;
267                                         default:
268                                             type=0;
269                                     }
270                                     ++state[type][total]; // count per type
271                                     switch(appr.getStatus()) {
272                                         case "pending":
273                                             ++state[type][pending];
274                                             approver = appr.getApprover();
275                                             Pending n = pendingTemp.get(approver);
276                                             if(n==null) {
277                                                 Date lastNotified = ln.lastNotified(approver,"pending",null);
278                                                 pendingTemp.put(approver,new Pending(lastNotified));
279                                             } else {
280                                                 n.inc();
281                                             }
282                                             break;
283                                         case "approved":
284                                             ++state[type][approved];
285                                             break;
286                                         default:
287                                             ++state[type][unknown];
288                                     }
289                                 }
290
291                                 // To Approve:
292                                 // Always must have at least 1 owner
293                                 if((state[owner][total]>0 && state[owner][approved]>0) &&
294                                     // If there are no Supervisors, that's ok
295                                     (state[supervisor][total]==0 ||
296                                     // But if there is a Supervisor, they must have approved
297                                     (state[supervisor][approved]>0))) {
298                                         UserRoleDAO.Data urdd = new UserRoleDAO.Data();
299                                         try {
300                                             urdd.reconstitute(ticket.f.fdd.construct);
301                                             if(urdd.expires.before(ticket.f.expires())) {
302                                                 extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires());
303                                             }
304                                         } catch (IOException e) {
305                                             trans.error().log("Could not reconstitute UserRole");
306                                         }
307                                 } else { // Load all the Pending.
308                                     for(Entry<String, Pending> es : pendingTemp.entrySet()) {
309                                         Pending p = pendingApprs.get(es.getKey());
310                                         if(p==null) {
311                                             pendingApprs.put(es.getKey(), es.getValue());
312                                         } else {
313                                             p.inc(es.getValue());
314                                         }
315                                     }
316                                 }
317                                 break;
318                         }
319                     } finally {
320                         if("user_role".equals(ticket.f.fdd.target)) {
321                             String key = ticket.f.fdd.target_key;
322                             if(key!=null) {
323                                 mur.put(key, ticket);
324                             }
325                         }
326                     }
327                 }
328             } finally {
329                 tt.done();
330             }
331             // Good Tickets no longer needed
332             goodTickets.clear();
333
334             /**
335              * Decide to Notify about Approvals, based on activity/last Notified
336              */
337             tt = trans.start("Analyze Approval Reminders", Trans.SUB);
338             try {
339                 GregorianCalendar gc = new GregorianCalendar();
340                 gc.add(GregorianCalendar.DAY_OF_WEEK, 5);
341                 Date remind = gc.getTime();
342
343                 for(Entry<String, Pending> es : pendingApprs.entrySet()) {
344                     Pending p = es.getValue();
345                     if(p.newApprovals()
346                             || p.earliest() == LastNotified.NEVER // yes, equals.
347                             || p.earliest().after(remind)) {
348                         p.row(needApproveCW,es.getKey());
349                     }
350                 }
351             } finally {
352                 tt.done();
353             }
354
355             // clear out Approval Intermediates
356             pendingTemp = null;
357             pendingApprs = null;
358         } finally {
359         }
360
361         /**
362            Run through User Roles.
363            Owners are treated specially in next section.
364            Regular roles are checked against Date Ranges.  If match Date Range, write out to appropriate file.
365         */
366
367         try {
368             Role.load(trans, session);
369
370             try {
371                 tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB);
372                 Set<String> specialCommented = new HashSet<>();
373                 Map<String, Set<UserRole>> owners = new TreeMap<>();
374                  try {
375                     UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> {
376                         Identity identity;
377                         try {
378                             identity = trans.org().getIdentity(noAvg,ur.user());
379                             if(identity==null) {
380                                 // Candidate for Delete, but not Users if Special
381                                 String id = ur.user();
382                                 for(String s : specialDomains) {
383                                     if(id.endsWith(s)) {
384                                         if(!specialCommented.contains(id)) {
385                                             deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s);
386                                             specialCommented.add(id);
387                                         }
388                                         return;
389                                     }
390                                 }
391                                 if(specialNames.contains(id)) {
392                                     if(!specialCommented.contains(id)) {
393                                         deleteCW.comment("ID %s is a special ID  (UR Org Check)", id);
394                                         specialCommented.add(id);
395                                     }
396                                     return;
397                                 }
398                                 Date revoked = org.isRevoked(trans, ur.user());
399                                 if(revoked!=null) {
400                                         GregorianCalendar gc = new GregorianCalendar();
401                                         gc.setTime(revoked);
402                                         GregorianCalendar gracePeriodEnds = org.expiration(gc, Expiration.RevokedGracePeriodEnds, ur.user());
403                                         if(now.after(gracePeriodEnds.getTime())) {
404                                         ur.row(deleteCW, UserRole.UR,"Revoked ID, no grace period left");
405                                         } else {
406                                                 ur.row(notCompliantCW, UserRole.UR, "Revoked ID: WARNING! GracePeriod Ends " + Chrono.dateOnlyStamp(gracePeriodEnds));
407                                         }
408                                         return;
409                                 }
410                                 ur.row(deleteCW, UserRole.UR,"Not in Organization");
411                                 return;
412                             } else if(Role.byName.get(ur.role())==null) {
413                                 ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role()));
414                                 return;
415                             // Make sure owners can still be owners.
416                             } else if(ur.role().endsWith(".owner")) {
417                                 String err = identity.mayOwn(); 
418                                 if(err!=null) {
419                                         ur.row(deleteCW, UserRole.UR,String.format("%s may not be an owner: %s",ur.user(),err));
420                                         return;
421                                 }
422                             }
423                             
424                             
425                             
426                             // Just let expired UserRoles sit until deleted
427                             if(futureRange.inRange(ur.expires())&&(!mur.containsKey(ur.user() + '|' + ur.role()))) {
428                                     // Cannot just delete owners, unless there is at least one left. Process later
429                                     if ("owner".equals(ur.rname())) {
430                                         Set<UserRole> urs = owners.get(ur.role());
431                                         if (urs == null) {
432                                             urs = new HashSet<UserRole>();
433                                             owners.put(ur.role(), urs);
434                                         }
435                                         urs.add(ur);
436                                     } else {
437                                         Range r = writeAnalysis(noAvg,ur);
438                                         if(r!=null) {
439                                             Approval existing = findApproval(ur);
440                                             if(existing==null) {
441                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
442                                             }
443                                         }
444                                     }
445                              }
446                         } catch (OrganizationException e) {
447                             noAvg.error().log(e);
448                         }
449                     });
450                  } finally {
451                      tt.done();
452                  }
453                  mur.clear();
454
455                 /**
456                   Now Process Owners, one owner Role at a time, ensuring one is left,
457                   preferably a good one. If so, process the others as normal.
458
459                   Otherwise, write to ExpiredOwners Report
460                 */
461                  tt = trans.start("Analyze Owners Separately",Trans.SUB);
462                  try {
463                     if (!owners.values().isEmpty()) {
464                         File file = new File(logDir(), EXPIRED_OWNERS + sdate + CSV);
465                         final CSV ownerCSV = new CSV(env.access(),file);
466                         CSV.Writer expOwner = ownerCSV.writer();
467                         expOwner.row(INFO,EXPIRED_OWNERS,sdate,2);
468
469                         try {
470                             for (Set<UserRole> sur : owners.values()) {
471                                 int goodOwners = 0;
472                                 for (UserRole ur : sur) {
473                                     if (ur.expires().after(now)) {
474                                         ++goodOwners;
475                                     }
476                                 }
477
478                                 for (UserRole ur : sur) {
479                                     if (goodOwners >= minOwners) {
480                                         Range r = writeAnalysis(noAvg, ur);
481                                         if(r!=null) {
482                                             Approval existing = findApproval(ur);
483                                             if(existing==null) {
484                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
485                                             }
486                                         }
487                                     } else {
488                                         expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
489                                         Approval existing = findApproval(ur);
490                                         if(existing==null) {
491                                             ur.row(needApproveCW,UserRole.APPROVE_UR);
492                                         }
493                                     }
494                                 }
495                             }
496                         } finally {
497                             if(expOwner!=null) {
498                                 expOwner.close();
499                             }
500                         }
501                     }
502                  } finally {
503                      tt.done();
504                  }
505             } finally {
506                 Role.clear();
507                 UserRole.clear();
508             }
509
510             /**
511              * Check for Expired Credentials
512              */
513             try {
514                  // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
515                  Cred.load(trans, session);
516
517                 tt = trans.start("Analyze Expired Credentials",Trans.SUB);
518                 try {
519                     for (Cred cred : Cred.data.values()) {
520                         List<Instance> linst = cred.instances;
521                         if(linst!=null) {
522                             Instance lastBath = null;
523                             for(Instance inst : linst) {
524                                 // All Creds go through Life Cycle
525                                 if(deleteDate!=null && inst.expires.before(deleteDate)) {
526                                     writeAnalysis(noAvg, cred, inst); // will go to Delete
527                                 // Basic Auth has Pre-EOL notifications IF there is no Newer Credential
528                                 } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
529                                     if(lastBath==null || lastBath.expires.before(inst.expires)) {
530                                         lastBath = inst;
531                                     }
532                                 }
533                             }
534                             if(lastBath!=null) {
535                                 writeAnalysis(noAvg, cred, lastBath);
536                             }
537                         }
538                     }
539                 } finally {
540                     tt.done();
541                 }
542             } finally {
543                 Cred.clear();
544             }
545
546             ////////////////////
547             tt = trans.start("Analyze Expired X509s",Trans.SUB);
548             try {
549                 X509.load(noAvg, session, x509 -> {
550                     try {
551                         for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
552                             writeAnalysis(noAvg, x509, (X509Certificate)cert);
553                         }
554                     } catch (CertificateException | IOException e) {
555                         noAvg.error().log(e, "Error Decrypting X509");
556                     }
557                 });
558             } finally {
559                 tt.done();
560             }
561         } catch (FileNotFoundException e) {
562             noAvg.info().log(e);
563         }
564     }
565
566     private Approval findApproval(UserRole ur) {
567         Approval existing = null;
568         List<Approval> apprs = Approval.byUser.get(ur.user());
569         if(apprs!=null) {
570             for(Approval appr : apprs) {
571                 if(ur.role().equals(appr.getRole()) &&
572                     appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) {
573                         existing = appr;
574                 }
575             }
576         }
577         return existing;
578     }
579
580     private Range writeAnalysis(AuthzTrans noAvg, UserRole ur) {
581         Range r = expireRange.getRange("ur", ur.expires());
582         if(r!=null) {
583             Date lnd = ln.lastNotified(LastNotified.newKey(ur));
584             // Note: lnd is NEVER null
585             Identity i;
586             try {
587                 i = org.getIdentity(noAvg, ur.user());
588             } catch (OrganizationException e) {
589                 i=null;
590             }
591             if(r.needsContact(lnd,i)) {
592                 CSV.Writer cw = writerList.get(r.name());
593                 if(cw!=null) {
594                     ur.row(cw,UserRole.UR);
595                 }
596             }
597         }
598         return r;
599     }
600
601     private void writeAnalysis(AuthzTrans noAvg, Cred cred, Instance inst) {
602         if(cred!=null && inst!=null) {
603             Range r = expireRange.getRange("cred", inst.expires);
604             if(r!=null) {
605                 Date lnd = ln.lastNotified(LastNotified.newKey(cred,inst));
606                 // Note: lnd is NEVER null
607                 Identity i;
608                 try {
609                     i = org.getIdentity(noAvg, cred.id);
610                 } catch (OrganizationException e) {
611                     i=null;
612                 }
613                 if(r.needsContact(lnd,i)) {
614                     CSV.Writer cw = writerList.get(r.name());
615                     if(cw!=null) {
616                         cred.row(cw,inst);
617                     }
618                 }
619             }
620         }
621     }
622
623     private void writeAnalysis(AuthzTrans noAvg, X509 x509, X509Certificate x509Cert) throws IOException {
624         Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
625         if(r!=null) {
626             Date lnd = ln.lastNotified(LastNotified.newKey(x509,x509Cert));
627             // Note: lnd is NEVER null
628             Identity i;
629             try {
630                 i = org.getIdentity(noAvg, x509.id);
631             } catch (OrganizationException e) {
632                 i=null;
633             }
634             if(r.needsContact(lnd,i)) {
635                 CSV.Writer cw = writerList.get(r.name());
636                 if(cw!=null) {
637                     x509.row(cw,x509Cert);
638                 }
639             }
640         }
641     }
642
643     @Override
644     protected void _close(AuthzTrans trans) {
645         session.close();
646         for(CSV.Writer cw : writerList.values()) {
647             cw.close();
648         }
649     }
650
651 }