auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Analyze.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  *
   7  * Modifications Copyright (C) 2019 IBM.
   8  * ===========================================================================
   9  * Licensed under the Apache License, Version 2.0 (the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  *      http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  * ============LICENSE_END====================================================
  21  *
  22  */
  23
  24 package org.onap.aaf.auth.batch.reports;
  25
  26 import java.io.File;
  27 import java.io.FileNotFoundException;
  28 import java.io.IOException;
  29 import java.security.cert.Certificate;
  30 import java.security.cert.CertificateException;
  31 import java.security.cert.X509Certificate;
  32 import java.util.Date;
  33 import java.util.GregorianCalendar;
  34 import java.util.HashMap;
  35 import java.util.HashSet;
  36 import java.util.List;
  37 import java.util.Map;
  38 import java.util.Map.Entry;
  39 import java.util.Set;
  40 import java.util.TreeMap;
  41 import java.util.TreeSet;
  42 import java.util.UUID;
  43
  44 import org.onap.aaf.auth.batch.Batch;
  45 import org.onap.aaf.auth.batch.approvalsets.Pending;
  46 import org.onap.aaf.auth.batch.approvalsets.Ticket;
  47 import org.onap.aaf.auth.batch.helpers.Approval;
  48 import org.onap.aaf.auth.batch.helpers.Cred;
  49 import org.onap.aaf.auth.batch.helpers.Cred.Instance;
  50 import org.onap.aaf.auth.batch.helpers.ExpireRange;
  51 import org.onap.aaf.auth.batch.helpers.ExpireRange.Range;
  52 import org.onap.aaf.auth.batch.helpers.Future;
  53 import org.onap.aaf.auth.batch.helpers.LastNotified;
  54 import org.onap.aaf.auth.batch.helpers.Role;
  55 import org.onap.aaf.auth.batch.helpers.UserRole;
  56 import org.onap.aaf.auth.batch.helpers.X509;
  57 import org.onap.aaf.auth.dao.cass.CredDAO;
  58 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
  59 import org.onap.aaf.auth.env.AuthzTrans;
  60 import org.onap.aaf.auth.org.Organization.Identity;
  61 import org.onap.aaf.auth.org.OrganizationException;
  62 import org.onap.aaf.cadi.configure.Factory;
  63 import org.onap.aaf.cadi.util.CSV;
  64 import org.onap.aaf.misc.env.APIException;
  65 import org.onap.aaf.misc.env.Env;
  66 import org.onap.aaf.misc.env.TimeTaken;
  67 import org.onap.aaf.misc.env.Trans;
  68 import org.onap.aaf.misc.env.util.Chrono;
  69
  70
  71 public class Analyze extends Batch {
  72     private static final int unknown=0;
  73     private static final int owner=1;
  74     private static final int supervisor=2;
  75     private static final int total=0;
  76     private static final int pending=1;
  77     private static final int approved=2;
  78
  79
  80     public static final String NEED_APPROVALS = "NeedApprovals";
  81     private static final String EXTEND = "Extend";
  82     private static final String EXPIRED_OWNERS = "ExpiredOwners";
  83     private static final String CSV = ".csv";
  84     private static final String INFO = "info";
  85     private int minOwners;
  86     private Map<String, CSV.Writer> writerList;
  87     private ExpireRange expireRange;
  88     private Date deleteDate;
  89     private CSV.Writer deleteCW;
  90     private CSV.Writer needApproveCW;
  91     private CSV.Writer extendCW;
  92     private Range futureRange;
  93     private final String sdate;
  94     private LastNotified ln;
  95
  96     public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException {
  97         super(trans.env());
  98         trans.info().log("Starting Connection Process");
  99
 100         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
 101         try {
 102             TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
 103             try {
 104                 session = cluster.connect();
 105             } finally {
 106                 tt.done();
 107             }
 108
 109
 110             minOwners=1;
 111
 112             // Create Intermediate Output
 113             writerList = new HashMap<>();
 114
 115             expireRange = new ExpireRange(trans.env().access());
 116             sdate = Chrono.dateOnlyStamp(now);
 117             for( List<Range> lr : expireRange.ranges.values()) {
 118                 for(Range r : lr ) {
 119                     if(writerList.get(r.name())==null) {
 120                         File file = new File(logDir(),r.name() + sdate +CSV);
 121                         CSV csv = new CSV(env.access(),file);
 122                         CSV.Writer cw = csv.writer(false);
 123                         cw.row(INFO,r.name(),sdate,r.reportingLevel());
 124                         writerList.put(r.name(),cw);
 125                         if("Delete".equals(r.name())) {
 126                             deleteDate = r.getEnd();
 127                             deleteCW = cw;
 128                         }
 129                         trans.init().log("Creating File:",file.getAbsolutePath());
 130                     }
 131                 }
 132             }
 133
 134             // Setup New Approvals file
 135             futureRange = expireRange.newFutureRange();
 136             File file = new File(logDir(),NEED_APPROVALS + sdate +CSV);
 137             CSV approveCSV = new CSV(env.access(),file);
 138             needApproveCW = approveCSV.writer();
 139             needApproveCW.row(INFO,NEED_APPROVALS,sdate,1);
 140             writerList.put(NEED_APPROVALS,needApproveCW);
 141
 142             // Setup Extend Approvals file
 143             file = new File(logDir(),EXTEND + sdate +CSV);
 144             CSV extendCSV = new CSV(env.access(),file);
 145             extendCW = extendCSV.writer();
 146             extendCW.row(INFO,EXTEND,sdate,1);
 147             writerList.put(EXTEND,extendCW);
 148
 149             // Load full data of the following
 150             ln = new LastNotified(session);
 151
 152         } finally {
 153             tt0.done();
 154         }
 155     }
 156
 157     @Override
 158     protected void run(AuthzTrans trans) {
 159         TimeTaken tt;
 160         AuthzTrans noAvg = trans.env().newTransNoAvg();
 161
 162         ////////////////////
 163         // Load all Notifieds, and either add to local Data, or mark for Deletion.
 164         ln.loadAll(noAvg,expireRange.approveDelete,deleteCW);
 165
 166         // Hold Good Tickets to keyed User/Role for UserRole Step
 167         Map<String,Ticket> mur = new TreeMap<>();
 168
 169         try {
 170             Approval.load(trans, session, Approval.v2_0_17);
 171
 172             ////////////////////
 173             final Map<UUID,Ticket> goodTickets = new TreeMap<>();
 174             tt = trans.start("Analyze Expired Futures",Trans.SUB);
 175             try {
 176                 Future.load(noAvg, session, Future.withConstruct, fut -> {
 177                     List<Approval> appls = Approval.byTicket.get(fut.id());
 178                     if(!futureRange.inRange(fut.expires())) {
 179                         deleteCW.comment("Future %s expired", fut.id());
 180                         Future.row(deleteCW,fut);
 181                         if(appls!=null) {
 182                             for(Approval a : appls) {
 183                                 Approval.row(deleteCW, a);
 184                             }
 185                         }
 186                     } else if(appls==null) { // Orphaned Future (no Approvals)
 187                         deleteCW.comment("Future is Orphaned");
 188                         Future.row(deleteCW,fut);
 189                     } else  {
 190                         goodTickets.put(fut.fdd.id, new Ticket(fut));
 191                     }
 192                 });
 193             } finally {
 194                 tt.done();
 195             }
 196
 197             Set<String> approvers = new TreeSet<>();
 198             tt = trans.start("Connect Approvals with Futures",Trans.SUB);
 199             try {
 200                 for(Approval appr : Approval.list) {
 201                     Ticket ticket=null;
 202                     UUID ticketID = appr.getTicket();
 203                     if(ticketID!=null) {
 204                         ticket = goodTickets.get(appr.getTicket());
 205                     }
 206                     if(ticket == null) { // Orphaned Approvals, no Futures
 207                         deleteCW.comment("Approval is Orphaned");
 208                         Approval.row(deleteCW, appr);
 209                     } else {
 210                         // for users and approvers still valid
 211                         String user = appr.getUser();
 212
 213                         if(org.isRevoked(noAvg, appr.getApprover())) {
 214                             deleteCW.comment("Approver ID is revoked");
 215                             Approval.row(deleteCW, appr);
 216                         } else if(user!=null && !user.isEmpty() && org.isRevoked(noAvg, user)) {
 217                             deleteCW.comment("USER ID is revoked");
 218                             Approval.row(deleteCW, appr);
 219                         } else {
 220                             ticket.approvals.add(appr); // add to found Ticket
 221                             approvers.add(appr.getApprover());
 222                         }
 223                     }
 224                 }
 225             } finally {
 226                 tt.done();
 227             }
 228
 229             /* Run through all Futures, and see if
 230              * 1) they have been executed (no longer valid)
 231              * 2) The current Approvals indicate they can proceed
 232              */
 233             Map<String,Pending> pendingApprs = new HashMap<>();
 234             Map<String,Pending> pendingTemp = new HashMap<>();
 235
 236             String approver;
 237
 238             tt = trans.start("Analyze Good Tickets",Trans.SUB);
 239             try {
 240                 for(Ticket ticket : goodTickets.values()) {
 241                     try {
 242                         pendingTemp.clear();
 243                         switch(ticket.f.target()) {
 244                             case "user_role":
 245                                 int state[][] = new int[3][3];
 246                                 int type;
 247
 248                                 for(Approval appr : ticket.approvals) {
 249                                     switch(appr.getType()) {
 250                                         case "owner":
 251                                             type=owner;
 252                                             break;
 253                                         case "supervisor":
 254                                             type=supervisor;
 255                                             break;
 256                                         default:
 257                                             type=0;
 258                                     }
 259                                     ++state[type][total]; // count per type
 260                                     switch(appr.getStatus()) {
 261                                         case "pending":
 262                                             ++state[type][pending];
 263                                             approver = appr.getApprover();
 264                                             Pending n = pendingTemp.get(approver);
 265                                             if(n==null) {
 266                                                 Date lastNotified = ln.lastNotified(approver,"pending",null);
 267                                                 pendingTemp.put(approver,new Pending(lastNotified));
 268                                             } else {
 269                                                 n.inc();
 270                                             }
 271                                             break;
 272                                         case "approved":
 273                                             ++state[type][approved];
 274                                             break;
 275                                         default:
 276                                             ++state[type][unknown];
 277                                     }
 278                                 }
 279
 280                                 // To Approve:
 281                                 // Always must have at least 1 owner
 282                                 if((state[owner][total]>0 && state[owner][approved]>0) &&
 283                                     // If there are no Supervisors, that's ok
 284                                     (state[supervisor][total]==0 ||
 285                                     // But if there is a Supervisor, they must have approved
 286                                     (state[supervisor][approved]>0))) {
 287                                         UserRoleDAO.Data urdd = new UserRoleDAO.Data();
 288                                         try {
 289                                             urdd.reconstitute(ticket.f.fdd.construct);
 290                                             if(urdd.expires.before(ticket.f.expires())) {
 291                                                 extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires());
 292                                             }
 293                                         } catch (IOException e) {
 294                                             trans.error().log("Could not reconstitute UserRole");
 295                                         }
 296                                 } else { // Load all the Pending.
 297                                     for(Entry<String, Pending> es : pendingTemp.entrySet()) {
 298                                         Pending p = pendingApprs.get(es.getKey());
 299                                         if(p==null) {
 300                                             pendingApprs.put(es.getKey(), es.getValue());
 301                                         } else {
 302                                             p.inc(es.getValue());
 303                                         }
 304                                     }
 305                                 }
 306                                 break;
 307                         }
 308                     } finally {
 309                         if("user_role".equals(ticket.f.fdd.target)) {
 310                             String key = ticket.f.fdd.target_key;
 311                             if(key!=null) {
 312                                 mur.put(key, ticket);
 313                             }
 314                         }
 315                     }
 316                 }
 317             } finally {
 318                 tt.done();
 319             }
 320             // Good Tickets no longer needed
 321             goodTickets.clear();
 322
 323             /**
 324              * Decide to Notify about Approvals, based on activity/last Notified
 325              */
 326             tt = trans.start("Analyze Approval Reminders", Trans.SUB);
 327             try {
 328                 GregorianCalendar gc = new GregorianCalendar();
 329                 gc.add(GregorianCalendar.DAY_OF_WEEK, 5);
 330                 Date remind = gc.getTime();
 331
 332                 for(Entry<String, Pending> es : pendingApprs.entrySet()) {
 333                     Pending p = es.getValue();
 334                     if(p.newApprovals()
 335                             || p.earliest() == LastNotified.NEVER // yes, equals.
 336                             || p.earliest().after(remind)) {
 337                         p.row(needApproveCW,es.getKey());
 338                     }
 339                 }
 340             } finally {
 341                 tt.done();
 342             }
 343
 344             // clear out Approval Intermediates
 345             pendingTemp = null;
 346             pendingApprs = null;
 347         } finally {
 348         }
 349
 350         /**
 351            Run through User Roles.
 352            Owners are treated specially in next section.
 353            Regular roles are checked against Date Ranges.  If match Date Range, write out to appropriate file.
 354         */
 355
 356         try {
 357             Role.load(trans, session);
 358
 359             try {
 360                 tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB);
 361                 Set<String> specialCommented = new HashSet<>();
 362                 Map<String, Set<UserRole>> owners = new TreeMap<>();
 363                  try {
 364                     UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> {
 365                         Identity identity;
 366                         try {
 367                             identity = trans.org().getIdentity(noAvg,ur.user());
 368                             if(identity==null) {
 369                                 // Candidate for Delete, but not Users if Special
 370                                 String id = ur.user();
 371                                 for(String s : specialDomains) {
 372                                     if(id.endsWith(s)) {
 373                                         if(!specialCommented.contains(id)) {
 374                                             deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s);
 375                                             specialCommented.add(id);
 376                                         }
 377                                         return;
 378                                     }
 379                                 }
 380                                 if(specialNames.contains(id)) {
 381                                     if(!specialCommented.contains(id)) {
 382                                         deleteCW.comment("ID %s is a special ID  (UR Org Check)", id);
 383                                         specialCommented.add(id);
 384                                     }
 385                                     return;
 386                                 }
 387                                 ur.row(deleteCW, UserRole.UR,"Not in Organization");
 388                                 return;
 389                             } else if(Role.byName.get(ur.role())==null) {
 390                                 ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role()));
 391                                 return;
 392                             }
 393                             // Just let expired UserRoles sit until deleted
 394                             if(futureRange.inRange(ur.expires())) {
 395                                 if(!mur.containsKey(ur.user() + '|' + ur.role())) {
 396                                     // Cannot just delete owners, unless there is at least one left. Process later
 397                                     if ("owner".equals(ur.rname())) {
 398                                         Set<UserRole> urs = owners.get(ur.role());
 399                                         if (urs == null) {
 400                                             urs = new HashSet<UserRole>();
 401                                             owners.put(ur.role(), urs);
 402                                         }
 403                                         urs.add(ur);
 404                                     } else {
 405                                         Range r = writeAnalysis(noAvg,ur);
 406                                         if(r!=null) {
 407                                             Approval existing = findApproval(ur);
 408                                             if(existing==null) {
 409                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
 410                                             }
 411                                         }
 412                                     }
 413                                 }
 414                             }
 415                         } catch (OrganizationException e) {
 416                             noAvg.error().log(e);
 417                         }
 418                     });
 419                  } finally {
 420                      tt.done();
 421                  }
 422                  mur.clear();
 423
 424                 /**
 425                   Now Process Owners, one owner Role at a time, ensuring one is left,
 426                   preferably a good one. If so, process the others as normal.
 427
 428                   Otherwise, write to ExpiredOwners Report
 429                 */
 430                  tt = trans.start("Analyze Owners Separately",Trans.SUB);
 431                  try {
 432                     if (!owners.values().isEmpty()) {
 433                         File file = new File(logDir(), EXPIRED_OWNERS + sdate + CSV);
 434                         final CSV ownerCSV = new CSV(env.access(),file);
 435                         CSV.Writer expOwner = ownerCSV.writer();
 436                         expOwner.row(INFO,EXPIRED_OWNERS,sdate,2);
 437
 438                         try {
 439                             for (Set<UserRole> sur : owners.values()) {
 440                                 int goodOwners = 0;
 441                                 for (UserRole ur : sur) {
 442                                     if (ur.expires().after(now)) {
 443                                         ++goodOwners;
 444                                     }
 445                                 }
 446
 447                                 for (UserRole ur : sur) {
 448                                     if (goodOwners >= minOwners) {
 449                                         Range r = writeAnalysis(noAvg, ur);
 450                                         if(r!=null) {
 451                                             Approval existing = findApproval(ur);
 452                                             if(existing==null) {
 453                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
 454                                             }
 455                                         }
 456                                     } else {
 457                                         expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
 458                                         Approval existing = findApproval(ur);
 459                                         if(existing==null) {
 460                                             ur.row(needApproveCW,UserRole.APPROVE_UR);
 461                                         }
 462                                     }
 463                                 }
 464                             }
 465                         } finally {
 466                             if(expOwner!=null) {
 467                                 expOwner.close();
 468                             }
 469                         }
 470                     }
 471                  } finally {
 472                      tt.done();
 473                  }
 474             } finally {
 475                 Role.clear();
 476                 UserRole.clear();
 477             }
 478
 479             /**
 480              * Check for Expired Credentials
 481              */
 482             try {
 483                  // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
 484                  Cred.load(trans, session);
 485
 486                 tt = trans.start("Analyze Expired Credentials",Trans.SUB);
 487                 try {
 488                     for (Cred cred : Cred.data.values()) {
 489                         List<Instance> linst = cred.instances;
 490                         if(linst!=null) {
 491                             Instance lastBath = null;
 492                             for(Instance inst : linst) {
 493                                 // All Creds go through Life Cycle
 494                                 if(deleteDate!=null && inst.expires.before(deleteDate)) {
 495                                     writeAnalysis(noAvg, cred, inst); // will go to Delete
 496                                 // Basic Auth has Pre-EOL notifications IF there is no Newer Credential
 497                                 } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
 498                                     if(lastBath==null || lastBath.expires.before(inst.expires)) {
 499                                         lastBath = inst;
 500                                     }
 501                                 }
 502                             }
 503                             if(lastBath!=null) {
 504                                 writeAnalysis(noAvg, cred, lastBath);
 505                             }
 506                         }
 507                     }
 508                 } finally {
 509                     tt.done();
 510                 }
 511             } finally {
 512                 Cred.clear();
 513             }
 514
 515             ////////////////////
 516             tt = trans.start("Analyze Expired X509s",Trans.SUB);
 517             try {
 518                 X509.load(noAvg, session, x509 -> {
 519                     try {
 520                         for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
 521                             writeAnalysis(noAvg, x509, (X509Certificate)cert);
 522                         }
 523                     } catch (CertificateException | IOException e) {
 524                         noAvg.error().log(e, "Error Decrypting X509");
 525                     }
 526                 });
 527             } finally {
 528                 tt.done();
 529             }
 530         } catch (FileNotFoundException e) {
 531             noAvg.info().log(e);
 532         }
 533     }
 534
 535     private Approval findApproval(UserRole ur) {
 536         Approval existing = null;
 537         List<Approval> apprs = Approval.byUser.get(ur.user());
 538         if(apprs!=null) {
 539             for(Approval appr : apprs) {
 540                 if(ur.role().equals(appr.getRole()) &&
 541                     appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) {
 542                         existing = appr;
 543                 }
 544             }
 545         }
 546         return existing;
 547     }
 548
 549     private Range writeAnalysis(AuthzTrans noAvg, UserRole ur) {
 550         Range r = expireRange.getRange("ur", ur.expires());
 551         if(r!=null) {
 552             Date lnd = ln.lastNotified(LastNotified.newKey(ur));
 553             // Note: lnd is NEVER null
 554             Identity i;
 555             try {
 556                 i = org.getIdentity(noAvg, ur.user());
 557             } catch (OrganizationException e) {
 558                 i=null;
 559             }
 560             if(r.needsContact(lnd,i)) {
 561                 CSV.Writer cw = writerList.get(r.name());
 562                 if(cw!=null) {
 563                     ur.row(cw,UserRole.UR);
 564                 }
 565             }
 566         }
 567         return r;
 568     }
 569
 570     private void writeAnalysis(AuthzTrans noAvg, Cred cred, Instance inst) {
 571         if(cred!=null && inst!=null) {
 572             Range r = expireRange.getRange("cred", inst.expires);
 573             if(r!=null) {
 574                 Date lnd = ln.lastNotified(LastNotified.newKey(cred,inst));
 575                 // Note: lnd is NEVER null
 576                 Identity i;
 577                 try {
 578                     i = org.getIdentity(noAvg, cred.id);
 579                 } catch (OrganizationException e) {
 580                     i=null;
 581                 }
 582                 if(r.needsContact(lnd,i)) {
 583                     CSV.Writer cw = writerList.get(r.name());
 584                     if(cw!=null) {
 585                         cred.row(cw,inst);
 586                     }
 587                 }
 588             }
 589         }
 590     }
 591
 592     private void writeAnalysis(AuthzTrans noAvg, X509 x509, X509Certificate x509Cert) throws IOException {
 593         Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
 594         if(r!=null) {
 595             Date lnd = ln.lastNotified(LastNotified.newKey(x509,x509Cert));
 596             // Note: lnd is NEVER null
 597             Identity i;
 598             try {
 599                 i = org.getIdentity(noAvg, x509.id);
 600             } catch (OrganizationException e) {
 601                 i=null;
 602             }
 603             if(r.needsContact(lnd,i)) {
 604                 CSV.Writer cw = writerList.get(r.name());
 605                 if(cw!=null) {
 606                     x509.row(cw,x509Cert);
 607                 }
 608             }
 609         }
 610     }
 611
 612     @Override
 613     protected void _close(AuthzTrans trans) {
 614         session.close();
 615         for(CSV.Writer cw : writerList.values()) {
 616             cw.close();
 617         }
 618     }
 619
 620 }