auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Analyze.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  *
   7  * Modifications Copyright (C) 2019 IBM.
   8  * ===========================================================================
   9  * Licensed under the Apache License, Version 2.0 (the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  *      http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  * ============LICENSE_END====================================================
  21  *
  22  */
  23
  24 package org.onap.aaf.auth.batch.reports;
  25
  26 import java.io.File;
  27 import java.io.FileNotFoundException;
  28 import java.io.IOException;
  29 import java.security.cert.Certificate;
  30 import java.security.cert.CertificateException;
  31 import java.security.cert.X509Certificate;
  32 import java.util.Date;
  33 import java.util.GregorianCalendar;
  34 import java.util.HashMap;
  35 import java.util.HashSet;
  36 import java.util.List;
  37 import java.util.Map;
  38 import java.util.Map.Entry;
  39 import java.util.Set;
  40 import java.util.TreeMap;
  41 import java.util.TreeSet;
  42 import java.util.UUID;
  43
  44 import org.onap.aaf.auth.batch.Batch;
  45 import org.onap.aaf.auth.batch.approvalsets.Pending;
  46 import org.onap.aaf.auth.batch.approvalsets.Ticket;
  47 import org.onap.aaf.auth.batch.helpers.Approval;
  48 import org.onap.aaf.auth.batch.helpers.Cred;
  49 import org.onap.aaf.auth.batch.helpers.Cred.Instance;
  50 import org.onap.aaf.auth.batch.helpers.ExpireRange;
  51 import org.onap.aaf.auth.batch.helpers.ExpireRange.Range;
  52 import org.onap.aaf.auth.batch.helpers.Future;
  53 import org.onap.aaf.auth.batch.helpers.LastNotified;
  54 import org.onap.aaf.auth.batch.helpers.Role;
  55 import org.onap.aaf.auth.batch.helpers.UserRole;
  56 import org.onap.aaf.auth.batch.helpers.X509;
  57 import org.onap.aaf.auth.dao.cass.CredDAO;
  58 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
  59 import org.onap.aaf.auth.env.AuthzTrans;
  60 import org.onap.aaf.auth.org.Organization.Identity;
  61 import org.onap.aaf.auth.org.OrganizationException;
  62 import org.onap.aaf.cadi.configure.Factory;
  63 import org.onap.aaf.cadi.util.CSV;
  64 import org.onap.aaf.misc.env.APIException;
  65 import org.onap.aaf.misc.env.Env;
  66 import org.onap.aaf.misc.env.TimeTaken;
  67 import org.onap.aaf.misc.env.Trans;
  68 import org.onap.aaf.misc.env.util.Chrono;
  69
  70
  71 public class Analyze extends Batch {
  72     private static final int unknown=0;
  73     private static final int owner=1;
  74     private static final int supervisor=2;
  75     private static final int total=0;
  76     private static final int pending=1;
  77     private static final int approved=2;
  78
  79
  80     public static final String NEED_APPROVALS = "NeedApprovals";
  81     private static final String EXTEND = "Extend";
  82     private static final String EXPIRED_OWNERS = "ExpiredOwners";
  83     private static final String CSV = ".csv";
  84     private static final String INFO = "info";
  85     private int minOwners;
  86     private Map<String, CSV.Writer> writerList;
  87     private ExpireRange expireRange;
  88     private Date deleteDate;
  89     private CSV.Writer deleteCW;
  90     private CSV.Writer needApproveCW;
  91     private CSV.Writer extendCW;
  92     private Range futureRange;
  93     private final String sdate;
  94     private LastNotified ln;
  95
  96     public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException {
  97         super(trans.env());
  98         trans.info().log("Starting Connection Process");
  99
 100         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
 101         try {
 102             TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
 103             try {
 104                 session = cluster.connect();
 105             } finally {
 106                 tt.done();
 107             }
 108
 109
 110             minOwners=1;
 111
 112             // Create Intermediate Output
 113             writerList = new HashMap<>();
 114
 115             expireRange = new ExpireRange(trans.env().access());
 116             sdate = Chrono.dateOnlyStamp(now);
 117             for( List<Range> lr : expireRange.ranges.values()) {
 118                 for(Range r : lr ) {
 119                     if(writerList.get(r.name())==null) {
 120                         File file = new File(logDir(),r.name() + sdate +CSV);
 121                         CSV csv = new CSV(env.access(),file);
 122                         CSV.Writer cw = csv.writer(false);
 123                         cw.row(INFO,r.name(),sdate,r.reportingLevel());
 124                         writerList.put(r.name(),cw);
 125                         if("Delete".equals(r.name())) {
 126                             deleteDate = r.getEnd();
 127                             deleteCW = cw;
 128                         }
 129                         trans.init().log("Creating File:",file.getAbsolutePath());
 130                     }
 131                 }
 132             }
 133
 134             // Setup New Approvals file
 135             futureRange = expireRange.newFutureRange();
 136             File file = new File(logDir(),NEED_APPROVALS + sdate +CSV);
 137             CSV approveCSV = new CSV(env.access(),file);
 138             needApproveCW = approveCSV.writer();
 139             needApproveCW.row(INFO,NEED_APPROVALS,sdate,1);
 140             writerList.put(NEED_APPROVALS,needApproveCW);
 141
 142             // Setup Extend Approvals file
 143             file = new File(logDir(),EXTEND + sdate +CSV);
 144             CSV extendCSV = new CSV(env.access(),file);
 145             extendCW = extendCSV.writer();
 146             extendCW.row(INFO,EXTEND,sdate,1);
 147             writerList.put(EXTEND,extendCW);
 148
 149             // Load full data of the following
 150             ln = new LastNotified(session);
 151
 152         } finally {
 153             tt0.done();
 154         }
 155     }
 156
 157     @Override
 158     protected void run(AuthzTrans trans) {
 159         TimeTaken tt;
 160         AuthzTrans noAvg = trans.env().newTransNoAvg();
 161
 162         ////////////////////
 163         // Load all Notifieds, and either add to local Data, or mark for Deletion.
 164         ln.loadAll(noAvg,expireRange.approveDelete,deleteCW);
 165
 166         // Hold Good Tickets to keyed User/Role for UserRole Step
 167         Map<String,Ticket> mur = new TreeMap<>();
 168
 169         try {
 170             Approval.load(trans, session, Approval.v2_0_17);
 171
 172             ////////////////////
 173             final Map<UUID,Ticket> goodTickets = new TreeMap<>();
 174             tt = trans.start("Analyze Expired Futures",Trans.SUB);
 175             try {
 176                 Future.load(noAvg, session, Future.withConstruct, fut -> {
 177                     List<Approval> appls = Approval.byTicket.get(fut.id());
 178                     if(!futureRange.inRange(fut.expires())) {
 179                         deleteCW.comment("Future %s expired", fut.id());
 180                         Future.row(deleteCW,fut);
 181                         if(appls!=null) {
 182                             for(Approval a : appls) {
 183                                 Approval.row(deleteCW, a);
 184                             }
 185                         }
 186                     } else if(appls==null) { // Orphaned Future (no Approvals)
 187                         deleteCW.comment("Future is Orphaned");
 188                         Future.row(deleteCW,fut);
 189                     } else  {
 190                         goodTickets.put(fut.fdd.id, new Ticket(fut));
 191                     }
 192                 });
 193             } finally {
 194                 tt.done();
 195             }
 196
 197             Set<String> approvers = new TreeSet<>();
 198             tt = trans.start("Connect Approvals with Futures",Trans.SUB);
 199             try {
 200                 for(Approval appr : Approval.list) {
 201                     Ticket ticket=null;
 202                     UUID ticketID = appr.getTicket();
 203                     if(ticketID!=null) {
 204                         ticket = goodTickets.get(appr.getTicket());
 205                     }
 206                     if(ticket == null) { // Orphaned Approvals, no Futures
 207                         deleteCW.comment("Approval is Orphaned");
 208                         Approval.row(deleteCW, appr);
 209                     } else {
 210                         // for users and approvers still valid
 211                         String user = appr.getUser();
 212
 213                         if(org.isRevoked(noAvg, appr.getApprover())) {
 214                             deleteCW.comment("Approver ID is revoked");
 215                             Approval.row(deleteCW, appr);
 216                         } else if(user!=null && !user.isEmpty() && org.isRevoked(noAvg, user)) {
 217                             deleteCW.comment("USER ID is revoked");
 218                             Approval.row(deleteCW, appr);
 219                         } else {
 220                             ticket.approvals.add(appr); // add to found Ticket
 221                             approvers.add(appr.getApprover());
 222                         }
 223                     }
 224                 }
 225             } finally {
 226                 tt.done();
 227             }
 228
 229             /* Run through all Futures, and see if
 230              * 1) they have been executed (no longer valid)
 231              * 2) The current Approvals indicate they can proceed
 232              */
 233             Map<String,Pending> pendingApprs = new HashMap<>();
 234             Map<String,Pending> pendingTemp = new HashMap<>();
 235
 236             String approver;
 237
 238             tt = trans.start("Analyze Good Tickets",Trans.SUB);
 239             try {
 240                 for(Ticket ticket : goodTickets.values()) {
 241                     try {
 242                         pendingTemp.clear();
 243                         switch(ticket.f.target()) {
 244                             case "user_role":
 245                                 int state[][] = new int[3][3];
 246                                 int type;
 247
 248                                 for(Approval appr : ticket.approvals) {
 249                                     switch(appr.getType()) {
 250                                         case "owner":
 251                                             type=owner;
 252                                             break;
 253                                         case "supervisor":
 254                                             type=supervisor;
 255                                             break;
 256                                         default:
 257                                             type=0;
 258                                     }
 259                                     ++state[type][total]; // count per type
 260                                     switch(appr.getStatus()) {
 261                                         case "pending":
 262                                             ++state[type][pending];
 263                                             approver = appr.getApprover();
 264                                             Pending n = pendingTemp.get(approver);
 265                                             if(n==null) {
 266                                                 Date lastNotified = ln.lastNotified(approver,"pending",null);
 267                                                 pendingTemp.put(approver,new Pending(lastNotified));
 268                                             } else {
 269                                                 n.inc();
 270                                             }
 271                                             break;
 272                                         case "approved":
 273                                             ++state[type][approved];
 274                                             break;
 275                                         default:
 276                                             ++state[type][unknown];
 277                                     }
 278                                 }
 279
 280                                 // To Approve:
 281                                 // Always must have at least 1 owner
 282                                 if((state[owner][total]>0 && state[owner][approved]>0) &&
 283                                     // If there are no Supervisors, that's ok
 284                                     (state[supervisor][total]==0 ||
 285                                     // But if there is a Supervisor, they must have approved
 286                                     (state[supervisor][approved]>0))) {
 287                                         UserRoleDAO.Data urdd = new UserRoleDAO.Data();
 288                                         try {
 289                                             urdd.reconstitute(ticket.f.fdd.construct);
 290                                             if(urdd.expires.before(ticket.f.expires())) {
 291                                                 extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires());
 292                                             }
 293                                         } catch (IOException e) {
 294                                             trans.error().log("Could not reconstitute UserRole");
 295                                         }
 296                                 } else { // Load all the Pending.
 297                                     for(Entry<String, Pending> es : pendingTemp.entrySet()) {
 298                                         Pending p = pendingApprs.get(es.getKey());
 299                                         if(p==null) {
 300                                             pendingApprs.put(es.getKey(), es.getValue());
 301                                         } else {
 302                                             p.inc(es.getValue());
 303                                         }
 304                                     }
 305                                 }
 306                                 break;
 307                         }
 308                     } finally {
 309                         if("user_role".equals(ticket.f.fdd.target)) {
 310                             String key = ticket.f.fdd.target_key;
 311                             if(key!=null) {
 312                                 mur.put(key, ticket);
 313                             }
 314                         }
 315                     }
 316                 }
 317             } finally {
 318                 tt.done();
 319             }
 320             // Good Tickets no longer needed
 321             goodTickets.clear();
 322
 323             /**
 324              * Decide to Notify about Approvals, based on activity/last Notified
 325              */
 326             tt = trans.start("Analyze Approval Reminders", Trans.SUB);
 327             try {
 328                 GregorianCalendar gc = new GregorianCalendar();
 329                 gc.add(GregorianCalendar.DAY_OF_WEEK, 5);
 330                 Date remind = gc.getTime();
 331
 332                 for(Entry<String, Pending> es : pendingApprs.entrySet()) {
 333                     Pending p = es.getValue();
 334                     if(p.newApprovals()
 335                             || p.earliest() == LastNotified.NEVER // yes, equals.
 336                             || p.earliest().after(remind)) {
 337                         p.row(needApproveCW,es.getKey());
 338                     }
 339                 }
 340             } finally {
 341                 tt.done();
 342             }
 343
 344             // clear out Approval Intermediates
 345             pendingTemp = null;
 346             pendingApprs = null;
 347         } finally {
 348         }
 349
 350         /**
 351            Run through User Roles.
 352            Owners are treated specially in next section.
 353            Regular roles are checked against Date Ranges.  If match Date Range, write out to appropriate file.
 354         */
 355
 356         try {
 357             Role.load(trans, session);
 358
 359             try {
 360                 tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB);
 361                 Set<String> specialCommented = new HashSet<>();
 362                 Map<String, Set<UserRole>> owners = new TreeMap<>();
 363                  try {
 364                     UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> {
 365                         Identity identity;
 366                         try {
 367                             identity = trans.org().getIdentity(noAvg,ur.user());
 368                             if(identity==null) {
 369                                 // Candidate for Delete, but not Users if Special
 370                                 String id = ur.user();
 371                                 for(String s : specialDomains) {
 372                                     if(id.endsWith(s)) {
 373                                         if(!specialCommented.contains(id)) {
 374                                             deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s);
 375                                             specialCommented.add(id);
 376                                         }
 377                                         return;
 378                                     }
 379                                 }
 380                                 if(specialNames.contains(id)) {
 381                                     if(!specialCommented.contains(id)) {
 382                                         deleteCW.comment("ID %s is a special ID  (UR Org Check)", id);
 383                                         specialCommented.add(id);
 384                                     }
 385                                     return;
 386                                 }
 387                                 ur.row(deleteCW, UserRole.UR,"Not in Organization");
 388                                 return;
 389                             } else if(Role.byName.get(ur.role())==null) {
 390                                 ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role()));
 391                                 return;
 392                             }
 393                             // Just let expired UserRoles sit until deleted
 394                             if(futureRange.inRange(ur.expires())&&(!mur.containsKey(ur.user() + '|' + ur.role()))) {
 395                                     // Cannot just delete owners, unless there is at least one left. Process later
 396                                     if ("owner".equals(ur.rname())) {
 397                                         Set<UserRole> urs = owners.get(ur.role());
 398                                         if (urs == null) {
 399                                             urs = new HashSet<UserRole>();
 400                                             owners.put(ur.role(), urs);
 401                                         }
 402                                         urs.add(ur);
 403                                     } else {
 404                                         Range r = writeAnalysis(noAvg,ur);
 405                                         if(r!=null) {
 406                                             Approval existing = findApproval(ur);
 407                                             if(existing==null) {
 408                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
 409                                             }
 410                                         }
 411                                     }
 412                              }
 413                         } catch (OrganizationException e) {
 414                             noAvg.error().log(e);
 415                         }
 416                     });
 417                  } finally {
 418                      tt.done();
 419                  }
 420                  mur.clear();
 421
 422                 /**
 423                   Now Process Owners, one owner Role at a time, ensuring one is left,
 424                   preferably a good one. If so, process the others as normal.
 425
 426                   Otherwise, write to ExpiredOwners Report
 427                 */
 428                  tt = trans.start("Analyze Owners Separately",Trans.SUB);
 429                  try {
 430                     if (!owners.values().isEmpty()) {
 431                         File file = new File(logDir(), EXPIRED_OWNERS + sdate + CSV);
 432                         final CSV ownerCSV = new CSV(env.access(),file);
 433                         CSV.Writer expOwner = ownerCSV.writer();
 434                         expOwner.row(INFO,EXPIRED_OWNERS,sdate,2);
 435
 436                         try {
 437                             for (Set<UserRole> sur : owners.values()) {
 438                                 int goodOwners = 0;
 439                                 for (UserRole ur : sur) {
 440                                     if (ur.expires().after(now)) {
 441                                         ++goodOwners;
 442                                     }
 443                                 }
 444
 445                                 for (UserRole ur : sur) {
 446                                     if (goodOwners >= minOwners) {
 447                                         Range r = writeAnalysis(noAvg, ur);
 448                                         if(r!=null) {
 449                                             Approval existing = findApproval(ur);
 450                                             if(existing==null) {
 451                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
 452                                             }
 453                                         }
 454                                     } else {
 455                                         expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
 456                                         Approval existing = findApproval(ur);
 457                                         if(existing==null) {
 458                                             ur.row(needApproveCW,UserRole.APPROVE_UR);
 459                                         }
 460                                     }
 461                                 }
 462                             }
 463                         } finally {
 464                             if(expOwner!=null) {
 465                                 expOwner.close();
 466                             }
 467                         }
 468                     }
 469                  } finally {
 470                      tt.done();
 471                  }
 472             } finally {
 473                 Role.clear();
 474                 UserRole.clear();
 475             }
 476
 477             /**
 478              * Check for Expired Credentials
 479              */
 480             try {
 481                  // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
 482                  Cred.load(trans, session);
 483
 484                 tt = trans.start("Analyze Expired Credentials",Trans.SUB);
 485                 try {
 486                     for (Cred cred : Cred.data.values()) {
 487                         List<Instance> linst = cred.instances;
 488                         if(linst!=null) {
 489                             Instance lastBath = null;
 490                             for(Instance inst : linst) {
 491                                 // All Creds go through Life Cycle
 492                                 if(deleteDate!=null && inst.expires.before(deleteDate)) {
 493                                     writeAnalysis(noAvg, cred, inst); // will go to Delete
 494                                 // Basic Auth has Pre-EOL notifications IF there is no Newer Credential
 495                                 } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
 496                                     if(lastBath==null || lastBath.expires.before(inst.expires)) {
 497                                         lastBath = inst;
 498                                     }
 499                                 }
 500                             }
 501                             if(lastBath!=null) {
 502                                 writeAnalysis(noAvg, cred, lastBath);
 503                             }
 504                         }
 505                     }
 506                 } finally {
 507                     tt.done();
 508                 }
 509             } finally {
 510                 Cred.clear();
 511             }
 512
 513             ////////////////////
 514             tt = trans.start("Analyze Expired X509s",Trans.SUB);
 515             try {
 516                 X509.load(noAvg, session, x509 -> {
 517                     try {
 518                         for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
 519                             writeAnalysis(noAvg, x509, (X509Certificate)cert);
 520                         }
 521                     } catch (CertificateException | IOException e) {
 522                         noAvg.error().log(e, "Error Decrypting X509");
 523                     }
 524                 });
 525             } finally {
 526                 tt.done();
 527             }
 528         } catch (FileNotFoundException e) {
 529             noAvg.info().log(e);
 530         }
 531     }
 532
 533     private Approval findApproval(UserRole ur) {
 534         Approval existing = null;
 535         List<Approval> apprs = Approval.byUser.get(ur.user());
 536         if(apprs!=null) {
 537             for(Approval appr : apprs) {
 538                 if(ur.role().equals(appr.getRole()) &&
 539                     appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) {
 540                         existing = appr;
 541                 }
 542             }
 543         }
 544         return existing;
 545     }
 546
 547     private Range writeAnalysis(AuthzTrans noAvg, UserRole ur) {
 548         Range r = expireRange.getRange("ur", ur.expires());
 549         if(r!=null) {
 550             Date lnd = ln.lastNotified(LastNotified.newKey(ur));
 551             // Note: lnd is NEVER null
 552             Identity i;
 553             try {
 554                 i = org.getIdentity(noAvg, ur.user());
 555             } catch (OrganizationException e) {
 556                 i=null;
 557             }
 558             if(r.needsContact(lnd,i)) {
 559                 CSV.Writer cw = writerList.get(r.name());
 560                 if(cw!=null) {
 561                     ur.row(cw,UserRole.UR);
 562                 }
 563             }
 564         }
 565         return r;
 566     }
 567
 568     private void writeAnalysis(AuthzTrans noAvg, Cred cred, Instance inst) {
 569         if(cred!=null && inst!=null) {
 570             Range r = expireRange.getRange("cred", inst.expires);
 571             if(r!=null) {
 572                 Date lnd = ln.lastNotified(LastNotified.newKey(cred,inst));
 573                 // Note: lnd is NEVER null
 574                 Identity i;
 575                 try {
 576                     i = org.getIdentity(noAvg, cred.id);
 577                 } catch (OrganizationException e) {
 578                     i=null;
 579                 }
 580                 if(r.needsContact(lnd,i)) {
 581                     CSV.Writer cw = writerList.get(r.name());
 582                     if(cw!=null) {
 583                         cred.row(cw,inst);
 584                     }
 585                 }
 586             }
 587         }
 588     }
 589
 590     private void writeAnalysis(AuthzTrans noAvg, X509 x509, X509Certificate x509Cert) throws IOException {
 591         Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
 592         if(r!=null) {
 593             Date lnd = ln.lastNotified(LastNotified.newKey(x509,x509Cert));
 594             // Note: lnd is NEVER null
 595             Identity i;
 596             try {
 597                 i = org.getIdentity(noAvg, x509.id);
 598             } catch (OrganizationException e) {
 599                 i=null;
 600             }
 601             if(r.needsContact(lnd,i)) {
 602                 CSV.Writer cw = writerList.get(r.name());
 603                 if(cw!=null) {
 604                     x509.row(cw,x509Cert);
 605                 }
 606             }
 607         }
 608     }
 609
 610     @Override
 611     protected void _close(AuthzTrans trans) {
 612         session.close();
 613         for(CSV.Writer cw : writerList.values()) {
 614             cw.close();
 615         }
 616     }
 617
 618 }