auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/reports/Analyze.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  *
   7  * Modifications Copyright (C) 2019 IBM.
   8  * ===========================================================================
   9  * Licensed under the Apache License, Version 2.0 (the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  *      http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  * ============LICENSE_END====================================================
  21  *
  22  */
  23
  24 package org.onap.aaf.auth.batch.reports;
  25
  26 import java.io.File;
  27 import java.io.FileNotFoundException;
  28 import java.io.IOException;
  29 import java.security.cert.Certificate;
  30 import java.security.cert.CertificateException;
  31 import java.security.cert.X509Certificate;
  32 import java.util.Date;
  33 import java.util.GregorianCalendar;
  34 import java.util.HashMap;
  35 import java.util.HashSet;
  36 import java.util.List;
  37 import java.util.Map;
  38 import java.util.Map.Entry;
  39 import java.util.Set;
  40 import java.util.TreeMap;
  41 import java.util.TreeSet;
  42 import java.util.UUID;
  43
  44 import org.onap.aaf.auth.batch.Batch;
  45 import org.onap.aaf.auth.batch.approvalsets.Pending;
  46 import org.onap.aaf.auth.batch.approvalsets.Ticket;
  47 import org.onap.aaf.auth.batch.helpers.Approval;
  48 import org.onap.aaf.auth.batch.helpers.Cred;
  49 import org.onap.aaf.auth.batch.helpers.Cred.Instance;
  50 import org.onap.aaf.auth.batch.helpers.ExpireRange;
  51 import org.onap.aaf.auth.batch.helpers.ExpireRange.Range;
  52 import org.onap.aaf.auth.batch.helpers.Future;
  53 import org.onap.aaf.auth.batch.helpers.LastNotified;
  54 import org.onap.aaf.auth.batch.helpers.Role;
  55 import org.onap.aaf.auth.batch.helpers.UserRole;
  56 import org.onap.aaf.auth.batch.helpers.X509;
  57 import org.onap.aaf.auth.dao.cass.CredDAO;
  58 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
  59 import org.onap.aaf.auth.env.AuthzTrans;
  60 import org.onap.aaf.auth.org.Organization.Identity;
  61 import org.onap.aaf.auth.org.OrganizationException;
  62 import org.onap.aaf.cadi.configure.Factory;
  63 import org.onap.aaf.cadi.util.CSV;
  64 import org.onap.aaf.misc.env.APIException;
  65 import org.onap.aaf.misc.env.Env;
  66 import org.onap.aaf.misc.env.TimeTaken;
  67 import org.onap.aaf.misc.env.Trans;
  68 import org.onap.aaf.misc.env.util.Chrono;
  69
  70
  71 public class Analyze extends Batch {
  72         private static final int unknown=0;
  73     private static final int owner=1;
  74     private static final int supervisor=2;
  75     private static final int total=0;
  76     private static final int pending=1;
  77     private static final int approved=2;
  78
  79
  80     public static final String NEED_APPROVALS = "NeedApprovals";
  81     private static final String EXTEND = "Extend";
  82     private static final String EXPIRED_OWNERS = "ExpiredOwners";
  83     private static final String CSV = ".csv";
  84     private static final String INFO = "info";
  85     private static final String NOT_COMPLIANT = "NotCompliant";
  86     private int minOwners;
  87     private Map<String, CSV.Writer> writerList;
  88     private ExpireRange expireRange;
  89     private Date deleteDate;
  90     private CSV.Writer deleteCW;
  91     private CSV.Writer needApproveCW;
  92     private CSV.Writer extendCW;
  93     private CSV.Writer notCompliantCW;
  94     private Range futureRange;
  95     private final String sdate;
  96     private LastNotified ln;
  97
  98     public Analyze(AuthzTrans trans) throws APIException, IOException, OrganizationException {
  99         super(trans.env());
 100         trans.info().log("Starting Connection Process");
 101
 102         TimeTaken tt0 = trans.start("Cassandra Initialization", Env.SUB);
 103         try {
 104             TimeTaken tt = trans.start("Connect to Cluster", Env.REMOTE);
 105             try {
 106                 session = cluster.connect();
 107             } finally {
 108                 tt.done();
 109             }
 110
 111
 112             minOwners=1;
 113
 114             // Create Intermediate Output
 115             writerList = new HashMap<>();
 116
 117             expireRange = new ExpireRange(trans.env().access());
 118             sdate = Chrono.dateOnlyStamp(now);
 119             for( List<Range> lr : expireRange.ranges.values()) {
 120                 for(Range r : lr ) {
 121                     if(writerList.get(r.name())==null) {
 122                         File file = new File(logDir(),r.name() + sdate +CSV);
 123                         CSV csv = new CSV(env.access(),file);
 124                         CSV.Writer cw = csv.writer(false);
 125                         cw.row(INFO,r.name(),sdate,r.reportingLevel());
 126                         writerList.put(r.name(),cw);
 127                         if("Delete".equals(r.name())) {
 128                             deleteDate = r.getEnd();
 129                             deleteCW = cw;
 130                         }
 131                         trans.init().log("Creating File:",file.getAbsolutePath());
 132                     }
 133                 }
 134             }
 135
 136             // Setup New Approvals file
 137             futureRange = expireRange.newFutureRange();
 138             File file = new File(logDir(),NEED_APPROVALS + sdate +CSV);
 139             CSV approveCSV = new CSV(env.access(),file);
 140             needApproveCW = approveCSV.writer();
 141             needApproveCW.row(INFO,NEED_APPROVALS,sdate,1);
 142             writerList.put(NEED_APPROVALS,needApproveCW);
 143
 144             // Setup Extend Approvals file
 145             file = new File(logDir(),EXTEND + sdate +CSV);
 146             CSV extendCSV = new CSV(env.access(),file);
 147             extendCW = extendCSV.writer();
 148             extendCW.row(INFO,EXTEND,sdate,1);
 149             writerList.put(EXTEND,extendCW);
 150
 151             // Setup NotCompliant Writer for Apps
 152             file = new File(logDir(),NOT_COMPLIANT + sdate + CSV);
 153             CSV ncCSV = new CSV(env.access(),file);
 154             notCompliantCW = ncCSV.writer();
 155             writerList.put(NOT_COMPLIANT, notCompliantCW);
 156
 157             // Load full data of the following
 158             ln = new LastNotified(session);
 159
 160         } finally {
 161             tt0.done();
 162         }
 163     }
 164
 165     @Override
 166     protected void run(AuthzTrans trans) {
 167         TimeTaken tt;
 168         AuthzTrans noAvg = trans.env().newTransNoAvg();
 169
 170         ////////////////////
 171         // Load all Notifieds, and either add to local Data, or mark for Deletion.
 172         ln.loadAll(noAvg,expireRange.approveDelete,deleteCW);
 173
 174         // Hold Good Tickets to keyed User/Role for UserRole Step
 175         Map<String,Ticket> mur = new TreeMap<>();
 176
 177         try {
 178             Approval.load(trans, session, Approval.v2_0_17);
 179
 180             ////////////////////
 181             final Map<UUID,Ticket> goodTickets = new TreeMap<>();
 182             tt = trans.start("Analyze Expired Futures",Trans.SUB);
 183             try {
 184                 Future.load(noAvg, session, Future.withConstruct, fut -> {
 185                     List<Approval> appls = Approval.byTicket.get(fut.id());
 186                     if(!futureRange.inRange(fut.expires())) {
 187                         deleteCW.comment("Future %s expired", fut.id());
 188                         Future.row(deleteCW,fut);
 189                         if(appls!=null) {
 190                             for(Approval a : appls) {
 191                                 Approval.row(deleteCW, a);
 192                             }
 193                         }
 194                     } else if(appls==null) { // Orphaned Future (no Approvals)
 195                         deleteCW.comment("Future is Orphaned");
 196                         Future.row(deleteCW,fut);
 197                     } else  {
 198                         goodTickets.put(fut.fdd.id, new Ticket(fut));
 199                     }
 200                 });
 201             } finally {
 202                 tt.done();
 203             }
 204
 205             Set<String> approvers = new TreeSet<>();
 206             tt = trans.start("Connect Approvals with Futures",Trans.SUB);
 207             try {
 208                 for(Approval appr : Approval.list) {
 209                     Ticket ticket=null;
 210                     UUID ticketID = appr.getTicket();
 211                     if(ticketID!=null) {
 212                         ticket = goodTickets.get(appr.getTicket());
 213                     }
 214                     if(ticket == null) { // Orphaned Approvals, no Futures
 215                         deleteCW.comment("Approval is Orphaned");
 216                         Approval.row(deleteCW, appr);
 217                     } else {
 218                         // for users and approvers still valid
 219                         String user = appr.getUser();
 220
 221                         if(org.isRevoked(noAvg, appr.getApprover())) {
 222                             deleteCW.comment("Approver ID is revoked");
 223                             Approval.row(deleteCW, appr);
 224                         } else if(user!=null && !user.isEmpty() && org.isRevoked(noAvg, user)) {
 225                             deleteCW.comment("USER ID is revoked");
 226                             Approval.row(deleteCW, appr);
 227                         } else {
 228                             ticket.approvals.add(appr); // add to found Ticket
 229                             approvers.add(appr.getApprover());
 230                         }
 231                     }
 232                 }
 233             } finally {
 234                 tt.done();
 235             }
 236
 237             /* Run through all Futures, and see if
 238              * 1) they have been executed (no longer valid)
 239              * 2) The current Approvals indicate they can proceed
 240              */
 241             Map<String,Pending> pendingApprs = new HashMap<>();
 242             Map<String,Pending> pendingTemp = new HashMap<>();
 243
 244             String approver;
 245
 246             tt = trans.start("Analyze Good Tickets",Trans.SUB);
 247             try {
 248                 for(Ticket ticket : goodTickets.values()) {
 249                     try {
 250                         pendingTemp.clear();
 251                         switch(ticket.f.target()) {
 252                             case "user_role":
 253                                 int state[][] = new int[3][3];
 254                                 int type;
 255
 256                                 for(Approval appr : ticket.approvals) {
 257                                     switch(appr.getType()) {
 258                                         case "owner":
 259                                             type=owner;
 260                                             break;
 261                                         case "supervisor":
 262                                             type=supervisor;
 263                                             break;
 264                                         default:
 265                                             type=0;
 266                                     }
 267                                     ++state[type][total]; // count per type
 268                                     switch(appr.getStatus()) {
 269                                         case "pending":
 270                                             ++state[type][pending];
 271                                             approver = appr.getApprover();
 272                                             Pending n = pendingTemp.get(approver);
 273                                             if(n==null) {
 274                                                 Date lastNotified = ln.lastNotified(approver,"pending",null);
 275                                                 pendingTemp.put(approver,new Pending(lastNotified));
 276                                             } else {
 277                                                 n.inc();
 278                                             }
 279                                             break;
 280                                         case "approved":
 281                                             ++state[type][approved];
 282                                             break;
 283                                         default:
 284                                             ++state[type][unknown];
 285                                     }
 286                                 }
 287
 288                                 // To Approve:
 289                                 // Always must have at least 1 owner
 290                                 if((state[owner][total]>0 && state[owner][approved]>0) &&
 291                                     // If there are no Supervisors, that's ok
 292                                     (state[supervisor][total]==0 ||
 293                                     // But if there is a Supervisor, they must have approved
 294                                     (state[supervisor][approved]>0))) {
 295                                         UserRoleDAO.Data urdd = new UserRoleDAO.Data();
 296                                         try {
 297                                             urdd.reconstitute(ticket.f.fdd.construct);
 298                                             if(urdd.expires.before(ticket.f.expires())) {
 299                                                 extendCW.row("extend_ur",urdd.user,urdd.role,ticket.f.expires());
 300                                             }
 301                                         } catch (IOException e) {
 302                                             trans.error().log("Could not reconstitute UserRole");
 303                                         }
 304                                 } else { // Load all the Pending.
 305                                     for(Entry<String, Pending> es : pendingTemp.entrySet()) {
 306                                         Pending p = pendingApprs.get(es.getKey());
 307                                         if(p==null) {
 308                                             pendingApprs.put(es.getKey(), es.getValue());
 309                                         } else {
 310                                             p.inc(es.getValue());
 311                                         }
 312                                     }
 313                                 }
 314                                 break;
 315                         }
 316                     } finally {
 317                         if("user_role".equals(ticket.f.fdd.target)) {
 318                             String key = ticket.f.fdd.target_key;
 319                             if(key!=null) {
 320                                 mur.put(key, ticket);
 321                             }
 322                         }
 323                     }
 324                 }
 325             } finally {
 326                 tt.done();
 327             }
 328             // Good Tickets no longer needed
 329             goodTickets.clear();
 330
 331             /**
 332              * Decide to Notify about Approvals, based on activity/last Notified
 333              */
 334             tt = trans.start("Analyze Approval Reminders", Trans.SUB);
 335             try {
 336                 GregorianCalendar gc = new GregorianCalendar();
 337                 gc.add(GregorianCalendar.DAY_OF_WEEK, 5);
 338                 Date remind = gc.getTime();
 339
 340                 for(Entry<String, Pending> es : pendingApprs.entrySet()) {
 341                     Pending p = es.getValue();
 342                     if(p.newApprovals()
 343                             || p.earliest() == LastNotified.NEVER // yes, equals.
 344                             || p.earliest().after(remind)) {
 345                         p.row(needApproveCW,es.getKey());
 346                     }
 347                 }
 348             } finally {
 349                 tt.done();
 350             }
 351
 352             // clear out Approval Intermediates
 353             pendingTemp = null;
 354             pendingApprs = null;
 355         } finally {
 356         }
 357
 358         /**
 359            Run through User Roles.
 360            Owners are treated specially in next section.
 361            Regular roles are checked against Date Ranges.  If match Date Range, write out to appropriate file.
 362         */
 363
 364         try {
 365             Role.load(trans, session);
 366
 367             try {
 368                 tt = trans.start("Analyze UserRoles, storing Owners",Trans.SUB);
 369                 Set<String> specialCommented = new HashSet<>();
 370                 Map<String, Set<UserRole>> owners = new TreeMap<>();
 371                  try {
 372                     UserRole.load(noAvg, session, UserRole.v2_0_11, ur -> {
 373                         Identity identity;
 374                         try {
 375                             identity = trans.org().getIdentity(noAvg,ur.user());
 376                             if(identity==null) {
 377                                 // Candidate for Delete, but not Users if Special
 378                                 String id = ur.user();
 379                                 for(String s : specialDomains) {
 380                                     if(id.endsWith(s)) {
 381                                         if(!specialCommented.contains(id)) {
 382                                             deleteCW.comment("ID %s is part of special Domain %s (UR Org Check)", id,s);
 383                                             specialCommented.add(id);
 384                                         }
 385                                         return;
 386                                     }
 387                                 }
 388                                 if(specialNames.contains(id)) {
 389                                     if(!specialCommented.contains(id)) {
 390                                         deleteCW.comment("ID %s is a special ID  (UR Org Check)", id);
 391                                         specialCommented.add(id);
 392                                     }
 393                                     return;
 394                                 }
 395                                 ur.row(deleteCW, UserRole.UR,"Not in Organization");
 396                                 return;
 397                             } else if(Role.byName.get(ur.role())==null) {
 398                                 ur.row(deleteCW, UserRole.UR,String.format("Role %s does not exist", ur.role()));
 399                                 return;
 400                             }
 401                             // Just let expired UserRoles sit until deleted
 402                             if(futureRange.inRange(ur.expires())&&(!mur.containsKey(ur.user() + '|' + ur.role()))) {
 403                                     // Cannot just delete owners, unless there is at least one left. Process later
 404                                     if ("owner".equals(ur.rname())) {
 405                                         Set<UserRole> urs = owners.get(ur.role());
 406                                         if (urs == null) {
 407                                             urs = new HashSet<UserRole>();
 408                                             owners.put(ur.role(), urs);
 409                                         }
 410                                         urs.add(ur);
 411                                     } else {
 412                                         Range r = writeAnalysis(noAvg,ur);
 413                                         if(r!=null) {
 414                                             Approval existing = findApproval(ur);
 415                                             if(existing==null) {
 416                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
 417                                             }
 418                                         }
 419                                     }
 420                              }
 421                         } catch (OrganizationException e) {
 422                             noAvg.error().log(e);
 423                         }
 424                     });
 425                  } finally {
 426                      tt.done();
 427                  }
 428                  mur.clear();
 429
 430                 /**
 431                   Now Process Owners, one owner Role at a time, ensuring one is left,
 432                   preferably a good one. If so, process the others as normal.
 433
 434                   Otherwise, write to ExpiredOwners Report
 435                 */
 436                  tt = trans.start("Analyze Owners Separately",Trans.SUB);
 437                  try {
 438                     if (!owners.values().isEmpty()) {
 439                         File file = new File(logDir(), EXPIRED_OWNERS + sdate + CSV);
 440                         final CSV ownerCSV = new CSV(env.access(),file);
 441                         CSV.Writer expOwner = ownerCSV.writer();
 442                         expOwner.row(INFO,EXPIRED_OWNERS,sdate,2);
 443
 444                         try {
 445                             for (Set<UserRole> sur : owners.values()) {
 446                                 int goodOwners = 0;
 447                                 for (UserRole ur : sur) {
 448                                     if (ur.expires().after(now)) {
 449                                         ++goodOwners;
 450                                     }
 451                                 }
 452
 453                                 for (UserRole ur : sur) {
 454                                     if (goodOwners >= minOwners) {
 455                                         Range r = writeAnalysis(noAvg, ur);
 456                                         if(r!=null) {
 457                                             Approval existing = findApproval(ur);
 458                                             if(existing==null) {
 459                                                 ur.row(needApproveCW,UserRole.APPROVE_UR);
 460                                             }
 461                                         }
 462                                     } else {
 463                                         expOwner.row("owner",ur.role(), ur.user(), Chrono.dateOnlyStamp(ur.expires()));
 464                                         Approval existing = findApproval(ur);
 465                                         if(existing==null) {
 466                                             ur.row(needApproveCW,UserRole.APPROVE_UR);
 467                                         }
 468                                     }
 469                                 }
 470                             }
 471                         } finally {
 472                             if(expOwner!=null) {
 473                                 expOwner.close();
 474                             }
 475                         }
 476                     }
 477                  } finally {
 478                      tt.done();
 479                  }
 480             } finally {
 481                 Role.clear();
 482                 UserRole.clear();
 483             }
 484
 485             /**
 486              * Check for Expired Credentials
 487              */
 488             try {
 489                  // Load Cred.  We don't follow Visitor, because we have to gather up everything into Identity Anyway
 490                  Cred.load(trans, session);
 491
 492                 tt = trans.start("Analyze Expired Credentials",Trans.SUB);
 493                 try {
 494                     for (Cred cred : Cred.data.values()) {
 495                         List<Instance> linst = cred.instances;
 496                         if(linst!=null) {
 497                             Instance lastBath = null;
 498                             for(Instance inst : linst) {
 499                                 // All Creds go through Life Cycle
 500                                 if(deleteDate!=null && inst.expires.before(deleteDate)) {
 501                                     writeAnalysis(noAvg, cred, inst); // will go to Delete
 502                                 // Basic Auth has Pre-EOL notifications IF there is no Newer Credential
 503                                 } else if (inst.type == CredDAO.BASIC_AUTH || inst.type == CredDAO.BASIC_AUTH_SHA256) {
 504                                     if(lastBath==null || lastBath.expires.before(inst.expires)) {
 505                                         lastBath = inst;
 506                                     }
 507                                 }
 508                             }
 509                             if(lastBath!=null) {
 510                                 writeAnalysis(noAvg, cred, lastBath);
 511                             }
 512                         }
 513                     }
 514                 } finally {
 515                     tt.done();
 516                 }
 517             } finally {
 518                 Cred.clear();
 519             }
 520
 521             ////////////////////
 522             tt = trans.start("Analyze Expired X509s",Trans.SUB);
 523             try {
 524                 X509.load(noAvg, session, x509 -> {
 525                     try {
 526                         for(Certificate cert : Factory.toX509Certificate(x509.x509)) {
 527                             writeAnalysis(noAvg, x509, (X509Certificate)cert);
 528                         }
 529                     } catch (CertificateException | IOException e) {
 530                         noAvg.error().log(e, "Error Decrypting X509");
 531                     }
 532                 });
 533             } finally {
 534                 tt.done();
 535             }
 536         } catch (FileNotFoundException e) {
 537             noAvg.info().log(e);
 538         }
 539     }
 540
 541     private Approval findApproval(UserRole ur) {
 542         Approval existing = null;
 543         List<Approval> apprs = Approval.byUser.get(ur.user());
 544         if(apprs!=null) {
 545             for(Approval appr : apprs) {
 546                 if(ur.role().equals(appr.getRole()) &&
 547                     appr.getMemo().contains(Chrono.dateOnlyStamp(ur.expires()))) {
 548                         existing = appr;
 549                 }
 550             }
 551         }
 552         return existing;
 553     }
 554
 555     private Range writeAnalysis(AuthzTrans noAvg, UserRole ur) {
 556         Range r = expireRange.getRange("ur", ur.expires());
 557         if(r!=null) {
 558             Date lnd = ln.lastNotified(LastNotified.newKey(ur));
 559             // Note: lnd is NEVER null
 560             Identity i;
 561             try {
 562                 i = org.getIdentity(noAvg, ur.user());
 563             } catch (OrganizationException e) {
 564                 i=null;
 565             }
 566             if(r.needsContact(lnd,i)) {
 567                 CSV.Writer cw = writerList.get(r.name());
 568                 if(cw!=null) {
 569                     ur.row(cw,UserRole.UR);
 570                 }
 571             }
 572         }
 573         return r;
 574     }
 575
 576     private void writeAnalysis(AuthzTrans noAvg, Cred cred, Instance inst) {
 577         if(cred!=null && inst!=null) {
 578             Range r = expireRange.getRange("cred", inst.expires);
 579             if(r!=null) {
 580                 Date lnd = ln.lastNotified(LastNotified.newKey(cred,inst));
 581                 // Note: lnd is NEVER null
 582                 Identity i;
 583                 try {
 584                     i = org.getIdentity(noAvg, cred.id);
 585                 } catch (OrganizationException e) {
 586                     i=null;
 587                 }
 588                 if(r.needsContact(lnd,i)) {
 589                     CSV.Writer cw = writerList.get(r.name());
 590                     if(cw!=null) {
 591                         cred.row(cw,inst);
 592                     }
 593                 }
 594             }
 595         }
 596     }
 597
 598     private void writeAnalysis(AuthzTrans noAvg, X509 x509, X509Certificate x509Cert) throws IOException {
 599         Range r = expireRange.getRange("x509", x509Cert.getNotAfter());
 600         if(r!=null) {
 601             Date lnd = ln.lastNotified(LastNotified.newKey(x509,x509Cert));
 602             // Note: lnd is NEVER null
 603             Identity i;
 604             try {
 605                 i = org.getIdentity(noAvg, x509.id);
 606             } catch (OrganizationException e) {
 607                 i=null;
 608             }
 609             if(r.needsContact(lnd,i)) {
 610                 CSV.Writer cw = writerList.get(r.name());
 611                 if(cw!=null) {
 612                     x509.row(cw,x509Cert);
 613                 }
 614             }
 615         }
 616     }
 617
 618     @Override
 619     protected void _close(AuthzTrans trans) {
 620         session.close();
 621         for(CSV.Writer cw : writerList.values()) {
 622             cw.close();
 623         }
 624     }
 625
 626 }