auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.batch;
  23
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.io.PrintStream;
  30 import java.lang.reflect.Constructor;
  31 import java.net.InetAddress;
  32 import java.net.URL;
  33 import java.net.UnknownHostException;
  34 import java.nio.ByteBuffer;
  35 import java.text.SimpleDateFormat;
  36 import java.util.ArrayList;
  37 import java.util.GregorianCalendar;
  38 import java.util.HashSet;
  39 import java.util.List;
  40 import java.util.Set;
  41 import java.util.TimeZone;
  42
  43 import org.onap.aaf.auth.common.Define;
  44 import org.onap.aaf.auth.dao.CassAccess;
  45 import org.onap.aaf.auth.dao.cass.RoleDAO;
  46 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
  47 import org.onap.aaf.auth.dao.hl.Question;
  48 import org.onap.aaf.auth.env.AuthzEnv;
  49 import org.onap.aaf.auth.env.AuthzTrans;
  50 import org.onap.aaf.auth.layer.Result;
  51 import org.onap.aaf.auth.org.Organization;
  52 import org.onap.aaf.auth.org.Organization.Identity;
  53 import org.onap.aaf.auth.org.OrganizationException;
  54 import org.onap.aaf.auth.org.OrganizationFactory;
  55 import org.onap.aaf.cadi.Access.Level;
  56 import org.onap.aaf.cadi.PropAccess;
  57 import org.onap.aaf.cadi.config.Config;
  58 import org.onap.aaf.misc.env.APIException;
  59 import org.onap.aaf.misc.env.Env;
  60 import org.onap.aaf.misc.env.StaticSlot;
  61 import org.onap.aaf.misc.env.TimeTaken;
  62
  63 import com.datastax.driver.core.Cluster;
  64 import com.datastax.driver.core.ResultSet;
  65 import com.datastax.driver.core.Row;
  66 import com.datastax.driver.core.Session;
  67 import com.datastax.driver.core.Statement;
  68
  69 public abstract class Batch {
  70
  71     private static String rootNs;
  72
  73     private static StaticSlot ssargs;
  74
  75     protected static final String STARS = "*****";
  76
  77     protected final Cluster cluster;
  78     protected static AuthzEnv env;
  79     protected static Session session;
  80     protected static Set<String> specialNames;
  81     protected static List<String> specialDomains;
  82     protected static boolean dryRun;
  83     protected static String batchEnv;
  84
  85     public static final String CASS_ENV = "CASS_ENV";
  86     public static final String LOG_DIR = "LOG_DIR";
  87     protected static final String PUNT="punt";
  88     protected static final String MAX_EMAILS="MAX_EMAILS";
  89     protected static final String VERSION="VERSION";
  90     public static final String GUI_URL="GUI_URL";
  91
  92     protected final Organization org;
  93
  94
  95
  96     protected Batch(AuthzEnv env) throws APIException, IOException, OrganizationException {
  97         // Be able to change Environments
  98         // load extra properties, i.e.
  99         // PERF.cassandra.clusters=....
 100         batchEnv = env.getProperty(CASS_ENV);
 101         if (batchEnv != null) {
 102             batchEnv = batchEnv.trim();
 103             env.info().log("Redirecting to ",batchEnv,"environment");
 104             String str;
 105             for (String key : new String[]{
 106                     CassAccess.CASSANDRA_CLUSTERS,
 107                     CassAccess.CASSANDRA_CLUSTERS_PORT,
 108                     CassAccess.CASSANDRA_CLUSTERS_USER_NAME,
 109                     CassAccess.CASSANDRA_CLUSTERS_PASSWORD,
 110                     VERSION,GUI_URL,PUNT,MAX_EMAILS,
 111                     LOG_DIR,
 112                     "SPECIAL_NAMES"
 113                     }) {
 114                 if ((str = env.getProperty(batchEnv+'.'+key))!=null) {
 115                     env.setProperty(key, str);
 116                 }
 117             }
 118         }
 119
 120         // Setup for Dry Run
 121         cluster = CassAccess.cluster(env,batchEnv);
 122         env.info().log("cluster name - ",cluster.getClusterName());
 123         String dryRunStr = env.getProperty( "DRY_RUN" );
 124         if ( dryRunStr == null || "false".equals(dryRunStr.trim()) ) {
 125             dryRun = false;
 126         } else {
 127             dryRun = true;
 128             env.info().log("dryRun set to TRUE");
 129         }
 130
 131         org = OrganizationFactory.init(env);
 132         org.setTestMode(dryRun);
 133
 134         // Special names to allow behaviors beyond normal rules
 135         specialNames = new HashSet<>();
 136         specialDomains = new ArrayList<>();
 137         String names = env.getProperty( "SPECIAL_NAMES" );
 138         if ( names != null )
 139         {
 140             env.info().log("Loading SPECIAL_NAMES");
 141             for (String s :names.split(",") ) {
 142                 env.info().log("\tspecial: " + s );
 143                 if(s.indexOf('@')>0) {
 144                         specialNames.add( s.trim() );
 145                 } else {
 146                         specialDomains.add(s.trim());
 147                 }
 148             }
 149         }
 150     }
 151
 152     protected abstract void run(AuthzTrans trans);
 153     protected abstract void _close(AuthzTrans trans);
 154
 155     public String[] args() {
 156         return env.get(ssargs);
 157     }
 158
 159     public boolean isDryRun()
 160     {
 161         return dryRun;
 162     }
 163
 164     public boolean isSpecial(String user) {
 165         if(user==null) {
 166                 return false;
 167         }
 168         if (specialNames != null && specialNames.contains(user)) {
 169             env.info().log("specialName: " + user);
 170             return (true);
 171         } else {
 172                 if(specialDomains!=null) {
 173                         for(String sd : specialDomains) {
 174                                 if(user.endsWith(sd)) {
 175                                         env.info().log("specialDomain: " + user + " matches " + sd);
 176                                         return (true);
 177                                 }
 178                         }
 179                 }
 180         }
 181         return (false);
 182     }
 183
 184
 185     protected PrintStream fallout(PrintStream inFallout, String logType)
 186             throws IOException {
 187         PrintStream fallout = inFallout;
 188         if (fallout == null) {
 189             File dir = new File("logs");
 190             if (!dir.exists()) {
 191                 dir.mkdirs();
 192             }
 193
 194             File f = null;
 195             long uniq = System.currentTimeMillis();
 196
 197             f = new File(dir, getClass().getSimpleName() + "_" + logType + "_"
 198                     + uniq + ".log");
 199
 200             fallout = new PrintStream(new FileOutputStream(f, true));
 201         }
 202         return fallout;
 203     }
 204
 205     public Organization getOrgFromID(AuthzTrans trans, String user) {
 206         Organization organization;
 207         try {
 208             organization = OrganizationFactory.obtain(trans.env(),user.toLowerCase());
 209         } catch (OrganizationException e1) {
 210             trans.error().log(e1);
 211             organization=null;
 212         }
 213
 214         if (organization == null) {
 215             PrintStream fallout = null;
 216
 217             try {
 218                 fallout = fallout(fallout, "Fallout");
 219                 fallout.print("INVALID_ID,");
 220                 fallout.println(user);
 221             } catch (Exception e) {
 222                 env.error().log("Could not write to Fallout File", e);
 223             }
 224             return (null);
 225         }
 226
 227         return (organization);
 228     }
 229
 230     public static Row executeDeleteQuery(Statement stmt) {
 231         Row row = null;
 232         if (!dryRun) {
 233             row = session.execute(stmt).one();
 234         }
 235
 236         return (row);
 237
 238     }
 239
 240     public static int acquireRunLock(String className) {
 241         Boolean testEnv = true;
 242         String envStr = env.getProperty("AFT_ENVIRONMENT");
 243
 244         if (envStr != null) {
 245             if ("AFTPRD".equals(envStr)) {
 246                 testEnv = false;
 247             }
 248         } else {
 249             env.fatal()
 250                     .log("AFT_ENVIRONMENT property is required and was not found. Exiting.");
 251             System.exit(1);
 252         }
 253
 254         if (testEnv) {
 255             env.info().log("TESTMODE: skipping RunLock");
 256             return (1);
 257         }
 258
 259         String hostname = null;
 260         try {
 261             hostname = InetAddress.getLocalHost().getHostName();
 262         } catch (UnknownHostException e) {
 263             e.printStackTrace();
 264             env.warn().log("Unable to get hostname");
 265             return (0);
 266         }
 267
 268         ResultSet existing = session.execute(String.format(
 269                 "select * from authz.run_lock where class = '%s'", className));
 270
 271         for (Row row : existing) {
 272             long curr = System.currentTimeMillis();
 273             ByteBuffer lastRun = row.getBytesUnsafe(2); // Can I get this field
 274                                                         // by name?
 275
 276             long interval = (1 * 60 * 1000); // @@ Create a value in props file
 277                                                 // for this
 278             long prev = lastRun.getLong();
 279
 280             if ((curr - prev) <= interval) {
 281                 env.warn().log(
 282                         String.format("Too soon! Last run was %d minutes ago.",
 283                                 ((curr - prev) / 1000) / 60));
 284                 env.warn().log(
 285                         String.format("Min time between runs is %d minutes ",
 286                                 (interval / 1000) / 60));
 287                 env.warn().log(
 288                         String.format("Last ran on machine: %s at %s",
 289                                 row.getString("host"), row.getDate("start")));
 290                 return (0);
 291             } else {
 292                 env.info().log("Delete old lock");
 293                 deleteLock(className);
 294             }
 295         }
 296
 297         GregorianCalendar current = new GregorianCalendar();
 298
 299         // We want our time in UTC, hence "+0000"
 300         SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss+0000");
 301         fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
 302
 303         String cql = String
 304                 .format("INSERT INTO authz.run_lock (class,host,start) VALUES ('%s','%s','%s') IF NOT EXISTS",
 305                         className, hostname, fmt.format(current.getTime()));
 306
 307         env.info().log(cql);
 308
 309         Row row = session.execute(cql).one();
 310         if (!row.getBool("[applied]")) {
 311             env.warn().log("Lightweight Transaction failed to write lock.");
 312             env.warn().log(
 313                     String.format("host with lock: %s, running at %s",
 314                             row.getString("host"), row.getDate("start")));
 315             return (0);
 316         }
 317         return (1);
 318     }
 319
 320     private static void deleteLock( String className) {
 321         Row row = session.execute( String.format( "DELETE FROM authz.run_lock WHERE class = '%s' IF EXISTS", className ) ).one();
 322         if (! row.getBool("[applied]")) {
 323             env.info().log( "delete failed" );
 324         }
 325     }
 326
 327     private static void transferVMProps(AuthzEnv env, String ... props) {
 328         String value;
 329         for (String key : props) {
 330             if ((value = System.getProperty(key))!=null) {
 331                 env.setProperty(key, value);
 332             }
 333         }
 334     }
 335
 336     // IMPORTANT! VALIDATE Organization isUser method
 337     protected void checkOrganizationAcccess(AuthzTrans trans, Question q) throws APIException, OrganizationException {
 338             Set<String> testUsers = new HashSet<>();
 339             Result<List<RoleDAO.Data>> rrd = q.roleDAO.readNS(trans, rootNs);
 340             if (rrd.isOK()) {
 341                 for (RoleDAO.Data r : rrd.value) {
 342                     Result<List<UserRoleDAO.Data>> rur = q.userRoleDAO.readByRole(trans, r.fullName());
 343                     if (!rur.isOK()) {
 344                         continue;
 345                     }
 346                     for (UserRoleDAO.Data udd : rur.value) {
 347                         testUsers.add(udd.user);
 348                     }
 349                 }
 350                 if (testUsers.size() < 2) {
 351                     throw new APIException("Not enough Users in Roles for " + rootNs + " to Validate");
 352                 }
 353
 354                 Identity iden;
 355                 for (String user : testUsers) {
 356                     if ((iden = org.getIdentity(trans, user)) == null) {
 357                         throw new APIException("Failed Organization Entity Validation Check: " + user);
 358                     } else {
 359                         trans.info().log("Organization Validation Check: " + iden.id());
 360                     }
 361                 }
 362             }
 363         }
 364
 365     protected static String logDir() {
 366         String ld = env.getProperty(LOG_DIR);
 367         if (ld==null) {
 368             if (batchEnv==null) { // Deployed Batch doesn't use different ENVs, and a common logdir
 369                 ld = "logs/";
 370             } else {
 371                 ld = "logs/"+batchEnv;
 372             }
 373         }
 374         return ld;
 375     }
 376     protected int count(String str, char c) {
 377         if (str==null || str.isEmpty()) {
 378             return 0;
 379         } else {
 380             int count=1;
 381             for (int i=str.indexOf(c);i>=0;i=str.indexOf(c,i+1)) {
 382                 ++count;
 383             }
 384             return count;
 385         }
 386     }
 387
 388     public final void close(AuthzTrans trans) {
 389         _close(trans);
 390         cluster.close();
 391     }
 392
 393     public static void main(String[] args) {
 394         PropAccess access = new PropAccess(args);
 395         InputStream is = null;
 396         String filename;
 397         String propLoc;
 398         try {
 399             Define.set(access);
 400             rootNs =Define.ROOT_NS();
 401             if(access.getProperty(Config.CADI_PROP_FILES)==null) {
 402                     File f = new File("authBatch.props");
 403                     try {
 404                         if (f.exists()) {
 405                             filename = f.getAbsolutePath();
 406                             is = new FileInputStream(f);
 407                             propLoc = f.getPath();
 408                         } else {
 409                             URL rsrc = ClassLoader.getSystemResource("authBatch.props");
 410                             filename = rsrc.toString();
 411                             is = rsrc.openStream();
 412                             propLoc = rsrc.getPath();
 413                         }
 414                         access.load(is);
 415                     } finally {
 416                         if (is == null) {
 417                             System.err.println("authBatch.props must exist in current dir, or in Classpath");
 418                             System.exit(1);
 419                         }
 420                         is.close();
 421                     }
 422                     if (filename != null) {
 423                         access.log(Level.INFO,"Instantiated properties from", filename);
 424                     }
 425
 426                     // Log where Config found
 427                     access.log(Level.INFO,"Configuring from", propLoc);
 428
 429             }
 430             env = new AuthzEnv(access);
 431
 432             transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization");
 433
 434             // Flow all Env Logs to Log4j, with ENV
 435
 436 //            LogFileNamer lfn;
 437 //            lfn = new LogFileNamer(logDir(),"").noPID();
 438 //            lfn.setAppender("authz-batch");
 439 //            lfn.setAppender("aspr|ASPR");
 440 //            lfn.setAppender("sync");
 441 //            lfn.setAppender("jobchange");
 442 //            lfn.setAppender("validateuser");
 443 //            aspr = Logger.getLogger("aspr");
 444 //            Log4JLogTarget.setLog4JEnv("authz-batch", env);
 445 //            propLoc = null;
 446
 447             Batch batch = null;
 448             // setup ATTUser and Organization Slots before starting this:
 449             // TODO redo this
 450             // env.slot(ATT.ATT_USERSLOT);
 451             //
 452             // OrganizationFactory.setDefaultOrg(env, ATT.class.getName());
 453             AuthzTrans trans = env.newTrans();
 454
 455             TimeTaken tt = trans.start("Total Run", Env.SUB);
 456             try {
 457                 int len = args.length;
 458                 if (len > 0) {
 459                     String toolName = args[0];
 460                     len -= 1;
 461                     if (len < 0)
 462                         len = 0;
 463                     String nargs[] = new String[len];
 464                     if (len > 0) {
 465                         System.arraycopy(args, 1, nargs, 0, len);
 466                     }
 467
 468                     env.put(ssargs = env.staticSlot("ARGS"), nargs);
 469
 470                     /*
 471                      * Add New Batch Programs (inherit from Batch) here
 472                      */
 473
 474                     // Might be a Report, Update or Temp Batch
 475                     Class<?> cls;
 476                     String classifier = "";
 477                     try {
 478                         cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.update." + toolName);
 479                         classifier = "Update:";
 480                     } catch (ClassNotFoundException e) {
 481                         try {
 482                             cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.batch.reports." + toolName);
 483                             classifier = "Report:";
 484                         } catch (ClassNotFoundException e2) {
 485                             try {
 486                                 cls = ClassLoader.getSystemClassLoader()
 487                                         .loadClass("org.onap.aaf.auth.batch.temp." + toolName);
 488                                 classifier = "Temp Utility:";
 489                             } catch (ClassNotFoundException e3) {
 490                                 cls = null;
 491                             }
 492                         }
 493                     }
 494                     if (cls != null) {
 495                         Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
 496                         batch = (Batch) cnst.newInstance(trans);
 497                         env.info().log("Begin", classifier, toolName);
 498                     }
 499
 500
 501                     if (batch == null) {
 502                         trans.error().log("No Batch named", toolName, "found");
 503                     }
 504                     /*
 505                      * End New Batch Programs (inherit from Batch) here
 506                      */
 507
 508                 }
 509                 if (batch != null) {
 510                     batch.run(trans);
 511                 }
 512             } finally {
 513                 tt.done();
 514                 if (batch != null) {
 515                     batch.close(trans);
 516                 }
 517                 StringBuilder sb = new StringBuilder("Task Times\n");
 518                 trans.auditTrail(4, sb, AuthzTrans.SUB, AuthzTrans.REMOTE);
 519                 trans.info().log(sb);
 520             }
 521         } catch (Exception e) {
 522             e.printStackTrace(System.err);
 523             // Exceptions thrown by DB aren't stopping the whole process.
 524             System.exit(1);
 525         }
 526     }
 527
 528 }
 529