auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.batch;
  23
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.io.PrintStream;
  30 import java.lang.reflect.Constructor;
  31 import java.net.InetAddress;
  32 import java.net.URL;
  33 import java.net.UnknownHostException;
  34 import java.nio.ByteBuffer;
  35 import java.text.SimpleDateFormat;
  36 import java.util.ArrayList;
  37 import java.util.Date;
  38 import java.util.GregorianCalendar;
  39 import java.util.HashSet;
  40 import java.util.List;
  41 import java.util.Set;
  42 import java.util.TimeZone;
  43
  44 import org.apache.log4j.Logger;
  45 import org.onap.aaf.auth.common.Define;
  46 import org.onap.aaf.auth.dao.CassAccess;
  47 import org.onap.aaf.auth.env.AuthzEnv;
  48 import org.onap.aaf.auth.env.AuthzTrans;
  49 import org.onap.aaf.auth.log4j.Log4JAccessAppender;
  50 import org.onap.aaf.auth.org.Organization;
  51 import org.onap.aaf.auth.org.OrganizationException;
  52 import org.onap.aaf.auth.org.OrganizationFactory;
  53 import org.onap.aaf.cadi.Access.Level;
  54 import org.onap.aaf.cadi.PropAccess;
  55 import org.onap.aaf.cadi.config.Config;
  56 import org.onap.aaf.misc.env.APIException;
  57 import org.onap.aaf.misc.env.Env;
  58 import org.onap.aaf.misc.env.TimeTaken;
  59 import org.onap.aaf.misc.env.util.Chrono;
  60 import org.onap.aaf.misc.env.util.Split;
  61 import org.onap.aaf.misc.env.util.StringBuilderOutputStream;
  62
  63 import com.datastax.driver.core.Cluster;
  64 import com.datastax.driver.core.ResultSet;
  65 import com.datastax.driver.core.Row;
  66 import com.datastax.driver.core.Session;
  67 import com.datastax.driver.core.Statement;
  68
  69 public abstract class Batch {
  70     protected static final String STARS = "*****";
  71
  72     protected static Cluster cluster;
  73     protected static AuthzEnv env;
  74     protected static Session session;
  75     protected static Set<String> specialNames;
  76     protected static List<String> specialDomains;
  77     protected static boolean dryRun;
  78     protected static String batchEnv;
  79
  80     private static File logdir;
  81
  82         private static String[] batchArgs;
  83
  84     public static final String CASS_ENV = "CASS_ENV";
  85     public static final String LOG_DIR = "LOG_DIR";
  86     protected static final String MAX_EMAILS="MAX_EMAILS";
  87     protected static final String VERSION="VERSION";
  88     public static final String GUI_URL="GUI_URL";
  89
  90     protected final Organization org;
  91     protected String version;
  92     protected static final Date now = new Date();
  93     protected static final Date never = new Date(0);
  94
  95     protected Batch(AuthzEnv env) throws APIException, IOException, OrganizationException {
  96         if (batchEnv != null) {
  97             env.info().log("Redirecting to ",batchEnv,"environment");
  98             String str;
  99             for (String key : new String[]{
 100                     CassAccess.CASSANDRA_CLUSTERS,
 101                     CassAccess.CASSANDRA_CLUSTERS_PORT,
 102                     CassAccess.CASSANDRA_CLUSTERS_USER_NAME,
 103                     CassAccess.CASSANDRA_CLUSTERS_PASSWORD,
 104                     VERSION,GUI_URL,MAX_EMAILS,
 105                     LOG_DIR,
 106                     "SPECIAL_NAMES",
 107                     "MAIL_TEST_TO"
 108                     }) {
 109                 if ((str = env.getProperty(batchEnv+'.'+key))!=null) {
 110                     env.setProperty(key, str);
 111                 }
 112             }
 113         }
 114
 115         // Setup for Dry Run
 116         if(cluster==null) {
 117             cluster = CassAccess.cluster(env,batchEnv);
 118         }
 119         env.info().log("cluster name - ",cluster.getClusterName());
 120         String dryRunStr = env.getProperty( "DRY_RUN" );
 121         if ( dryRunStr == null || "false".equals(dryRunStr.trim()) ) {
 122             dryRun = false;
 123         } else {
 124             dryRun = true;
 125             env.info().log("dryRun set to TRUE");
 126         }
 127
 128         org = OrganizationFactory.init(env);
 129         if(org==null) {
 130             throw new OrganizationException("Organization MUST be defined for Batch");
 131         }
 132         org.setTestMode(dryRun);
 133
 134         // Special names to allow behaviors beyond normal rules
 135         specialNames = new HashSet<>();
 136         specialDomains = new ArrayList<>();
 137         String names = env.getProperty( "SPECIAL_NAMES" );
 138         if ( names != null )
 139         {
 140             env.info().log("Loading SPECIAL_NAMES");
 141             for (String s :names.split(",") ) {
 142                 env.info().log("\tspecial: " + s );
 143                 if(s.indexOf('@')>0) {
 144                     specialNames.add( s.trim() );
 145                 } else {
 146                     specialDomains.add(s.trim());
 147                 }
 148             }
 149         }
 150
 151         version = env.getProperty(VERSION,Config.AAF_DEFAULT_API_VERSION);
 152     }
 153
 154     protected abstract void run(AuthzTrans trans);
 155     protected void _close(AuthzTrans trans) {}
 156
 157     public String[] args() {
 158         return batchArgs;
 159     }
 160
 161     public boolean isDryRun()
 162     {
 163         return dryRun;
 164     }
 165
 166     public boolean isSpecial(String user) {
 167         if(user==null) {
 168             return false;
 169         }
 170         if (specialNames != null && specialNames.contains(user)) {
 171             env.info().log("specialName: " + user);
 172             return (true);
 173         } else {
 174             if(specialDomains!=null) {
 175                 for(String sd : specialDomains) {
 176                     if(user.endsWith(sd)) {
 177                         env.info().log("specialDomain: " + user + " matches " + sd);
 178                         return (true);
 179                     }
 180                 }
 181             }
 182         }
 183         return (false);
 184     }
 185
 186
 187     protected PrintStream fallout(PrintStream inFallout, String logType)
 188             throws IOException {
 189         PrintStream fallout = inFallout;
 190         if (fallout == null) {
 191             File dir = new File("logs");
 192             if (!dir.exists()) {
 193                 dir.mkdirs();
 194             }
 195
 196             File f = null;
 197             long uniq = System.currentTimeMillis();
 198
 199             f = new File(dir, getClass().getSimpleName() + "_" + logType + "_"
 200                     + uniq + ".log");
 201
 202             fallout = new PrintStream(new FileOutputStream(f, true));
 203         }
 204         return fallout;
 205     }
 206
 207     public Organization getOrgFromID(AuthzTrans trans, String user) {
 208         Organization organization;
 209         try {
 210             organization = OrganizationFactory.obtain(trans.env(),user.toLowerCase());
 211         } catch (OrganizationException e1) {
 212             trans.error().log(e1);
 213             organization=null;
 214         }
 215
 216         if (organization == null) {
 217             PrintStream fallout = null;
 218
 219             try {
 220                 fallout = fallout(fallout, "Fallout");
 221                 fallout.print("INVALID_ID,");
 222                 fallout.println(user);
 223             } catch (Exception e) {
 224                 env.error().log("Could not write to Fallout File", e);
 225             }
 226             return (null);
 227         }
 228
 229         return (organization);
 230     }
 231
 232     public static Row executeDeleteQuery(Statement stmt) {
 233         Row row = null;
 234         if (!dryRun) {
 235             row = session.execute(stmt).one();
 236         }
 237
 238         return (row);
 239
 240     }
 241
 242     public static int acquireRunLock(String className) {
 243         Boolean testEnv = true;
 244         String envStr = env.getProperty("AFT_ENVIRONMENT");
 245
 246         if (envStr != null) {
 247             if ("AFTPRD".equals(envStr)) {
 248                 testEnv = false;
 249             }
 250         } else {
 251             env.fatal()
 252                     .log("AFT_ENVIRONMENT property is required and was not found. Exiting.");
 253             System.exit(1);
 254         }
 255
 256         if (testEnv) {
 257             env.info().log("TESTMODE: skipping RunLock");
 258             return (1);
 259         }
 260
 261         String hostname = null;
 262         try {
 263             hostname = InetAddress.getLocalHost().getHostName();
 264         } catch (UnknownHostException e) {
 265             env.warn().log("Unable to get hostname : "+e.getMessage());
 266             return (0);
 267         }
 268
 269         ResultSet existing = session.execute(String.format(
 270                 "select * from authz.run_lock where class = '%s'", className));
 271
 272         for (Row row : existing) {
 273             long curr = System.currentTimeMillis();
 274             ByteBuffer lastRun = row.getBytesUnsafe(2); // Can I get this field
 275                                                         // by name?
 276
 277             long interval = (1 * 60 * 1000); // @@ Create a value in props file
 278                                                 // for this
 279             long prev = lastRun.getLong();
 280
 281             if ((curr - prev) <= interval) {
 282                 env.warn().log(
 283                         String.format("Too soon! Last run was %d minutes ago.",
 284                                 ((curr - prev) / 1000) / 60));
 285                 env.warn().log(
 286                         String.format("Min time between runs is %d minutes ",
 287                                 (interval / 1000) / 60));
 288                 env.warn().log(
 289                         String.format("Last ran on machine: %s at %s",
 290                                 row.getString("host"), row.getDate("start")));
 291                 return (0);
 292             } else {
 293                 env.info().log("Delete old lock");
 294                 deleteLock(className);
 295             }
 296         }
 297
 298         GregorianCalendar current = new GregorianCalendar();
 299
 300         // We want our time in UTC, hence "+0000"
 301         SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss+0000");
 302         fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
 303
 304         String cql = String
 305                 .format("INSERT INTO authz.run_lock (class,host,start) VALUES ('%s','%s','%s') IF NOT EXISTS",
 306                         className, hostname, fmt.format(current.getTime()));
 307
 308         env.info().log(cql);
 309
 310         Row row = session.execute(cql).one();
 311         if (!row.getBool("[applied]")) {
 312             env.warn().log("Lightweight Transaction failed to write lock.");
 313             env.warn().log(
 314                     String.format("host with lock: %s, running at %s",
 315                             row.getString("host"), row.getDate("start")));
 316             return (0);
 317         }
 318         return (1);
 319     }
 320
 321     private static void deleteLock( String className) {
 322         Row row = session.execute( String.format( "DELETE FROM authz.run_lock WHERE class = '%s' IF EXISTS", className ) ).one();
 323         if (! row.getBool("[applied]")) {
 324             env.info().log( "delete failed" );
 325         }
 326     }
 327
 328     private static void transferVMProps(AuthzEnv env, String ... props) {
 329         String value;
 330         for (String key : props) {
 331             if ((value = System.getProperty(key))!=null) {
 332                 env.setProperty(key, value);
 333             }
 334         }
 335     }
 336
 337     protected static File logDir() {
 338         if(logdir == null) {
 339             String ld = env.getProperty(LOG_DIR);
 340             if (ld==null) {
 341                 if (batchEnv==null) { // Deployed Batch doesn't use different ENVs, and a common logdir
 342                     ld = "logs/";
 343                 } else {
 344                     ld = "logs/"+batchEnv;
 345                 }
 346             }
 347             logdir = new File(ld);
 348             if(!logdir.exists()) {
 349                 logdir.mkdirs();
 350             }
 351         }
 352         return logdir;
 353     }
 354     protected int count(String str, char c) {
 355         if (str==null || str.isEmpty()) {
 356             return 0;
 357         } else {
 358             int count=1;
 359             for (int i=str.indexOf(c);i>=0;i=str.indexOf(c,i+1)) {
 360                 ++count;
 361             }
 362             return count;
 363         }
 364     }
 365
 366     public final void close(AuthzTrans trans) {
 367         _close(trans);
 368         if(session!=null) {
 369             session.close();
 370             session = null;
 371         }
 372         if(cluster!=null && !cluster.isClosed()) {
 373             cluster.close();
 374         }
 375     }
 376
 377     public static void main(String[] args) {
 378         // Use a StringBuilder to save off logs until a File can be setup
 379         StringBuilderOutputStream sbos = new StringBuilderOutputStream();
 380         PropAccess access = new PropAccess(new PrintStream(sbos),args);
 381         access.log(Level.INFO, "------- Starting Batch ------\n  Args: ");
 382         for(String s: args) {
 383             sbos.getBuffer().append(s);
 384             sbos.getBuffer().append(' ');
 385         }
 386         sbos.getBuffer().append('\n');
 387
 388         InputStream is = null;
 389         String filename;
 390         String propLoc;
 391         try {
 392             Define.set(access);
 393
 394             if(access.getProperty(Config.CADI_PROP_FILES)==null) {
 395                 File f = new File("authBatch.props");
 396                 try {
 397                     if (f.exists()) {
 398                         filename = f.getAbsolutePath();
 399                         is = new FileInputStream(f);
 400                         propLoc = f.getPath();
 401                     } else {
 402                         URL rsrc = ClassLoader.getSystemResource("authBatch.props");
 403                         filename = rsrc.toString();
 404                         is = rsrc.openStream();
 405                         propLoc = rsrc.getPath();
 406                     }
 407                     access.load(is);
 408                 } finally {
 409                     if (is == null) {
 410                         System.err.println("authBatch.props must exist in current dir, or in Classpath");
 411                         System.exit(1);
 412                     }
 413                     is.close();
 414                 }
 415                 if (filename != null) {
 416                     access.log(Level.INFO,"Instantiated properties from", filename);
 417                 }
 418
 419                 // Log where Config found
 420                 access.log(Level.INFO,"Configuring from", propLoc);
 421
 422             }
 423
 424             env = new AuthzEnv(access);
 425
 426             transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization");
 427
 428             // Be able to change Environments
 429             // load extra properties, i.e.
 430             // PERF.cassandra.clusters=....
 431             batchEnv = env.getProperty(CASS_ENV);
 432             if(batchEnv!=null) {
 433                 batchEnv = batchEnv.trim();
 434             }
 435
 436             File logFile = new File(logDir() + "/batch" + Chrono.dateOnlyStamp(new Date()) + ".log" );
 437             PrintStream batchLog = new PrintStream(new FileOutputStream(logFile,true));
 438             try {
 439                 access.setStreamLogIt(batchLog);
 440                 sbos.flush();
 441                 batchLog.print(sbos.getBuffer());
 442                 sbos = null;
 443                 Logger.getRootLogger().addAppender(new Log4JAccessAppender(access));
 444
 445                 Batch batch = null;
 446                 AuthzTrans trans = env.newTrans();
 447
 448                 TimeTaken tt = trans.start("Total Run", Env.SUB);
 449                 try {
 450                     int len = args.length;
 451                     if (len > 0) {
 452                         String toolName = args[0];
 453                         len -= 1;
 454                         if (len < 0)
 455                             len = 0;
 456                         batchArgs = new String[len];
 457                         if (len > 0) {
 458                             System.arraycopy(args, 1, batchArgs, 0, len);
 459                         }
 460                         /*
 461                          * Add New Batch Programs (inherit from Batch) here
 462                          */
 463
 464                         // Might be a Report, Update or Temp Batch
 465                         Class<?> cls = null;
 466                         String classifier = "";
 467
 468                         String[] pkgs = new String[] {
 469                                 "org.onap.aaf.auth.batch.update",
 470                                 "org.onap.aaf.auth.batch.reports",
 471                                 "org.onap.aaf.auth.batch.temp"
 472                                 };
 473
 474                         String ebp = env.getProperty("EXTRA_BATCH_PKGS");
 475                         if(ebp!=null) {
 476                             String[] ebps = Split.splitTrim(':', ebp);
 477                             String[] temp = new String[ebps.length + pkgs.length];
 478                             System.arraycopy(pkgs,0, temp, 0, pkgs.length);
 479                             System.arraycopy(ebps,0,temp,pkgs.length,ebps.length);
 480                             pkgs = temp;
 481                         }
 482
 483                         for(String p : pkgs) {
 484                             try {
 485                                 cls = ClassLoader.getSystemClassLoader().loadClass(p + '.' + toolName);
 486                                 int lastDot = p.lastIndexOf('.');
 487                                 if(p.length()>0 || p.length()!=lastDot) {
 488                                     StringBuilder sb = new StringBuilder();
 489                                     sb.append(Character.toUpperCase(p.charAt(++lastDot)));
 490                                     while(++lastDot<p.length()) {
 491                                         sb.append(p.charAt(lastDot));
 492                                     }
 493                                     sb.append(':');
 494                                     classifier = sb.toString();
 495                                     break;
 496                                 }
 497                             } catch (ClassNotFoundException e) {
 498                                 cls = null;
 499                             }
 500                         }
 501                         if (cls != null) {
 502                             Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
 503                             batch = (Batch) cnst.newInstance(trans);
 504                             System.out.println(batch.getClass().getCanonicalName());
 505                             env.info().log("Begin", classifier, toolName);
 506                         }
 507
 508
 509                         if (batch == null) {
 510                             trans.error().log("No Batch named", toolName, "found");
 511                         }
 512                         /*
 513                          * End New Batch Programs (inherit from Batch) here
 514                          */
 515
 516                     }
 517                     if (batch != null) {
 518                         try {
 519                                 for(String s : batchArgs) {
 520                                         System.out.println(s);
 521                                 }
 522                             batch.run(trans);
 523                         } catch (Exception e) {
 524                                 trans.error().log(e);
 525                             if(cluster!=null && !cluster.isClosed()) {
 526                                 cluster.close();
 527                             }
 528                             trans.error().log(e);
 529                         }
 530                     }
 531                 } finally {
 532                     tt.done();
 533                     if (batch != null) {
 534                         batch.close(trans);
 535                     }
 536                     StringBuilder sb = new StringBuilder("Task Times\n");
 537                     trans.auditTrail(4, sb, AuthzTrans.SUB, AuthzTrans.REMOTE);
 538                     trans.info().log(sb);
 539                 }
 540             } catch (Exception e) {
 541                 env.warn().log(e);
 542             } finally {
 543                 batchLog.close();
 544             }
 545
 546         } catch (Exception e) {
 547             if(cluster!=null && !cluster.isClosed()) {
 548                 cluster.close();
 549             }
 550             env.warn().log(System.err);
 551         }
 552     }
 553
 554 }
 555