auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.batch;
  23
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.io.PrintStream;
  30 import java.lang.reflect.Constructor;
  31 import java.net.InetAddress;
  32 import java.net.URL;
  33 import java.net.UnknownHostException;
  34 import java.nio.ByteBuffer;
  35 import java.text.SimpleDateFormat;
  36 import java.util.ArrayList;
  37 import java.util.Date;
  38 import java.util.GregorianCalendar;
  39 import java.util.HashSet;
  40 import java.util.List;
  41 import java.util.Set;
  42 import java.util.TimeZone;
  43
  44 import org.apache.log4j.Logger;
  45 import org.onap.aaf.auth.common.Define;
  46 import org.onap.aaf.auth.dao.CassAccess;
  47 import org.onap.aaf.auth.env.AuthzEnv;
  48 import org.onap.aaf.auth.env.AuthzTrans;
  49 import org.onap.aaf.auth.log4j.Log4JAccessAppender;
  50 import org.onap.aaf.auth.org.Organization;
  51 import org.onap.aaf.auth.org.OrganizationException;
  52 import org.onap.aaf.auth.org.OrganizationFactory;
  53 import org.onap.aaf.cadi.Access.Level;
  54 import org.onap.aaf.cadi.PropAccess;
  55 import org.onap.aaf.cadi.config.Config;
  56 import org.onap.aaf.misc.env.APIException;
  57 import org.onap.aaf.misc.env.Env;
  58 import org.onap.aaf.misc.env.StaticSlot;
  59 import org.onap.aaf.misc.env.TimeTaken;
  60 import org.onap.aaf.misc.env.util.Chrono;
  61 import org.onap.aaf.misc.env.util.Split;
  62 import org.onap.aaf.misc.env.util.StringBuilderOutputStream;
  63
  64 import com.datastax.driver.core.Cluster;
  65 import com.datastax.driver.core.ResultSet;
  66 import com.datastax.driver.core.Row;
  67 import com.datastax.driver.core.Session;
  68 import com.datastax.driver.core.Statement;
  69
  70 public abstract class Batch {
  71
  72     private static StaticSlot ssargs;
  73
  74     protected static final String STARS = "*****";
  75
  76     protected final Cluster cluster;
  77     protected static AuthzEnv env;
  78     protected static Session session;
  79     protected static Set<String> specialNames;
  80     protected static List<String> specialDomains;
  81     protected static boolean dryRun;
  82     protected static String batchEnv;
  83
  84         private static File logdir;
  85
  86     public static final String CASS_ENV = "CASS_ENV";
  87     public static final String LOG_DIR = "LOG_DIR";
  88     protected static final String MAX_EMAILS="MAX_EMAILS";
  89     protected static final String VERSION="VERSION";
  90     public static final String GUI_URL="GUI_URL";
  91
  92     protected final Organization org;
  93         protected String version;
  94
  95     protected Batch(AuthzEnv env) throws APIException, IOException, OrganizationException {
  96         if (batchEnv != null) {
  97             env.info().log("Redirecting to ",batchEnv,"environment");
  98             String str;
  99             for (String key : new String[]{
 100                     CassAccess.CASSANDRA_CLUSTERS,
 101                     CassAccess.CASSANDRA_CLUSTERS_PORT,
 102                     CassAccess.CASSANDRA_CLUSTERS_USER_NAME,
 103                     CassAccess.CASSANDRA_CLUSTERS_PASSWORD,
 104                     VERSION,GUI_URL,MAX_EMAILS,
 105                     LOG_DIR,
 106                     "SPECIAL_NAMES"
 107                     }) {
 108                 if ((str = env.getProperty(batchEnv+'.'+key))!=null) {
 109                     env.setProperty(key, str);
 110                 }
 111             }
 112         }
 113
 114         // Setup for Dry Run
 115         cluster = CassAccess.cluster(env,batchEnv);
 116         env.info().log("cluster name - ",cluster.getClusterName());
 117         String dryRunStr = env.getProperty( "DRY_RUN" );
 118         if ( dryRunStr == null || "false".equals(dryRunStr.trim()) ) {
 119             dryRun = false;
 120         } else {
 121             dryRun = true;
 122             env.info().log("dryRun set to TRUE");
 123         }
 124
 125         org = OrganizationFactory.init(env);
 126         if(org==null) {
 127                 throw new OrganizationException("Organization MUST be defined for Batch");
 128         }
 129         org.setTestMode(dryRun);
 130
 131         // Special names to allow behaviors beyond normal rules
 132         specialNames = new HashSet<>();
 133         specialDomains = new ArrayList<>();
 134         String names = env.getProperty( "SPECIAL_NAMES" );
 135         if ( names != null )
 136         {
 137             env.info().log("Loading SPECIAL_NAMES");
 138             for (String s :names.split(",") ) {
 139                 env.info().log("\tspecial: " + s );
 140                 if(s.indexOf('@')>0) {
 141                         specialNames.add( s.trim() );
 142                 } else {
 143                         specialDomains.add(s.trim());
 144                 }
 145             }
 146         }
 147
 148         version = env.getProperty(VERSION,Config.AAF_DEFAULT_API_VERSION);
 149     }
 150
 151     protected abstract void run(AuthzTrans trans);
 152     protected void _close(AuthzTrans trans) {}
 153
 154     public String[] args() {
 155         return env.get(ssargs);
 156     }
 157
 158     public boolean isDryRun()
 159     {
 160         return dryRun;
 161     }
 162
 163     public boolean isSpecial(String user) {
 164         if(user==null) {
 165                 return false;
 166         }
 167         if (specialNames != null && specialNames.contains(user)) {
 168             env.info().log("specialName: " + user);
 169             return (true);
 170         } else {
 171                 if(specialDomains!=null) {
 172                         for(String sd : specialDomains) {
 173                                 if(user.endsWith(sd)) {
 174                                         env.info().log("specialDomain: " + user + " matches " + sd);
 175                                         return (true);
 176                                 }
 177                         }
 178                 }
 179         }
 180         return (false);
 181     }
 182
 183
 184     protected PrintStream fallout(PrintStream inFallout, String logType)
 185             throws IOException {
 186         PrintStream fallout = inFallout;
 187         if (fallout == null) {
 188             File dir = new File("logs");
 189             if (!dir.exists()) {
 190                 dir.mkdirs();
 191             }
 192
 193             File f = null;
 194             long uniq = System.currentTimeMillis();
 195
 196             f = new File(dir, getClass().getSimpleName() + "_" + logType + "_"
 197                     + uniq + ".log");
 198
 199             fallout = new PrintStream(new FileOutputStream(f, true));
 200         }
 201         return fallout;
 202     }
 203
 204     public Organization getOrgFromID(AuthzTrans trans, String user) {
 205         Organization organization;
 206         try {
 207             organization = OrganizationFactory.obtain(trans.env(),user.toLowerCase());
 208         } catch (OrganizationException e1) {
 209             trans.error().log(e1);
 210             organization=null;
 211         }
 212
 213         if (organization == null) {
 214             PrintStream fallout = null;
 215
 216             try {
 217                 fallout = fallout(fallout, "Fallout");
 218                 fallout.print("INVALID_ID,");
 219                 fallout.println(user);
 220             } catch (Exception e) {
 221                 env.error().log("Could not write to Fallout File", e);
 222             }
 223             return (null);
 224         }
 225
 226         return (organization);
 227     }
 228
 229     public static Row executeDeleteQuery(Statement stmt) {
 230         Row row = null;
 231         if (!dryRun) {
 232             row = session.execute(stmt).one();
 233         }
 234
 235         return (row);
 236
 237     }
 238
 239     public static int acquireRunLock(String className) {
 240         Boolean testEnv = true;
 241         String envStr = env.getProperty("AFT_ENVIRONMENT");
 242
 243         if (envStr != null) {
 244             if ("AFTPRD".equals(envStr)) {
 245                 testEnv = false;
 246             }
 247         } else {
 248             env.fatal()
 249                     .log("AFT_ENVIRONMENT property is required and was not found. Exiting.");
 250             System.exit(1);
 251         }
 252
 253         if (testEnv) {
 254             env.info().log("TESTMODE: skipping RunLock");
 255             return (1);
 256         }
 257
 258         String hostname = null;
 259         try {
 260             hostname = InetAddress.getLocalHost().getHostName();
 261         } catch (UnknownHostException e) {
 262             e.printStackTrace();
 263             env.warn().log("Unable to get hostname");
 264             return (0);
 265         }
 266
 267         ResultSet existing = session.execute(String.format(
 268                 "select * from authz.run_lock where class = '%s'", className));
 269
 270         for (Row row : existing) {
 271             long curr = System.currentTimeMillis();
 272             ByteBuffer lastRun = row.getBytesUnsafe(2); // Can I get this field
 273                                                         // by name?
 274
 275             long interval = (1 * 60 * 1000); // @@ Create a value in props file
 276                                                 // for this
 277             long prev = lastRun.getLong();
 278
 279             if ((curr - prev) <= interval) {
 280                 env.warn().log(
 281                         String.format("Too soon! Last run was %d minutes ago.",
 282                                 ((curr - prev) / 1000) / 60));
 283                 env.warn().log(
 284                         String.format("Min time between runs is %d minutes ",
 285                                 (interval / 1000) / 60));
 286                 env.warn().log(
 287                         String.format("Last ran on machine: %s at %s",
 288                                 row.getString("host"), row.getDate("start")));
 289                 return (0);
 290             } else {
 291                 env.info().log("Delete old lock");
 292                 deleteLock(className);
 293             }
 294         }
 295
 296         GregorianCalendar current = new GregorianCalendar();
 297
 298         // We want our time in UTC, hence "+0000"
 299         SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss+0000");
 300         fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
 301
 302         String cql = String
 303                 .format("INSERT INTO authz.run_lock (class,host,start) VALUES ('%s','%s','%s') IF NOT EXISTS",
 304                         className, hostname, fmt.format(current.getTime()));
 305
 306         env.info().log(cql);
 307
 308         Row row = session.execute(cql).one();
 309         if (!row.getBool("[applied]")) {
 310             env.warn().log("Lightweight Transaction failed to write lock.");
 311             env.warn().log(
 312                     String.format("host with lock: %s, running at %s",
 313                             row.getString("host"), row.getDate("start")));
 314             return (0);
 315         }
 316         return (1);
 317     }
 318
 319     private static void deleteLock( String className) {
 320         Row row = session.execute( String.format( "DELETE FROM authz.run_lock WHERE class = '%s' IF EXISTS", className ) ).one();
 321         if (! row.getBool("[applied]")) {
 322             env.info().log( "delete failed" );
 323         }
 324     }
 325
 326     private static void transferVMProps(AuthzEnv env, String ... props) {
 327         String value;
 328         for (String key : props) {
 329             if ((value = System.getProperty(key))!=null) {
 330                 env.setProperty(key, value);
 331             }
 332         }
 333     }
 334
 335     protected static File logDir() {
 336         if(logdir == null) {
 337                 String ld = env.getProperty(LOG_DIR);
 338                 if (ld==null) {
 339                     if (batchEnv==null) { // Deployed Batch doesn't use different ENVs, and a common logdir
 340                         ld = "logs/";
 341                     } else {
 342                         ld = "logs/"+batchEnv;
 343                     }
 344                 }
 345                 logdir = new File(ld);
 346                 if(!logdir.exists()) {
 347                         logdir.mkdirs();
 348                 }
 349         }
 350         return logdir;
 351     }
 352     protected int count(String str, char c) {
 353         if (str==null || str.isEmpty()) {
 354             return 0;
 355         } else {
 356             int count=1;
 357             for (int i=str.indexOf(c);i>=0;i=str.indexOf(c,i+1)) {
 358                 ++count;
 359             }
 360             return count;
 361         }
 362     }
 363
 364     public final void close(AuthzTrans trans) {
 365         _close(trans);
 366         if(session!=null) {
 367                 session.close();
 368         }
 369         if(cluster!=null) {
 370             cluster.close();
 371         }
 372     }
 373
 374     public static void main(String[] args) {
 375         // Use a StringBuilder to save off logs until a File can be setup
 376         StringBuilderOutputStream sbos = new StringBuilderOutputStream();
 377         PropAccess access = new PropAccess(new PrintStream(sbos),args);
 378         access.log(Level.INIT, "------- Starting Batch ------\n  Args: ");
 379         for(String s: args) {
 380                 sbos.getBuffer().append(s);
 381                 sbos.getBuffer().append(' ');
 382         }
 383
 384         InputStream is = null;
 385         String filename;
 386         String propLoc;
 387         try {
 388             Define.set(access);
 389
 390             if(access.getProperty(Config.CADI_PROP_FILES)==null) {
 391                     File f = new File("authBatch.props");
 392                     try {
 393                         if (f.exists()) {
 394                             filename = f.getAbsolutePath();
 395                             is = new FileInputStream(f);
 396                             propLoc = f.getPath();
 397                         } else {
 398                             URL rsrc = ClassLoader.getSystemResource("authBatch.props");
 399                             filename = rsrc.toString();
 400                             is = rsrc.openStream();
 401                             propLoc = rsrc.getPath();
 402                         }
 403                         access.load(is);
 404                     } finally {
 405                         if (is == null) {
 406                             System.err.println("authBatch.props must exist in current dir, or in Classpath");
 407                             System.exit(1);
 408                         }
 409                         is.close();
 410                     }
 411                     if (filename != null) {
 412                         access.log(Level.INFO,"Instantiated properties from", filename);
 413                     }
 414
 415                     // Log where Config found
 416                     access.log(Level.INFO,"Configuring from", propLoc);
 417
 418             }
 419
 420             env = new AuthzEnv(access);
 421
 422             transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization");
 423
 424             // Be able to change Environments
 425             // load extra properties, i.e.
 426             // PERF.cassandra.clusters=....
 427             batchEnv = env.getProperty(CASS_ENV);
 428             if(batchEnv!=null) {
 429                 batchEnv = batchEnv.trim();
 430             }
 431
 432             File logFile = new File(logDir() + "/batch" + Chrono.dateOnlyStamp(new Date()) + ".log" );
 433             PrintStream batchLog = new PrintStream(new FileOutputStream(logFile,true));
 434             try {
 435                     access.setStreamLogIt(batchLog);
 436                     sbos.flush();
 437                     batchLog.print(sbos.getBuffer());
 438                     sbos = null;
 439                     Logger.getRootLogger().addAppender(new Log4JAccessAppender(access));
 440
 441                     Batch batch = null;
 442                     AuthzTrans trans = env.newTrans();
 443
 444                     TimeTaken tt = trans.start("Total Run", Env.SUB);
 445                     try {
 446                         int len = args.length;
 447                         if (len > 0) {
 448                             String toolName = args[0];
 449                             len -= 1;
 450                             if (len < 0)
 451                                 len = 0;
 452                             String nargs[] = new String[len];
 453                             if (len > 0) {
 454                                 System.arraycopy(args, 1, nargs, 0, len);
 455                             }
 456
 457                             env.put(ssargs = env.staticSlot("ARGS"), nargs);
 458
 459                             /*
 460                              * Add New Batch Programs (inherit from Batch) here
 461                              */
 462
 463                             // Might be a Report, Update or Temp Batch
 464                             Class<?> cls = null;
 465                             String classifier = "";
 466
 467                             String[] pkgs = new String[] {
 468                                         "org.onap.aaf.auth.batch.update",
 469                                         "org.onap.aaf.auth.batch.reports",
 470                                         "org.onap.aaf.auth.batch.temp"
 471                                         };
 472
 473                             String ebp = env.getProperty("EXTRA_BATCH_PKGS");
 474                             if(ebp!=null) {
 475                                 String[] ebps = Split.splitTrim(':', ebp);
 476                                 String[] temp = new String[ebps.length + pkgs.length];
 477                                 System.arraycopy(pkgs,0, temp, 0, pkgs.length);
 478                                 System.arraycopy(ebps,0,temp,pkgs.length,ebps.length);
 479                                 pkgs = temp;
 480                             }
 481
 482                             for(String p : pkgs) {
 483                                     try {
 484                                         cls = ClassLoader.getSystemClassLoader().loadClass(p + '.' + toolName);
 485                                         int lastDot = p.lastIndexOf('.');
 486                                         if(p.length()>0 || p.length()!=lastDot) {
 487                                                 StringBuilder sb = new StringBuilder();
 488                                                 sb.append(Character.toUpperCase(p.charAt(++lastDot)));
 489                                                 while(++lastDot<p.length()) {
 490                                                         sb.append(p.charAt(lastDot));
 491                                                 }
 492                                                 sb.append(':');
 493                                                 classifier = sb.toString();
 494                                                 break;
 495                                         }
 496                                     } catch (ClassNotFoundException e) {
 497                                         cls = null;
 498                                     }
 499                             }
 500                             if (cls != null) {
 501                                 Constructor<?> cnst = cls.getConstructor(AuthzTrans.class);
 502                                 batch = (Batch) cnst.newInstance(trans);
 503                                 env.info().log("Begin", classifier, toolName);
 504                             }
 505
 506
 507                             if (batch == null) {
 508                                 trans.error().log("No Batch named", toolName, "found");
 509                             }
 510                             /*
 511                              * End New Batch Programs (inherit from Batch) here
 512                              */
 513
 514                         }
 515                         if (batch != null) {
 516                             batch.run(trans);
 517                         }
 518                     } finally {
 519                         tt.done();
 520                         if (batch != null) {
 521                             batch.close(trans);
 522                         }
 523                         StringBuilder sb = new StringBuilder("Task Times\n");
 524                         trans.auditTrail(4, sb, AuthzTrans.SUB, AuthzTrans.REMOTE);
 525                         trans.info().log(sb);
 526                     }
 527             } finally {
 528                 batchLog.close();
 529             }
 530
 531         } catch (Exception e) {
 532             e.printStackTrace(System.err);
 533             // Exceptions thrown by DB aren't stopping the whole process.
 534             System.exit(1);
 535         }
 536     }
 537
 538 }
 539