auth/auth-batch/src/main/java/org/onap/aaf/auth/batch/Batch.java

   1 /**
   2  * ============LICENSE_START====================================================
   3  * org.onap.aaf
   4  * ===========================================================================
   5  * Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
   6  * ===========================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END====================================================
  19  *
  20  */
  21
  22 package org.onap.aaf.auth.batch;
  23
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.InputStream;
  29 import java.io.PrintStream;
  30 import java.lang.reflect.Constructor;
  31 import java.net.InetAddress;
  32 import java.net.URL;
  33 import java.net.UnknownHostException;
  34 import java.nio.ByteBuffer;
  35 import java.text.SimpleDateFormat;
  36 import java.util.GregorianCalendar;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Set;
  40 import java.util.TimeZone;
  41
  42 import org.onap.aaf.auth.common.Define;
  43 import org.onap.aaf.auth.dao.CassAccess;
  44 import org.onap.aaf.auth.dao.cass.RoleDAO;
  45 import org.onap.aaf.auth.dao.cass.UserRoleDAO;
  46 import org.onap.aaf.auth.dao.hl.Question;
  47 import org.onap.aaf.auth.env.AuthzEnv;
  48 import org.onap.aaf.auth.env.AuthzTrans;
  49 import org.onap.aaf.auth.layer.Result;
  50 import org.onap.aaf.auth.org.Organization;
  51 import org.onap.aaf.auth.org.Organization.Identity;
  52 import org.onap.aaf.auth.org.OrganizationException;
  53 import org.onap.aaf.auth.org.OrganizationFactory;
  54 import org.onap.aaf.cadi.PropAccess;
  55 import org.onap.aaf.cadi.Access.Level;
  56 import org.onap.aaf.cadi.config.Config;
  57 import org.onap.aaf.misc.env.APIException;
  58 import org.onap.aaf.misc.env.Env;
  59 import org.onap.aaf.misc.env.StaticSlot;
  60 import org.onap.aaf.misc.env.TimeTaken;
  61
  62 import com.datastax.driver.core.Cluster;
  63 import com.datastax.driver.core.ResultSet;
  64 import com.datastax.driver.core.Row;
  65 import com.datastax.driver.core.Session;
  66 import com.datastax.driver.core.Statement;
  67
  68 public abstract class Batch {
  69
  70     private static String rootNs;
  71
  72     private static StaticSlot ssargs;
  73
  74     protected static final String STARS = "*****";
  75
  76     protected final Cluster cluster;
  77     protected static AuthzEnv env;
  78     protected static Session session;
  79     protected static Set<String> specialNames;
  80     protected static boolean dryRun;
  81     protected static String batchEnv;
  82
  83     public static final String CASS_ENV = "CASS_ENV";
  84     public static final String LOG_DIR = "LOG_DIR";
  85     protected static final String PUNT="punt";
  86     protected static final String MAX_EMAILS="MAX_EMAILS";
  87     protected static final String VERSION="VERSION";
  88     public static final String GUI_URL="GUI_URL";
  89
  90     protected final Organization org;
  91
  92
  93
  94     protected Batch(AuthzEnv env) throws APIException, IOException, OrganizationException {
  95         // Be able to change Environments
  96         // load extra properties, i.e.
  97         // PERF.cassandra.clusters=....
  98         batchEnv = env.getProperty(CASS_ENV);
  99         if (batchEnv != null) {
 100             batchEnv = batchEnv.trim();
 101             env.info().log("Redirecting to ",batchEnv,"environment");
 102             String str;
 103             for (String key : new String[]{
 104                     CassAccess.CASSANDRA_CLUSTERS,
 105                     CassAccess.CASSANDRA_CLUSTERS_PORT,
 106                     CassAccess.CASSANDRA_CLUSTERS_USER_NAME,
 107                     CassAccess.CASSANDRA_CLUSTERS_PASSWORD,
 108                     VERSION,GUI_URL,PUNT,MAX_EMAILS,
 109                     LOG_DIR,
 110                     "SPECIAL_NAMES"
 111                     }) {
 112                 if ((str = env.getProperty(batchEnv+'.'+key))!=null) {
 113                     env.setProperty(key, str);
 114                 }
 115             }
 116         }
 117
 118         // Setup for Dry Run
 119         cluster = CassAccess.cluster(env,batchEnv);
 120         env.info().log("cluster name - ",cluster.getClusterName());
 121         String dryRunStr = env.getProperty( "DRY_RUN" );
 122         if ( dryRunStr == null || "false".equals(dryRunStr.trim()) ) {
 123             dryRun = false;
 124         } else {
 125             dryRun = true;
 126             env.info().log("dryRun set to TRUE");
 127         }
 128
 129         org = OrganizationFactory.init(env);
 130         org.setTestMode(dryRun);
 131
 132         // Special names to allow behaviors beyond normal rules
 133         specialNames = new HashSet<>();
 134         String names = env.getProperty( "SPECIAL_NAMES" );
 135         if ( names != null )
 136         {
 137             env.info().log("Loading SPECIAL_NAMES");
 138             for (String s :names.split(",") )
 139             {
 140                 env.info().log("\tspecial: " + s );
 141                 specialNames.add( s.trim() );
 142             }
 143         }
 144     }
 145
 146     protected abstract void run(AuthzTrans trans);
 147     protected abstract void _close(AuthzTrans trans);
 148
 149     public String[] args() {
 150         return env.get(ssargs);
 151     }
 152
 153     public boolean isDryRun()
 154     {
 155         return dryRun;
 156     }
 157
 158     public boolean isSpecial(String user) {
 159         if (specialNames != null && specialNames.contains(user)) {
 160             env.info().log("specialName: " + user);
 161
 162             return (true);
 163         } else {
 164             return (false);
 165         }
 166     }
 167
 168
 169     protected PrintStream fallout(PrintStream inFallout, String logType)
 170             throws IOException {
 171         PrintStream fallout = inFallout;
 172         if (fallout == null) {
 173             File dir = new File("logs");
 174             if (!dir.exists()) {
 175                 dir.mkdirs();
 176             }
 177
 178             File f = null;
 179             long uniq = System.currentTimeMillis();
 180
 181             f = new File(dir, getClass().getSimpleName() + "_" + logType + "_"
 182                     + uniq + ".log");
 183
 184             fallout = new PrintStream(new FileOutputStream(f, true));
 185         }
 186         return fallout;
 187     }
 188
 189     public Organization getOrgFromID(AuthzTrans trans, String user) {
 190         Organization organization;
 191         try {
 192             organization = OrganizationFactory.obtain(trans.env(),user.toLowerCase());
 193         } catch (OrganizationException e1) {
 194             trans.error().log(e1);
 195             organization=null;
 196         }
 197
 198         if (organization == null) {
 199             PrintStream fallout = null;
 200
 201             try {
 202                 fallout = fallout(fallout, "Fallout");
 203                 fallout.print("INVALID_ID,");
 204                 fallout.println(user);
 205             } catch (Exception e) {
 206                 env.error().log("Could not write to Fallout File", e);
 207             }
 208             return (null);
 209         }
 210
 211         return (organization);
 212     }
 213
 214     public static Row executeDeleteQuery(Statement stmt) {
 215         Row row = null;
 216         if (!dryRun) {
 217             row = session.execute(stmt).one();
 218         }
 219
 220         return (row);
 221
 222     }
 223
 224     public static int acquireRunLock(String className) {
 225         Boolean testEnv = true;
 226         String envStr = env.getProperty("AFT_ENVIRONMENT");
 227
 228         if (envStr != null) {
 229             if ("AFTPRD".equals(envStr)) {
 230                 testEnv = false;
 231             }
 232         } else {
 233             env.fatal()
 234                     .log("AFT_ENVIRONMENT property is required and was not found. Exiting.");
 235             System.exit(1);
 236         }
 237
 238         if (testEnv) {
 239             env.info().log("TESTMODE: skipping RunLock");
 240             return (1);
 241         }
 242
 243         String hostname = null;
 244         try {
 245             hostname = InetAddress.getLocalHost().getHostName();
 246         } catch (UnknownHostException e) {
 247             e.printStackTrace();
 248             env.warn().log("Unable to get hostname");
 249             return (0);
 250         }
 251
 252         ResultSet existing = session.execute(String.format(
 253                 "select * from authz.run_lock where class = '%s'", className));
 254
 255         for (Row row : existing) {
 256             long curr = System.currentTimeMillis();
 257             ByteBuffer lastRun = row.getBytesUnsafe(2); // Can I get this field
 258                                                         // by name?
 259
 260             long interval = (1 * 60 * 1000); // @@ Create a value in props file
 261                                                 // for this
 262             long prev = lastRun.getLong();
 263
 264             if ((curr - prev) <= interval) {
 265                 env.warn().log(
 266                         String.format("Too soon! Last run was %d minutes ago.",
 267                                 ((curr - prev) / 1000) / 60));
 268                 env.warn().log(
 269                         String.format("Min time between runs is %d minutes ",
 270                                 (interval / 1000) / 60));
 271                 env.warn().log(
 272                         String.format("Last ran on machine: %s at %s",
 273                                 row.getString("host"), row.getDate("start")));
 274                 return (0);
 275             } else {
 276                 env.info().log("Delete old lock");
 277                 deleteLock(className);
 278             }
 279         }
 280
 281         GregorianCalendar current = new GregorianCalendar();
 282
 283         // We want our time in UTC, hence "+0000"
 284         SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss+0000");
 285         fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
 286
 287         String cql = String
 288                 .format("INSERT INTO authz.run_lock (class,host,start) VALUES ('%s','%s','%s') IF NOT EXISTS",
 289                         className, hostname, fmt.format(current.getTime()));
 290
 291         env.info().log(cql);
 292
 293         Row row = session.execute(cql).one();
 294         if (!row.getBool("[applied]")) {
 295             env.warn().log("Lightweight Transaction failed to write lock.");
 296             env.warn().log(
 297                     String.format("host with lock: %s, running at %s",
 298                             row.getString("host"), row.getDate("start")));
 299             return (0);
 300         }
 301         return (1);
 302     }
 303
 304     private static void deleteLock( String className) {
 305         Row row = session.execute( String.format( "DELETE FROM authz.run_lock WHERE class = '%s' IF EXISTS", className ) ).one();
 306         if (! row.getBool("[applied]")) {
 307             env.info().log( "delete failed" );
 308         }
 309     }
 310
 311     private static void transferVMProps(AuthzEnv env, String ... props) {
 312         String value;
 313         for (String key : props) {
 314             if ((value = System.getProperty(key))!=null) {
 315                 env.setProperty(key, value);
 316             }
 317         }
 318     }
 319
 320     // IMPORTANT! VALIDATE Organization isUser method
 321     protected void checkOrganizationAcccess(AuthzTrans trans, Question q) throws APIException, OrganizationException {
 322             Set<String> testUsers = new HashSet<>();
 323             Result<List<RoleDAO.Data>> rrd = q.roleDAO.readNS(trans, rootNs);
 324             if (rrd.isOK()) {
 325                 for (RoleDAO.Data r : rrd.value) {
 326                     Result<List<UserRoleDAO.Data>> rur = q.userRoleDAO.readByRole(trans, r.fullName());
 327                     if (!rur.isOK()) {
 328                         continue;
 329                     }
 330                     for (UserRoleDAO.Data udd : rur.value) {
 331                         testUsers.add(udd.user);
 332                     }
 333                 }
 334                 if (testUsers.size() < 2) {
 335                     throw new APIException("Not enough Users in Roles for " + rootNs + " to Validate");
 336                 }
 337
 338                 Identity iden;
 339                 for (String user : testUsers) {
 340                     if ((iden = org.getIdentity(trans, user)) == null) {
 341                         throw new APIException("Failed Organization Entity Validation Check: " + user);
 342                     } else {
 343                         trans.info().log("Organization Validation Check: " + iden.id());
 344                     }
 345                 }
 346             }
 347         }
 348
 349     protected static String logDir() {
 350         String ld = env.getProperty(LOG_DIR);
 351         if (ld==null) {
 352             if (batchEnv==null) { // Deployed Batch doesn't use different ENVs, and a common logdir
 353                 ld = "logs/";
 354             } else {
 355                 ld = "logs/"+batchEnv;
 356             }
 357         }
 358         return ld;
 359     }
 360     protected int count(String str, char c) {
 361         if (str==null || str.isEmpty()) {
 362             return 0;
 363         } else {
 364             int count=1;
 365             for (int i=str.indexOf(c);i>=0;i=str.indexOf(c,i+1)) {
 366                 ++count;
 367             }
 368             return count;
 369         }
 370     }
 371
 372     public final void close(AuthzTrans trans) {
 373         _close(trans);
 374         cluster.close();
 375     }
 376
 377     public static void main(String[] args) {
 378         PropAccess access = new PropAccess(args);
 379         InputStream is = null;
 380         String filename;
 381         String propLoc;
 382         try {
 383             Define.set(access);
 384             rootNs =Define.ROOT_NS();
 385             if(access.getProperty(Config.CADI_PROP_FILES)==null) {
 386                     File f = new File("authBatch.props");
 387                     try {
 388                         if (f.exists()) {
 389                             filename = f.getAbsolutePath();
 390                             is = new FileInputStream(f);
 391                             propLoc = f.getPath();
 392                         } else {
 393                             URL rsrc = ClassLoader.getSystemResource("authBatch.props");
 394                             filename = rsrc.toString();
 395                             is = rsrc.openStream();
 396                             propLoc = rsrc.getPath();
 397                         }
 398                         access.load(is);
 399                     } finally {
 400                         if (is == null) {
 401                             System.err.println("authBatch.props must exist in current dir, or in Classpath");
 402                             System.exit(1);
 403                         }
 404                         is.close();
 405                     }
 406                     if (filename != null) {
 407                         access.log(Level.INFO,"Instantiated properties from", filename);
 408                     }
 409
 410                     // Log where Config found
 411                     access.log(Level.INFO,"Configuring from", propLoc);
 412
 413             }
 414             env = new AuthzEnv(access);
 415
 416             transferVMProps(env, CASS_ENV, "DRY_RUN", "NS", "Organization");
 417
 418             // Flow all Env Logs to Log4j, with ENV
 419
 420 //            LogFileNamer lfn;
 421 //            lfn = new LogFileNamer(logDir(),"").noPID();
 422 //            lfn.setAppender("authz-batch");
 423 //            lfn.setAppender("aspr|ASPR");
 424 //            lfn.setAppender("sync");
 425 //            lfn.setAppender("jobchange");
 426 //            lfn.setAppender("validateuser");
 427 //            aspr = Logger.getLogger("aspr");
 428 //            Log4JLogTarget.setLog4JEnv("authz-batch", env);
 429 //            propLoc = null;
 430
 431             Batch batch = null;
 432             // setup ATTUser and Organization Slots before starting this:
 433             // TODO redo this
 434             // env.slot(ATT.ATT_USERSLOT);
 435             //
 436             // OrganizationFactory.setDefaultOrg(env, ATT.class.getName());
 437             AuthzTrans trans = env.newTrans();
 438
 439             TimeTaken tt = trans.start("Total Run", Env.SUB);
 440             try {
 441                 int len = args.length;
 442                 if (len > 0) {
 443                     String toolName = args[0];
 444                     len -= 1;
 445                     if (len < 0)
 446                         len = 0;
 447                     String nargs[] = new String[len];
 448                     if (len > 0) {
 449                         System.arraycopy(args, 1, nargs, 0, len);
 450                     }
 451
 452                     env.put(ssargs = env.staticSlot("ARGS"), nargs);
 453
 454                     /*
 455                      * Add New Batch Programs (inherit from Batch) here
 456                      */
 457
 458                     // Might be a Report, Update or Temp Batch
 459                     Class<?> cls;
 460                     String classifier = "";
 461                     try {
 462                         cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.update." + toolName);
 463                         classifier = "Update:";
 464                     } catch (ClassNotFoundException e) {
 465                         try {
 466                             cls = ClassLoader.getSystemClassLoader().loadClass("org.onap.aaf.auth.reports." + toolName);
 467                             classifier = "Report:";
 468                         } catch (ClassNotFoundException e2) {
 469                             try {
 470                                 cls = ClassLoader.getSystemClassLoader()
 471                                         .loadClass("org.onap.aaf.auth.temp." + toolName);
 472                                 classifier = "Temp Utility:";
 473                             } catch (ClassNotFoundException e3) {
 474                                 cls = null;
 475                             }
 476                         }
 477                     }
 478                     if (cls != null) {
 479                         Constructor<?> cnst = cls.getConstructor(new Class[] { AuthzTrans.class });
 480                         batch = (Batch) cnst.newInstance(trans);
 481                         env.info().log("Begin", classifier, toolName);
 482                     }
 483
 484
 485                     if (batch == null) {
 486                         trans.error().log("No Batch named", toolName, "found");
 487                     }
 488                     /*
 489                      * End New Batch Programs (inherit from Batch) here
 490                      */
 491
 492                 }
 493                 if (batch != null) {
 494                     batch.run(trans);
 495                 }
 496             } finally {
 497                 tt.done();
 498                 if (batch != null) {
 499                     batch.close(trans);
 500                 }
 501                 StringBuilder sb = new StringBuilder("Task Times\n");
 502                 trans.auditTrail(4, sb, AuthzTrans.SUB, AuthzTrans.REMOTE);
 503                 trans.info().log(sb);
 504             }
 505         } catch (Exception e) {
 506             e.printStackTrace(System.err);
 507             // Exceptions thrown by DB aren't stopping the whole process.
 508             System.exit(1);
 509         }
 510     }
 511
 512 }
 513