ECOMP-PDP/src/test/java/org/openecomp/policy/pdp/test/custom/TestBase.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ECOMP-PDP
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.policy.pdp.test.custom;
  22
  23 import java.io.BufferedReader;
  24 import java.io.ByteArrayInputStream;
  25 import java.io.IOException;
  26 import java.io.InputStream;
  27 import java.io.InputStreamReader;
  28 import java.io.OutputStream;
  29 import java.net.HttpURLConnection;
  30 import java.net.MalformedURLException;
  31 import java.net.URL;
  32 import java.nio.file.FileVisitResult;
  33 import java.nio.file.Files;
  34 import java.nio.file.Path;
  35 import java.nio.file.Paths;
  36 import java.nio.file.SimpleFileVisitor;
  37 import java.nio.file.attribute.BasicFileAttributes;
  38 import java.util.ArrayList;
  39 import java.util.List;
  40 import java.util.regex.Matcher;
  41 import java.util.regex.Pattern;
  42
  43 import org.apache.commons.cli.CommandLine;
  44 import org.apache.commons.cli.GnuParser;
  45 import org.apache.commons.cli.HelpFormatter;
  46 import org.apache.commons.cli.Option;
  47 import org.apache.commons.cli.Options;
  48 import org.apache.commons.cli.ParseException;
  49 import org.apache.commons.io.IOUtils;
  50 import org.apache.http.entity.ContentType;
  51 import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
  52 import org.openecomp.policy.common.logging.flexlogger.Logger;
  53
  54 import com.att.research.xacml.api.AttributeValue;
  55 import com.att.research.xacml.api.DataType;
  56 import com.att.research.xacml.api.DataTypeException;
  57 import com.att.research.xacml.api.DataTypeFactory;
  58 import com.att.research.xacml.api.Decision;
  59 import com.att.research.xacml.api.Identifier;
  60 import com.att.research.xacml.api.Request;
  61 import com.att.research.xacml.api.RequestAttributes;
  62 import com.att.research.xacml.api.Response;
  63 import com.att.research.xacml.api.Result;
  64 import com.att.research.xacml.api.pdp.PDPEngine;
  65 import com.att.research.xacml.api.pdp.PDPEngineFactory;
  66 import com.att.research.xacml.api.pdp.PDPException;
  67 import com.att.research.xacml.api.pep.PEPException;
  68 import com.att.research.xacml.std.IdentifierImpl;
  69 import com.att.research.xacml.std.StdAttributeValue;
  70 import com.att.research.xacml.std.StdMutableAttribute;
  71 import com.att.research.xacml.std.StdMutableRequest;
  72 import com.att.research.xacml.std.StdMutableRequestAttributes;
  73 import com.att.research.xacml.std.dom.DOMRequest;
  74 import com.att.research.xacml.std.dom.DOMResponse;
  75 import com.att.research.xacml.std.dom.DOMStructureException;
  76 import com.att.research.xacml.std.json.JSONRequest;
  77 import com.att.research.xacml.std.json.JSONResponse;
  78 import com.att.research.xacml.std.json.JSONStructureException;
  79 import com.att.research.xacml.util.FactoryException;
  80 import com.att.research.xacml.util.XACMLProperties;
  81 import com.google.common.base.Splitter;
  82 import com.google.common.collect.Lists;
  83
  84 /**
  85  * This is a base class for setting up a test environment. Using properties files, it contains the
  86  * necessary information for
  87  * 1. defining and providing attributes
  88  * 2. defining and instantiating the PDP engine
  89  * 3. creating PEP requests and calling the PDP engine
  90  *
  91  *
  92  */
  93 public class TestBase extends SimpleFileVisitor<Path> {
  94         private static final Logger logger      = FlexLogger.getLogger(TestBase.class);
  95
  96         public class HelpException extends Exception {
  97                 private static final long serialVersionUID = 1L;
  98
  99         }
 100
 101         /**
 102          * This private class holds information for properties defined for attribute
 103          * generation. The user can configure the properties file such that attributes
 104          * can be automatically generated and added into each request.
 105          *
 106          *
 107          */
 108         class Generator {
 109                 Path file;
 110                 InputStream is;
 111                 BufferedReader reader;
 112                 List<StdMutableAttribute> attributes = new ArrayList<StdMutableAttribute>();
 113
 114                 public Generator(Path path) {
 115                         this.file = path;
 116                 }
 117
 118                 /**
 119                  * read - reads in the next line of data
 120                  *
 121                  * @return      String - a line from the csv containing attribute data
 122                  */
 123                 public String   read() {
 124                         String str = null;
 125                         if (is == null) {
 126                                 try {
 127                                         is = Files.newInputStream(file);
 128                                 } catch (IOException e) {
 129                                         logger.error(e);
 130                                         return null;
 131                                 }
 132                         }
 133                         if (reader == null) {
 134                                 reader = new BufferedReader(new InputStreamReader(this.is));
 135                         }
 136                         try {
 137                                 str = reader.readLine();
 138                                 if (str == null) {
 139                                         //
 140                                         // No more strings, close up
 141                                         //
 142                                         this.close();
 143                                 }
 144                                 if (logger.isDebugEnabled()) {
 145                                         logger.debug(str);
 146                                 }
 147                         } catch (IOException e) {
 148                                 logger.error(e);
 149                         }
 150                         return str;
 151                 }
 152
 153                 public void     close() {
 154                         if (this.reader != null) {
 155                                 try {
 156                                         this.reader.close();
 157                                 } catch (IOException idontcare) {
 158                                 } finally {
 159                                         this.reader = null;
 160                                         this.is = null;
 161                                 }
 162                         }
 163                 }
 164
 165         }
 166
 167         public static final String PROP_GENERATOR = "xacml.attribute.generator";
 168
 169         public static final String OPTION_HELP = "help";
 170         public static final String OPTION_TESTDIR = "dir";
 171         public static final String OPTION_TESTREST = "rest";
 172         public static final String OPTION_TESTURL = "url";
 173         public static final String OPTION_TESTOUTPUT = "output";
 174         public static final String OPTION_LOOP = "loop";
 175         public static final String OPTION_TESTNUMBERS = "testNumbers";
 176
 177         public static final String DEFAULT_RESTURL = "https://localhost:8080/pdp/";     // Modified for test purpose. Port no. 8443 to 8080
 178
 179         public static Options options = new Options();
 180         static {
 181                 options.addOption(new Option(OPTION_HELP, false, "Prints help."));
 182                 options.addOption(new Option(OPTION_TESTDIR, true, "Directory path where all the test properties and data are located."));
 183                 options.addOption(new Option(OPTION_TESTREST, false, "Test against RESTful PDP."));
 184                 options.addOption(new Option(OPTION_TESTURL, true, "URL to the RESTful PDP. Default is " + DEFAULT_RESTURL));
 185                 options.addOption(new Option(OPTION_TESTOUTPUT, true, "Specify a different location for dumping responses."));
 186                 options.addOption(new Option(OPTION_LOOP, true, "Number of times to loop through the tests. Default is 1. A value of -1 runs indefinitely."));
 187                 options.addOption(new Option(OPTION_TESTNUMBERS, true, "Comma-separated list of numbers found in the names of the test files to be run.  Numbers must exactly match the file name, e.g. '02'.  Used to limit testing to specific set of tests."));
 188         }
 189
 190         protected String directory = null;
 191         protected Path output = null;
 192         protected boolean isREST;
 193         protected URL restURL = null;
 194         protected int loop = 1;
 195         protected PDPEngine engine = null;
 196         protected List<Generator> generators = new ArrayList<Generator>();
 197         protected static DataTypeFactory dataTypeFactory                = null;
 198
 199         private long    permits = 0;
 200         private long    denies = 0;
 201         private long    notapplicables = 0;
 202         private long    indeterminates = 0;
 203
 204         private long    expectedPermits = 0;
 205         private long    expectedDenies = 0;
 206         private long    expectedNotApplicables = 0;
 207         private long    expectedIndeterminates = 0;
 208
 209         private long    generatedpermits = 0;
 210         private long    generateddenies = 0;
 211         private long    generatednotapplicables = 0;
 212         private long    generatedindeterminates = 0;
 213
 214         private long    responseMatches = 0;
 215         private long    responseNotMatches = 0;
 216
 217         private String[]        testNumbersArray = null;
 218
 219         protected final Pattern pattern = Pattern.compile("Request[.]\\d+[.](Permit|Deny|NA|Indeterminate|Generate|Unknown)\\.(json|xml)");
 220
 221         public static boolean isJSON(Path file) {
 222                 return file.toString().endsWith(".json");
 223         }
 224
 225         public static boolean isXML(Path file) {
 226                 return file.toString().endsWith(".xml");
 227         }
 228
 229         public TestBase(String[] args) throws ParseException, MalformedURLException, HelpException {
 230                 //
 231                 // Finish Initialization
 232                 //
 233                 this.restURL = new URL(DEFAULT_RESTURL);
 234                 //
 235                 // Parse arguments
 236                 //
 237                 this.parseCommands(args);
 238         }
 239
 240         /**
 241          * Parse in the command line arguments that the following parameters:
 242          *
 243          * @param args - command line arguments
 244          * @throws ParseException
 245          * @throws MalformedURLException
 246          * @throws HelpException
 247          */
 248         protected void parseCommands(String[] args) throws ParseException, MalformedURLException, HelpException {
 249                 //
 250                 // Parse the command line options
 251                 //
 252                 CommandLine cl;
 253                 cl = new GnuParser().parse(options, args);
 254                 //
 255                 // Check for what we have
 256                 //
 257                 if (cl.hasOption(OPTION_HELP)) {
 258                 new HelpFormatter().printHelp("Usage: -dir testdirectory OPTIONS",
 259                                 options);
 260                 throw new HelpException();
 261                 }
 262                 if (cl.hasOption(OPTION_TESTDIR)) {
 263                         this.directory = cl.getOptionValue(OPTION_TESTDIR);
 264                 } else {
 265                         throw new IllegalArgumentException("You must specify a test directory. -dir path/to/some/where");
 266                 }
 267                 if (cl.hasOption(OPTION_TESTREST)) {
 268                         this.isREST = true;
 269                 } else {
 270                         this.isREST = false;
 271                 }
 272                 if (cl.hasOption(OPTION_TESTURL)) {
 273                         this.restURL = new URL(cl.getOptionValue(OPTION_TESTURL));
 274                 }
 275                 if (cl.hasOption(OPTION_TESTOUTPUT)) {
 276                         this.output = Paths.get(cl.getOptionValue(OPTION_TESTOUTPUT));
 277                 } else {
 278                         this.output = Paths.get(this.directory, "results");
 279                 }
 280                 if (cl.hasOption(OPTION_LOOP)) {
 281                         this.loop = Integer.parseInt(cl.getOptionValue(OPTION_LOOP));
 282                 }
 283                 if (cl.hasOption(OPTION_TESTNUMBERS)) {
 284                         String testNumberString = cl.getOptionValue(OPTION_TESTNUMBERS);
 285                         testNumbersArray = testNumberString.split(",");
 286                         //
 287                         // reset strings to include dots so they exactly match pattern in file name
 288                         //
 289                         for (int i = 0; i < testNumbersArray.length; i++) {
 290                                 testNumbersArray[i] = "." + testNumbersArray[i] + ".";
 291                         }
 292                 }
 293         }
 294
 295         /**
 296          * Using the command line options that were parsed, configures our test instance.
 297          *
 298          * @throws FactoryException
 299          */
 300         protected void configure() throws FactoryException {
 301                 //
 302                 // Setup the xacml.properties file
 303                 //
 304                 if (this.directory == null) {
 305                         throw new IllegalArgumentException("Must supply a path to a test directory.");
 306                 }
 307                 Path pathDir = Paths.get(this.directory, "xacml.properties");
 308                 if (Files.notExists(pathDir)) {
 309                         throw new IllegalArgumentException(pathDir.toString() + " does not exist.");
 310                 }
 311                 //
 312                 // Set it as the System variable so the XACML factories know where the properties are
 313                 // loaded from.
 314                 //
 315                 System.setProperty(XACMLProperties.XACML_PROPERTIES_NAME, pathDir.toString());
 316                 //
 317                 // Now we can create the data type factory
 318                 //
 319                 dataTypeFactory = DataTypeFactory.newInstance();
 320                 //
 321                 // Load in what generators we are to create
 322                 //
 323                 String generators = XACMLProperties.getProperty(PROP_GENERATOR);
 324                 if (generators != null) {
 325                         //
 326                         // Parse the generators
 327                         //
 328                         for (String generator : Splitter.on(',').trimResults().omitEmptyStrings().split(generators)) {
 329                                 this.configureGenerator(generator);
 330                         }
 331                 }
 332                 //
 333                 // If we are embedded, create our engine
 334                 //
 335                 if (this.isREST == false) {
 336                         PDPEngineFactory factory = PDPEngineFactory.newInstance();
 337                         this.engine = factory.newEngine();
 338                 }
 339                 //
 340                 // Remove all the responses from the results directory
 341                 //
 342                 this.removeResults();
 343         }
 344
 345         /**
 346          * Removes all the Response* files from the results directory.
 347          *
 348          */
 349         public void     removeResults() {
 350                 try {
 351                         //
 352                         // Determine where the results are supposed to be written to
 353                         //
 354                         Path resultsPath;
 355                         if (this.output != null) {
 356                                 resultsPath = this.output;
 357                         } else {
 358                                 resultsPath = Paths.get(this.directory.toString(), "results");
 359                         }
 360                         //
 361                         // Walk the files
 362                         //
 363                         Files.walkFileTree(resultsPath, new SimpleFileVisitor<Path>() {
 364
 365                                 @Override
 366                                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 367                                         if (file.getFileName().toString().startsWith("Response")) {
 368                                                 Files.delete(file);
 369                                         }
 370                                         return super.visitFile(file, attrs);
 371                                 }
 372                         });
 373                 } catch (IOException e) {
 374                         logger.error("Failed to removeRequests from " + this.directory + " " + e);
 375                 }
 376         }
 377
 378         /**
 379          * Configure's a specific generator instance from the properties file.
 380          *
 381          * @param generator
 382          */
 383         protected void configureGenerator(String generator) {
 384                 String prefix = PROP_GENERATOR + "." + generator;
 385                 String file = XACMLProperties.getProperty(prefix + ".file");
 386                 //
 387                 // Create a generator object
 388                 //
 389                 Generator gen = new Generator(Paths.get(this.directory, file));
 390                 this.generators.add(gen);
 391                 //
 392                 // Grab attributes
 393                 //
 394                 String attributes = XACMLProperties.getProperty(prefix + ".attributes");
 395                 for (String attribute : Splitter.on(',').trimResults().omitEmptyStrings().split(attributes)) {
 396                         String attributePrefix = prefix + ".attributes." + attribute;
 397                         //
 398                         // Create an attribute value. It is simply a placeholder for the field within
 399                         // the CSV that contains the actual attribute value. It mainly holds the data type
 400                         //
 401                         Identifier datatype = new IdentifierImpl(XACMLProperties.getProperty(attributePrefix + ".datatype"));
 402                         Integer field = Integer.parseInt(XACMLProperties.getProperty(attributePrefix + ".field"));
 403                         StdAttributeValue<?> value = new StdAttributeValue<>(datatype, field);
 404                         //
 405                         // Get the rest of the attribute properties
 406                         //
 407                         Identifier category = new IdentifierImpl(XACMLProperties.getProperty(attributePrefix + ".category"));
 408                         Identifier id = new IdentifierImpl(XACMLProperties.getProperty(attributePrefix + ".id"));
 409                         String issuer = XACMLProperties.getProperty(attributePrefix + ".issuer");
 410                         boolean include = Boolean.parseBoolean(XACMLProperties.getProperty(attributePrefix + ".include", "false"));
 411                         //
 412                         // Now we have a skeleton attribute
 413                         //
 414                         gen.attributes.add(new StdMutableAttribute(category, id, value, issuer, include));
 415                 }
 416         }
 417
 418         /**
 419          * This runs() the test instance. It first configure's itself and then walks the
 420          * requests directory issue each request to the PDP engine.
 421          *
 422          * @throws IOException
 423          * @throws FactoryException
 424          *
 425          */
 426         public void run() throws IOException, FactoryException {
 427                 //
 428                 // Configure ourselves
 429                 //
 430                 this.configure();
 431                 //
 432                 // Loop and run
 433                 //
 434                 int runs = 1;
 435                 do {
 436                         long lTimeStart = System.currentTimeMillis();
 437                         logger.info("Run number: " + runs);
 438                         //
 439                         // Walk the request directory
 440                         //
 441                         Files.walkFileTree(Paths.get(this.directory.toString(), "requests"), this);
 442                         long lTimeEnd = System.currentTimeMillis();
 443                         logger.info("Run elapsed time: " + (lTimeEnd - lTimeStart) + "ms");
 444                         //
 445                         // Dump the stats
 446                         //
 447                         this.dumpStats();
 448                         this.resetStats();
 449                         //
 450                         // Increment
 451                         //
 452                         runs++;
 453                 } while ((this.loop == -1 ? true : runs <= this.loop));
 454         }
 455
 456         @Override
 457         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 458                 //
 459                 // Sanity check the file name
 460                 //
 461                 Matcher matcher = this.pattern.matcher(file.getFileName().toString());
 462                 if (matcher.matches()) {
 463                         //
 464                         // if user has limited which files to use, check that here
 465                         //
 466                         if (testNumbersArray != null) {
 467                                 String fileNameString = file.getFileName().toString();
 468                                 boolean found = false;
 469                                 for (String numberString : testNumbersArray) {
 470                                         if (fileNameString.contains(numberString)) {
 471                                                 found = true;
 472                                                 break;
 473                                         }
 474                                 }
 475                                 if (found == false) {
 476                                         //
 477                                         // this test is not in the list to be run, so skip it
 478                                         //
 479                                         return super.visitFile(file, attrs);
 480                                 }
 481                         }
 482                         try {
 483                                 //
 484                                 // Pull what this request is supposed to be
 485                                 //
 486                                 String group = null;
 487                                 int count = matcher.groupCount();
 488                                 if (count >= 1) {
 489                                         group = matcher.group(count-1);
 490                                 }
 491                                 //
 492                                 // Send it
 493                                 //
 494                                 this.sendRequest(file, group);
 495                         } catch (Exception e) {
 496                                 logger.error(e);
 497                                 logger.error("Exception Occured"+e);
 498                         }
 499                 }
 500                 return super.visitFile(file, attrs);
 501         }
 502
 503         /**
 504          * When a request file is encountered, this method is called send the request to the PDP engine. It will also dump
 505          * the response object. If the group equals "Generate", then it will loop and send the request with generated attributes
 506          * until that list is empty.
 507          *
 508          * @param file - Request file. Eg. Request-01-Permit.json
 509          * @param group - This is the parsed out string of the request file that defines if it is a Permit/Deny/Generate etc.
 510          * @throws Exception
 511          */
 512         protected void sendRequest(Path file, String group) throws Exception {
 513                 logger.info(file.toString());
 514                 int requestCount = 0;
 515                 do {
 516                         //
 517                         // Generate the request
 518                         //
 519                         Request request = this.generateRequest(file, group);
 520                         //
 521                         // Was something generated?
 522                         //
 523                         if (request == null) {
 524                                 //
 525                                 // Get out of the loop
 526                                 //
 527                                 logger.info("NULL request generated.");
 528                                 break;
 529                         }
 530                         logger.info(request);
 531                         //
 532                         // Call the PDP
 533                         //
 534                         Response response = this.callPDP(request);
 535                         //
 536                         // Process the response
 537                         //
 538                         this.processResponse(file, request, response, group, requestCount);
 539                         //
 540                         // Is this a generated request?
 541                         //
 542                         if (group.equals("Generate")) {
 543                                 //
 544                                 // Yes, increment counter and move
 545                                 // on to the next generated request.
 546                                 //
 547                                 requestCount++;
 548                         } else {
 549                                 //
 550                                 // Nope, exit the loop
 551                                 //
 552                                 break;
 553                         }
 554                 } while (group.equals("Generate"));
 555         }
 556
 557         /**
 558          * Sends the request object to the PDP engine. Either the embedded engine or the RESTful engine.
 559          *
 560          * @param request - XACML request object
 561          * @return Response - returns the XACML response object
 562          */
 563         protected Response callPDP(Request request) {
 564                 //
 565                 // Send it to the PDP
 566                 //
 567                 Response response = null;
 568                 if (this.isREST) {
 569                         try {
 570                                 String jsonString = JSONRequest.toString(request, false);
 571                                 //
 572                                 // Call RESTful PDP
 573                                 //
 574                                 response = this.callRESTfulPDP(new ByteArrayInputStream(jsonString.getBytes()));
 575                         } catch (Exception e) {
 576                                 logger.error("Error in sending RESTful request: " + e, e);
 577                         }
 578                 } else {
 579                         //
 580                         // Embedded call to PDP
 581                         //
 582                         long lTimeStart = System.currentTimeMillis();
 583                         try {
 584                                 response = this.engine.decide(request);
 585                         } catch (PDPException e) {
 586                                 logger.error(e);
 587                         }
 588                         long lTimeEnd = System.currentTimeMillis();
 589                         logger.info("Elapsed Time: " + (lTimeEnd - lTimeStart) + "ms");
 590                 }
 591                 return response;
 592         }
 593
 594         /**
 595          * Reads the request file into a Request object based on its type.
 596          *
 597          * If the request has "Generate" in its filename, then this function will add
 598          * generated attributes into the request.
 599          *
 600          * @param file - Request file. Eg. Request-01-Permit.json
 601          * @param group - This is the parsed out string of the request file that defines if it is a Permit/Deny/Generate etc.
 602          * @return
 603          * @throws JSONStructureException
 604          * @throws DOMStructureException
 605          * @throws PEPException
 606          */
 607         protected Request generateRequest(Path file, String group) throws JSONStructureException, DOMStructureException, PEPException {
 608                 //
 609                 // Convert to a XACML Request Object
 610                 //
 611                 Request request = null;
 612                 if (TestBase.isJSON(file)) {
 613                         request = JSONRequest.load(file.toFile());
 614                 } else if (TestBase.isXML(file)) {
 615                         request = DOMRequest.load(file.toFile());
 616                 }
 617                 if (request == null) {
 618                         throw new PEPException("Invalid Request File: " + file.toString());
 619                 }
 620                 //
 621                 // Only if this request has "Generate"
 622                 // Request.XX.Generate.[json|xml]
 623                 //
 624                 if (group.equals("Generate")) {
 625                         //
 626                         // Add attributes to it
 627                         //
 628                         request = this.onNextRequest(request);
 629                 }
 630                 //
 631                 // Done
 632                 //
 633                 return request;
 634         }
 635
 636         /**
 637          * Called to add in generated attributes into the request.
 638          *
 639          * @param request
 640          * @return
 641          */
 642         protected Request onNextRequest(Request request) {
 643                 //
 644                 // If we have no generators, just return
 645                 //
 646                 if (this.generators.isEmpty()) {
 647                         return request;
 648                 }
 649                 //
 650                 // Copy the request attributes
 651                 //
 652                 List<StdMutableRequestAttributes> attributes = new ArrayList<StdMutableRequestAttributes>();
 653                 for (RequestAttributes a : request.getRequestAttributes()) {
 654                         attributes.add(new StdMutableRequestAttributes(a));
 655                 }
 656                 //
 657                 // Iterate the generators
 658                 //
 659                 for (Generator generator : this.generators) {
 660                         //
 661                         // Read a row in
 662                         //
 663                         String line = generator.read();
 664                         //
 665                         // Was something read?
 666                         //
 667                         if (line == null) {
 668                                 //
 669                                 // No more rows to read, return null
 670                                 //
 671                                 return null;
 672                         }
 673                         //
 674                         // Split the line
 675                         //
 676                         List<String> fields = Lists.newArrayList(Splitter.on(',').trimResults().split(line));
 677                         //
 678                         // Now work on the attributes
 679                         //
 680                         for (StdMutableAttribute attribute : generator.attributes) {
 681                                 //
 682                                 // Grab the attribute holder, which holds the datatype and field. There should
 683                                 // be only ONE object in the collection.
 684                                 //
 685                                 AttributeValue<?> value = attribute.getValues().iterator().next();
 686                                 Integer field = (Integer) value.getValue();
 687                                 //
 688                                 // Is the field number valid?
 689                                 //
 690                                 if (field >= fields.size()) {
 691                                         logger.error("Not enough fields: " + field + "(" + fields.size() + ")");
 692                                         return null;
 693                                 }
 694                                 //
 695                                 // Determine what datatype it is
 696                                 //
 697                                 DataType<?> dataTypeExtended    = dataTypeFactory.getDataType(value.getDataTypeId());
 698                                 if (dataTypeExtended == null) {
 699                                         logger.error("Failed to determine datatype");
 700                                         return null;
 701                                 }
 702                                 //
 703                                 // Create the attribute value
 704                                 //
 705                                 try {
 706                                         AttributeValue<?> attributeValue = dataTypeExtended.createAttributeValue(fields.get(field));
 707                                         //
 708                                         // Create the attribute
 709                                         //
 710                                         StdMutableAttribute newAttribute = new StdMutableAttribute(attribute.getCategory(),
 711                                                                                                                                                                 attribute.getAttributeId(),
 712                                                                                                                                                                 attributeValue,
 713                                                                                                                                                                 attribute.getIssuer(),
 714                                                                                                                                                                 attribute.getIncludeInResults());
 715                                         boolean added = false;
 716                                         for (StdMutableRequestAttributes a : attributes) {
 717                                                 //
 718                                                 // Does the category exist?
 719                                                 //
 720                                                 if (a.getCategory().equals(attribute.getCategory())) {
 721                                                         //
 722                                                         // Yes - add in the new attribute value
 723                                                         //
 724                                                         a.add(newAttribute);
 725                                                         added = true;
 726                                                         break;
 727                                                 }
 728                                         }
 729                                         if (added == false) {
 730                                                 //
 731                                                 // New category - create it and add it in
 732                                                 //
 733                                                 StdMutableRequestAttributes a = new StdMutableRequestAttributes();
 734                                                 a.setCategory(newAttribute.getCategory());
 735                                                 a.add(newAttribute);
 736                                                 attributes.add(a);
 737                                         }
 738                                 } catch (DataTypeException e) {
 739                                         logger.error(e);
 740                                         return null;
 741                                 }
 742                         }
 743                 }
 744                 //
 745                 // Now form our final request
 746                 //
 747                 StdMutableRequest newRequest = new StdMutableRequest();
 748                 newRequest.setCombinedDecision(request.getCombinedDecision());
 749                 newRequest.setRequestDefaults(request.getRequestDefaults());
 750                 newRequest.setReturnPolicyIdList(request.getReturnPolicyIdList());
 751                 newRequest.setStatus(request.getStatus());
 752                 for (StdMutableRequestAttributes a : attributes) {
 753                         newRequest.add(a);
 754                 }
 755                 return newRequest;
 756         }
 757
 758         /**
 759          * This makes an HTTP POST call to a running PDP RESTful servlet to get a decision.
 760          *
 761          * @param file
 762          * @return
 763          */
 764         protected Response callRESTfulPDP(InputStream is) {
 765                 Response response = null;
 766                 HttpURLConnection connection = null;
 767                 try {
 768
 769                         //
 770                         // Open up the connection
 771                         //
 772                         connection = (HttpURLConnection) this.restURL.openConnection();
 773                         connection.setRequestProperty("Content-Type", "application/json");
 774                         //
 775                         // Setup our method and headers
 776                         //
 777             connection.setRequestMethod("POST");
 778             connection.setUseCaches(false);
 779             //
 780             // Adding this in. It seems the HttpUrlConnection class does NOT
 781             // properly forward our headers for POST re-direction. It does so
 782             // for a GET re-direction.
 783             //
 784             // So we need to handle this ourselves.
 785             //
 786             connection.setInstanceFollowRedirects(false);
 787                         connection.setDoOutput(true);
 788                         connection.setDoInput(true);
 789                         //
 790                         // Send the request
 791                         //
 792                         try (OutputStream os = connection.getOutputStream()) {
 793                                 IOUtils.copy(is, os);
 794                         }
 795             //
 796             // Do the connect
 797             //
 798             connection.connect();
 799             if (connection.getResponseCode() == 200) {
 800                 //
 801                 // Read the response
 802                 //
 803                         ContentType contentType = null;
 804                         try {
 805                                 contentType = ContentType.parse(connection.getContentType());
 806
 807                                 if (contentType.getMimeType().equalsIgnoreCase(ContentType.APPLICATION_JSON.getMimeType())) {
 808                                 response = JSONResponse.load(connection.getInputStream());
 809                                 } else if (contentType.getMimeType().equalsIgnoreCase(ContentType.APPLICATION_XML.getMimeType()) ||
 810                                                 contentType.getMimeType().equalsIgnoreCase("application/xacml+xml") ) {
 811                                 response = DOMResponse.load(connection.getInputStream());
 812                                 } else {
 813                                 logger.error("unknown content-type: " + contentType);
 814                         }
 815
 816                 } catch (Exception e) {
 817                                 String message = "Parsing Content-Type: " + connection.getContentType() + ", error=" + e.getMessage();
 818                                 logger.error(message, e);
 819                         }
 820
 821             } else {
 822                 logger.error(connection.getResponseCode() + " " + connection.getResponseMessage());
 823             }
 824                 } catch (Exception e) {
 825                         logger.error(e);
 826                 }
 827
 828                 return response;
 829         }
 830
 831         /**
 832          * This processes a response. Saves the response out to disk. If there is a corresponding response file for the request located
 833          * in the "responses" sub-directory, then this method will compare that response file with what the engine returned to see if it
 834          * matched.
 835          *
 836          * @param requestFile
 837          * @param request
 838          * @param response
 839          * @param group
 840          * @param count
 841          * @throws Exception
 842          */
 843         protected void processResponse(Path requestFile, Request request, Response response, String group, int count) throws Exception {
 844                 //
 845                 // Construct the output filename
 846                 //
 847                 Path responseFile = null;
 848                 Path resultFile = null;
 849                 int num = requestFile.getNameCount();
 850                 if (num < 2) {
 851                         logger.error("Too few dir's in request filename.");
 852                         throw new Exception("Too few dir's in request filename. Format should be Request.[0-9]+.{Permit|Deny|NA|Indeterminate}.{json|xml}");
 853                 }
 854                 String filename = requestFile.getFileName().toString();
 855                 if (group.equals("Generate")) {
 856                         //
 857                         // Using count variable, construct a filename
 858                         //
 859                         //              i.e. Response.03.Generate.{count}.json
 860                         //
 861                         filename = "Response" + filename.substring(filename.indexOf('.'), filename.lastIndexOf('.')) + String.format("%03d", count) + filename.substring(filename.lastIndexOf('.'));
 862                 } else {
 863                         //
 864                         // Construct filename
 865                         //
 866                         filename = "Response" + filename.substring(filename.indexOf('.'));
 867                 }
 868                 //
 869                 // Determine equivalent response file path
 870                 //
 871                 responseFile = Paths.get(requestFile.subpath(0, num - 2).toString(), "responses");
 872                 if (Files.notExists(responseFile)) {
 873                         //
 874                         // Create it
 875                         //
 876                         logger.warn(responseFile.toString() + " does NOT exist, creating...");
 877                         try {
 878                                 Files.createDirectories(responseFile);
 879                         } catch (IOException e) {
 880                                 logger.error(e);
 881                                 throw new Exception("Cannot proceed without an output directory.");
 882                         }
 883                 }
 884                 responseFile = Paths.get(responseFile.toString(), filename);
 885                 //
 886                 // Determine path to write result file
 887                 //
 888                 if (this.output != null) {
 889                         //
 890                         // User specified an output path
 891                         //
 892                         resultFile = this.output;
 893                 } else {
 894                         //
 895                         // Default path
 896                         //
 897                         resultFile = Paths.get(requestFile.subpath(0, num - 2).toString(), "results");
 898                 }
 899                 //
 900                 // Check if the path exists
 901                 //
 902                 if (Files.notExists(resultFile)) {
 903                         //
 904                         // Create it
 905                         //
 906                         logger.warn(resultFile.toString() + " does NOT exist, creating...");
 907                         try {
 908                                 Files.createDirectories(resultFile);
 909                         } catch (IOException e) {
 910                                 logger.error(e);
 911                                 throw new Exception("Cannot proceed without an output directory.");
 912                         }
 913                 }
 914                 //
 915                 // Add the filename to the path
 916                 //
 917                 resultFile = Paths.get(resultFile.toString(), filename);
 918                 //
 919                 // Check if there is an equivalent response in the response
 920                 // directory. If so, compare our response result with that one.
 921                 //
 922                 boolean succeeded = true;
 923                 if (responseFile != null && Files.exists(responseFile)) {
 924                         //
 925                         // Do comparison
 926                         //
 927                         Response expectedResponse = null;
 928                         if (TestBase.isJSON(responseFile)) {
 929                                 expectedResponse = JSONResponse.load(responseFile);
 930                         } else if (TestBase.isXML(responseFile)) {
 931                                 expectedResponse = DOMResponse.load(responseFile);
 932                         }
 933                         if (expectedResponse != null) {
 934                                 //
 935                                 // Do the compare
 936                                 //
 937                                 if (response == null) {
 938                                         logger.error("NULL response returned.");
 939                                         this.responseNotMatches++;
 940                                         succeeded = false;
 941                                 } else {
 942                                         if (response.equals(expectedResponse)) {
 943                                                 logger.info("Response matches expected response.");
 944                                                 this.responseMatches++;
 945                                         } else {
 946                                                 logger.error("Response does not match expected response.");
 947                                                 logger.error("Expected: ");
 948                                                 logger.error(expectedResponse.toString());
 949                                                 this.responseNotMatches++;
 950                                                 succeeded = false;
 951                                         }
 952                                 }
 953                         }
 954                 }
 955                 //
 956                 // Write the response to the result file
 957                 //
 958                 logger.info("Request: " + requestFile.getFileName() + " response is: " + (response == null ? "null" : response.toString()));
 959                 if (resultFile != null && response != null) {
 960                         if (TestBase.isJSON(resultFile)) {
 961                                 Files.write(resultFile, JSONResponse.toString(response, true).getBytes());
 962                         } else if (TestBase.isXML(resultFile)) {
 963                                 Files.write(resultFile, DOMResponse.toString(response, true).getBytes());
 964                         }
 965                 }
 966                 //
 967                 // Stats
 968                 //
 969                 if (group.equals("Permit")) {
 970                         this.expectedPermits++;
 971                 } else if (group.equals("Deny")) {
 972                         this.expectedDenies++;
 973                 } else if (group.equals("NA")) {
 974                         this.expectedNotApplicables++;
 975                 } else if (group.equals("Indeterminate")) {
 976                         this.expectedIndeterminates++;
 977                 }
 978                 if (response != null) {
 979                         for (Result result : response.getResults()) {
 980                                 Decision decision = result.getDecision();
 981                                 if (group.equals("Generate")) {
 982                                         if (decision.equals(Decision.PERMIT)) {
 983                                                 this.generatedpermits++;
 984                                         } else if (decision.equals(Decision.DENY)) {
 985                                                 this.generateddenies++;
 986                                         } else if (decision.equals(Decision.NOTAPPLICABLE)) {
 987                                                 this.generatednotapplicables++;
 988                                         } else if (decision.equals(Decision.INDETERMINATE)) {
 989                                                 this.generatedindeterminates++;
 990                                         }
 991                                         continue;
 992                                 }
 993                                 if (decision.equals(Decision.PERMIT)) {
 994                                         this.permits++;
 995                                         if (group.equals("Permit") == false) {
 996                                                 succeeded = false;
 997                                                 logger.error("Expected " + group + " got " + decision);
 998                                         }
 999                                 } else if (decision.equals(Decision.DENY)) {
1000                                         this.denies++;
1001                                         if (group.equals("Deny") == false) {
1002                                                 succeeded = false;
1003                                                 logger.error("Expected " + group + " got " + decision);
1004                                         }
1005                                 } else if (decision.equals(Decision.NOTAPPLICABLE)) {
1006                                         this.notapplicables++;
1007                                         if (group.equals("NA") == false) {
1008                                                 succeeded = false;
1009                                                 logger.error("Expected " + group + " got " + decision);
1010                                         }
1011                                 } else if (decision.equals(Decision.INDETERMINATE)) {
1012                                         this.indeterminates++;
1013                                         if (group.equals("Indeterminate") == false) {
1014                                                 succeeded = false;
1015                                                 logger.error("Expected " + group + " got " + decision);
1016                                         }
1017                                 }
1018                         }
1019                 }
1020                 if (succeeded) {
1021                         logger.info("REQUEST SUCCEEDED");
1022                 } else {
1023                         logger.info("REQUEST FAILED");
1024                 }
1025         }
1026
1027         protected void  dumpStats() {
1028                 StringBuilder dump = new StringBuilder();
1029                 dump.append(System.lineSeparator());
1030                 dump.append("Permits: " + this.permits + " Expected: " + this.expectedPermits);
1031                 dump.append(System.lineSeparator());
1032                 dump.append("Denies: " + this.denies + " Expected: " + this.expectedDenies);
1033                 dump.append(System.lineSeparator());
1034                 dump.append("NA: " + this.notapplicables + " Expected: " + this.expectedNotApplicables);
1035                 dump.append(System.lineSeparator());
1036                 dump.append("Indeterminates: " + this.indeterminates + " Expected: " + this.expectedIndeterminates);
1037                 dump.append(System.lineSeparator());
1038                 dump.append("Generated Permits: " + this.generatedpermits);
1039                 dump.append(System.lineSeparator());
1040                 dump.append("Generated Denies: " + this.generateddenies);
1041                 dump.append(System.lineSeparator());
1042                 dump.append("Generated NA: " + this.generatednotapplicables);
1043                 dump.append(System.lineSeparator());
1044                 dump.append("Generated Indeterminates: " + this.generatedindeterminates);
1045                 dump.append(System.lineSeparator());
1046                 dump.append("Responses Matched: " + this.responseMatches);
1047                 dump.append(System.lineSeparator());
1048                 dump.append("Responses NOT Matched: " + this.responseNotMatches);
1049
1050                 if (this.permits != this.expectedPermits ||
1051                         this.denies != this.expectedDenies ||
1052                         this.notapplicables != this.expectedNotApplicables ||
1053                         this.indeterminates != this.expectedIndeterminates ||
1054                         this.responseNotMatches > 0) {
1055                         logger.error(dump.toString());
1056                 } else {
1057                         logger.info(dump.toString());
1058                 }
1059         }
1060
1061         protected void  resetStats() {
1062                 this.permits = 0;
1063                 this.denies = 0;
1064                 this.notapplicables = 0;
1065                 this.indeterminates = 0;
1066                 this.generatedpermits = 0;
1067                 this.generateddenies = 0;
1068                 this.generatednotapplicables = 0;
1069                 this.generatedindeterminates = 0;
1070                 this.responseMatches = 0;
1071                 this.responseNotMatches = 0;
1072         }
1073
1074         public static void main(String[] args) {
1075                 try {
1076                         new TestBase(args).run();
1077                 } catch (ParseException | IOException | FactoryException e) {
1078                         logger.error(e);
1079                 } catch (HelpException e) {
1080                 }
1081         }
1082 }