ECOMP-PDP/src/test/java/org/openecomp/policy/pdp/test/conformance/Conformance.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ECOMP-PDP
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.policy.pdp.test.conformance;
  22
  23 import java.io.File;
  24 import java.io.FileOutputStream;
  25 import java.io.IOException;
  26 import java.io.PrintWriter;
  27 import java.net.URI;
  28 import java.util.ArrayList;
  29 import java.util.Collection;
  30 import java.util.Iterator;
  31 import java.util.List;
  32
  33 import com.att.research.xacml.api.Advice;
  34 import com.att.research.xacml.api.Attribute;
  35 import com.att.research.xacml.api.AttributeAssignment;
  36 import com.att.research.xacml.api.AttributeCategory;
  37 import com.att.research.xacml.api.AttributeValue;
  38 import com.att.research.xacml.api.IdReference;
  39 import com.att.research.xacml.api.Obligation;
  40 import com.att.research.xacml.api.Response;
  41 import com.att.research.xacml.api.Result;
  42
  43 /**
  44  * Conformance is an application that runs a <code>ConformanceTestSet</code> and dumps results comparing the actual and
  45  * expected results.
  46  *
  47  * TO RUN in Eclipse:
  48  * This is run as a Java Application.
  49  * You must first create a Run/Debug Configuration:
  50  *              Under the Argument tab, in Program Arguments you must set the -i or --input command line argument.
  51  *              You should also direct the output to a file using -o or --output. (default is Console)
  52  *              See the init() method in this file for other useful arguments.
  53  *              Example for a Windows machine:
  54  *                      -i testsets/conformance/xacml3.0-ct-v.0.4
  55  *                      -o \Users\yourLogin\Downloads\conformance.txt
  56  *              You must also set the VM arguments:
  57  *                      -Dxacml.properties=testsets/conformance/xacml.properties .
  58  *                      -Dlog4j.configuration=.\logging.properties
  59  *
  60  * @version $Revision: 1.2 $
  61  */
  62 public class Conformance {
  63         private ConformanceScopeResolver scopeResolver;
  64         private ConformanceTestEngine testEngine;
  65         private ConformanceTestSet testSet                      = new ConformanceTestSet();
  66         private File outputFile;
  67         private PrintWriter outputFileWriter;
  68
  69         private List<String> testNamesToRun = new ArrayList<String>();
  70
  71         private boolean verbose;
  72         private boolean failuresOnly;
  73         private boolean strict;
  74         private boolean stopOnFirstError;
  75
  76         private int testsRun;
  77         private int decisionsMatch;
  78         private int statusCodesMatch;
  79         private int attributesMatch;
  80         private int policyIdsMatch;
  81         private int policySetIdsMatch;
  82         private int associatedAdviceMatch;
  83         private int obligationsMatch;
  84         private int unknownFunctions;
  85
  86
  87
  88
  89         protected synchronized ConformanceScopeResolver getScopeResolver() {
  90                 if (this.scopeResolver == null) {
  91                         this.scopeResolver      = new ConformanceScopeResolver();
  92
  93                         /*
  94                          * TODO:
  95                          * Add the known scopes for the 2.0 conformance test.  This could be made more general by allowing loading
  96                          * from a properties file eventually.
  97                          */
  98                         try {
  99                                 URI ID_SCOPE_ROOT       = new URI("urn:root");
 100                                 URI ID_SCOPE_CHILD1     = new URI("urn:root:child1");
 101                                 URI ID_SCOPE_CHILD2     = new URI("urn:root:child2");
 102                                 URI ID_SCOPE_C1D1       = new URI("urn:root:child1:descendant1");
 103                                 URI ID_SCOPE_C1D2       = new URI("urn:root:child1:descendant2");
 104                                 URI ID_SCOPE_C2D1       = new URI("urn:root:child2:descendant1");
 105                                 URI ID_SCOPE_C2D2       = new URI("urn:root:child2:descendant2");
 106
 107                                 this.scopeResolver.add(ID_SCOPE_ROOT, ID_SCOPE_CHILD1);
 108                                 this.scopeResolver.add(ID_SCOPE_CHILD1, ID_SCOPE_C1D1);
 109                                 this.scopeResolver.add(ID_SCOPE_CHILD1, ID_SCOPE_C1D2);
 110                                 this.scopeResolver.add(ID_SCOPE_ROOT, ID_SCOPE_CHILD2);
 111                                 this.scopeResolver.add(ID_SCOPE_CHILD2, ID_SCOPE_C2D1);
 112                                 this.scopeResolver.add(ID_SCOPE_CHILD2, ID_SCOPE_C2D2);
 113                         } catch (Exception ex) {
 114                                 ex.printStackTrace(System.err);
 115                         }
 116
 117                 }
 118                 return this.scopeResolver;
 119         }
 120
 121         private void close() throws IOException {
 122                 if (this.outputFileWriter != null) {
 123                         this.outputFileWriter.close();
 124                 }
 125         }
 126
 127         private boolean init(String[] args) {
 128                 boolean lenientRequests = true;
 129                 boolean lenientPolicies = false;
 130                 // default is to not run any non-first-time iterations
 131                 int iterations                  = -1;
 132                 String testSetDirectoryNames = "";
 133                 for (int i = 0 ; i < args.length ; ) {
 134
 135                         if (args[i].equals("-h") || args[i].equals("--help") || args[i].equals("-help")) {
 136                                 printHelp();
 137                                 return false;
 138                         }
 139
 140
 141                         // where the XML Request/Response files are located
 142                         if (args[i].equals("-i") || args[i].equals("--input")) {
 143                                 i++;
 144                                 while (i < args.length && !args[i].startsWith("-")) {
 145                                         testSetDirectoryNames += " " + args[i];
 146                                         try {
 147                                                 testSet.addConformanceTestSet(ConformanceTestSet.loadDirectory(new File(args[i])));
 148                                         } catch (Exception ex) {
 149                                                 ex.printStackTrace(System.err);
 150                                                 return false;
 151                                         }
 152                                         i++;
 153                                 }
 154
 155                         // File path name where output will be put - default is stdout == Console
 156                         } else if (args[i].equals("-o") || args[i].equals("--output")) {
 157                                 if (i+1 < args.length) {
 158                                         this.outputFile = new File(args[i+1]);
 159                                         i       += 2;
 160                                 } else {
 161                                         System.err.println("Missing argument to " + args[i] + " command line option");
 162                                         return false;
 163                                 }
 164                         // A list of specific test names (e.g.: -t IIA001 IIA007 IIIE301) - default is to run all tests
 165                         } else if (args[i].equals("-t") || args[i].equals("--tests")) {
 166                                 i++;
 167                                 while (i < args.length && !args[i].startsWith("-")) {
 168                                         testNamesToRun.add(args[i]);
 169                                         i++;
 170                                 }
 171                                 if (testNamesToRun.size() == 0) {
 172                                         System.err.println("Missing test names after -t or --tests argument");
 173                                         return false;
 174                                 }
 175                         // Include full details in the response, both the expected reqsponse (from file) and the actual response
 176                         } else if (args[i].equals("-v") || args[i].equals("--verbose")) {
 177                                 this.verbose    = true;
 178                                 i++;
 179                         // Report only failures (success is silent)
 180                         } else if (args[i].equals("-f") || args[i].equals("--failures")) {
 181                                 this.failuresOnly       = true;
 182                                 i++;
 183                         // When set, the XML must not contain extra attibutes/elements.  Default is "lenient" where unexpected entries are ignored
 184                         } else if (args[i].equals("-s") || args[i].equals("--strict")) {
 185                                 this.strict     = true;
 186                                 i++;
 187                         // (self explanatory)
 188                         } else if (args[i].equals("--stop-on-error")) {
 189                                 this.stopOnFirstError   = true;
 190                                 i++;
 191                         } else if (args[i].equals("--lenient")) {
 192                                 lenientPolicies = true;
 193                                 lenientRequests = true;
 194                                 i++;
 195                         } else if (args[i].equals("--lenient-policies")) {
 196                                 lenientPolicies = true;
 197                                 i++;
 198                         } else if (args[i].equals("--lenient-requests")) {
 199                                 lenientRequests = true;
 200                                 i++;
 201                         } else if (args[i].equals("--strict-policies")) {
 202                                 lenientPolicies = false;
 203                                 i++;
 204                         } else if (args[i].equals("--strict-requests")) {
 205                                 lenientRequests = false;
 206                                 i++;
 207                         } else if (args[i].equals("--iterations")) {
 208                                 // this is a count of how many ADDITIONAL times the decide() should be called.
 209                                 // The first time decide() is called it takes a long time to set up,
 210                                 // so to get an accurate number for how fast a single Request is handled we need to ignore the time for the first run
 211                                 // and timings for 1 or more non-first-time calls to decide().
 212                                 if (i+1 < args.length) {
 213                                         try {
 214                                                 iterations      = Integer.parseInt(args[i+1]);
 215                                                 i       += 2;
 216                                         } catch (NumberFormatException ex) {
 217                                                 System.err.println("Invalid iteration count '" + args[i+1] + "'");
 218                                                 return false;
 219                                         }
 220                                 } else {
 221                                         System.err.println("Missing argument to " + args[i] + " command line option");
 222                                         return false;
 223                                 }
 224                                 if (iterations < 1) {
 225                                         System.err.println("Cannot use --iterations " + iterations + ".  Must use an integer greater than 0");
 226                                         return false;
 227                                 }
 228                         } else {
 229                                 System.err.println("Unknown command line option " + args[i]);
 230                                 return false;
 231                         }
 232                 }
 233
 234                 this.testEngine = new ConformanceTestEngine(this.getScopeResolver(), lenientRequests, lenientPolicies, iterations);
 235
 236                 if (testSetDirectoryNames.length() == 0) {
 237                         System.err.println("No test set directory given (need -i or --iniput command line option)");
 238                         return false;
 239                 }
 240                 if (testSet.getListConformanceTests().size() == 0) {
 241                         System.err.println("No tests in given directories: " + testSetDirectoryNames);
 242                 }
 243
 244                 if (testNamesToRun.size() > 0) {
 245                         String s = "";
 246                         for (String name : testNamesToRun) {
 247                                 s += ", " + name;
 248                         }
 249                         System.out.println("Tests limited to: " + s.substring(1));
 250                 }
 251
 252                 if (this.outputFile == null) {
 253                         this.outputFileWriter   = new PrintWriter(System.out);
 254                 } else {
 255                         try {
 256                                 this.outputFileWriter   = new PrintWriter(new FileOutputStream(this.outputFile));
 257                         } catch (IOException ex) {
 258                                 System.err.println("Cannot open " + this.outputFile.getAbsolutePath() + " for writing.");
 259                                 return false;
 260                         }
 261                 }
 262
 263                 return true;
 264         }
 265
 266         private void printHelp() {
 267                 System.out.println("usage: Conformance --input <tests_directory> OPTIONS");
 268                 System.out.println("");
 269                 System.out.println(" -f, --failures             Only include failed tests in the output.  \n"+
 270                                                         "                       Default is to include all test's results in the output file.");
 271                 System.out.println("");
 272                 System.out.println(" -h, --help         Prints help.");
 273
 274                 System.out.println("");
 275                 System.out.println(" -i, --input <dir>  Directory containing the XML Request/Response files.  \n"+
 276                                                         "                       This may be multiple space-separated directory paths.  REQUIRED");
 277
 278                 System.out.println("");
 279                 System.out.println(" --iterations               The number of times to run through the set of tests in the input directory.");
 280
 281                 System.out.println("");
 282                 System.out.println(" --lenient          Allow both Requests and Policies to have unexpected elements, no data in <Content>, etc. \n"+
 283                                                         "                       Default is to not allow anything that is not explicitly listed in the XACML spec.");
 284
 285                 System.out.println("");
 286                 System.out.println(" --lenient-policies Allow Policies to have unexpected elements, no data in <Content>, etc.  \n" +
 287                                                         "                       Default is to not allow anything that is not explicitly listed in the XACML spec.");
 288
 289                 System.out.println("");
 290                 System.out.println(" --lenient-requests Allow Requests to have unexpected elements, no data in <Content>, etc.  \n" +
 291                                                         "                       Default is to not allow anything that is not explicitly listed in the XACML spec.");
 292
 293                 System.out.println("");
 294                 System.out.println(" -o, --output <dir> Directory where the output results file will be put.");
 295
 296                 System.out.println("");
 297                 System.out.println(" -s, --strict               Check both the Decision and all other parts of the Response (Attributes, Obligations and Advice). \n "+
 298                                                         "                       Default is to check just the Decision.");
 299
 300                 System.out.println("");
 301                 System.out.println(" --stop-on-error    Stop running conformance tests the first time one fails.  Default is to continue through all tests.");
 302
 303                 System.out.println("");
 304                 System.out.println(" --strict-policies  Require Policies to have no unexpected elements, data in <Content>, etc.  \n" +
 305                                                         "                       This is the default, but can be used to override Policies when option --lenient is used.");
 306
 307                 System.out.println("");
 308                 System.out.println(" --strict-requests  Require Requests to have no unexpected elements, data in <Content>, etc.  \n" +
 309                                                         "                       This is the default, but can be used to override Requests when option --lenient is used.");
 310
 311                 System.out.println("");
 312                 System.out.println(" -t, --tests <list of test names>   A space-separated list of specific tests to be run. \n" +
 313                                                         "                       These are just the names of the tests as in 'IIA001 IIC178'.  \n" +
 314                                                         "                       Default is to run all tests in the input directory.");
 315
 316                 System.out.println("");
 317                 System.out.println(" -v, --verbose              The entire expected and actual Response objects in the output.  \n"+
 318                                                         "                       Default is just a summary line.");
 319
 320         }
 321
 322         private boolean failed(ConformanceTestResult conformanceTestResult) {
 323                 ResponseMatchResult responseMatchResult = conformanceTestResult.getResponseMatchResult();
 324                 if (responseMatchResult == null) {
 325                         return true;
 326                 }
 327                 if (!responseMatchResult.decisionsMatch() || !responseMatchResult.statusCodesMatch()) {
 328                         return true;
 329                 } else if (this.strict) {
 330                         if (!responseMatchResult.associatedAdviceMatches() ||
 331                                 !responseMatchResult.attributesMatch() ||
 332                                 !responseMatchResult.obligationsMatch() ||
 333                                 !responseMatchResult.policyIdentifiersMatch() ||
 334                                 !responseMatchResult.policySetIdentifiersMatch()
 335                                         ) {
 336                                 return true;
 337                         }
 338                 }
 339                 return false;
 340         }
 341
 342         private void dump(AttributeAssignment attributeAssignment) {
 343                 this.outputFileWriter.println("\t\t\t\tAttributeAssignment:");
 344                 if (attributeAssignment.getCategory() != null) {
 345                         this.outputFileWriter.println("\t\t\t\t\tCategory: " + attributeAssignment.getCategory().stringValue());
 346                 }
 347                 if (attributeAssignment.getAttributeId() != null) {
 348                         this.outputFileWriter.println("\t\t\t\t\tAttributeId: " + attributeAssignment.getAttributeId().stringValue());
 349                 }
 350                 if (attributeAssignment.getDataTypeId() != null) {
 351                         this.outputFileWriter.println("\t\t\t\t\tDataType: " + attributeAssignment.getDataTypeId().stringValue());
 352                 }
 353                 if (attributeAssignment.getIssuer() != null) {
 354                         this.outputFileWriter.println("\t\t\t\t\tIssuer: " + attributeAssignment.getIssuer());
 355                 }
 356                 if (attributeAssignment.getAttributeValue() != null && attributeAssignment.getAttributeValue().getValue() != null) {
 357                         this.outputFileWriter.println("\t\t\t\t\tValue: " + attributeAssignment.getAttributeValue().getValue().toString());
 358                 }
 359         }
 360
 361         private void dump(Attribute attribute) {
 362                 this.outputFileWriter.println("\t\t\t\t\tAttribute: " + (attribute.getAttributeId() == null ? "" : attribute.getAttributeId().stringValue()));
 363                 if (attribute.getIssuer() != null) {
 364                         this.outputFileWriter.println("\t\t\t\t\t\tIssuer: " + attribute.getIssuer());
 365                 }
 366                 Iterator<AttributeValue<?>> iterAttributeValues = attribute.getValues().iterator();
 367                 if (iterAttributeValues.hasNext()) {
 368                         this.outputFileWriter.println("\t\t\t\t\t\tValues: ");
 369                         while (iterAttributeValues.hasNext()) {
 370                                 this.outputFileWriter.print("\t\t\t\t\t\t\t");
 371                                 AttributeValue<?> attributeValue        = iterAttributeValues.next();
 372                                 if (attributeValue.getDataTypeId() != null) {
 373                                         this.outputFileWriter.print("DataType: " + attributeValue.getDataTypeId().stringValue() + " ");
 374                                 }
 375                                 if (attributeValue.getValue() != null) {
 376                                         this.outputFileWriter.print("Value: " + attributeValue.getValue().toString());
 377                                 }
 378                                 this.outputFileWriter.println();
 379                         }
 380                 }
 381         }
 382
 383         private void dump(AttributeCategory attributeCategory) {
 384                 this.outputFileWriter.println("\t\t\tAttributeCategory: " + (attributeCategory.getCategory() == null ? "" : attributeCategory.getCategory().stringValue()));
 385                 Collection<Attribute> listAttributes    = attributeCategory.getAttributes();
 386                 if (listAttributes.size() > 0) {
 387                         this.outputFileWriter.println("\t\t\t\tAttributes:");
 388                         for (Attribute attribute: listAttributes) {
 389                                 this.dump(attribute);
 390                         }
 391                 }
 392         }
 393
 394         private void dump(Result result) {
 395                 this.outputFileWriter.println("\t\t======== Result ==========");
 396                 this.outputFileWriter.println("\t\tDecision: " + (result.getDecision() == null ? "null" : result.getDecision().name()));
 397                 if (result.getStatus() == null) {
 398                         this.outputFileWriter.println("\t\tStatus: null");
 399                 } else {
 400                         this.outputFileWriter.println("\t\tStatus:");
 401                         if (result.getStatus().getStatusCode() != null) {
 402                                 this.outputFileWriter.println("\t\t\tStatusCode: " + result.getStatus().getStatusCode().toString());
 403                         }
 404                         if (result.getStatus().getStatusMessage() != null) {
 405                                 this.outputFileWriter.println("\t\t\tStatusMessage: " + result.getStatus().getStatusMessage());
 406                         }
 407                         if (result.getStatus().getStatusDetail() != null) {
 408                                 this.outputFileWriter.println("\t\t\tStatusDetail: " + result.getStatus().getStatusDetail().toString());
 409                         }
 410                 }
 411                 Collection<Advice> listAdvice   = result.getAssociatedAdvice();
 412                 if (listAdvice.size() > 0) {
 413                         this.outputFileWriter.println("\t\tAdvice:");
 414                         for (Advice advice : listAdvice) {
 415                                 if (advice.getId() != null) {
 416                                         this.outputFileWriter.println("\t\t\tId: " + advice.getId().stringValue());
 417                                 }
 418                                 Collection<AttributeAssignment> attributeAssignments    = advice.getAttributeAssignments();
 419                                 if (attributeAssignments.size() > 0) {
 420                                         this.outputFileWriter.println("\t\t\tAttributeAssignments:");
 421                                         for (AttributeAssignment attributeAssignment: attributeAssignments) {
 422                                                 this.dump(attributeAssignment);
 423                                         }
 424                                 }
 425                         }
 426                 }
 427                 Collection<Obligation> listObligations  = result.getObligations();
 428                 if (listObligations.size() > 0) {
 429                         for (Obligation obligation: listObligations) {
 430                                 if (obligation.getId() != null) {
 431                                         this.outputFileWriter.println("\t\t\tId: " + obligation.getId().stringValue());
 432                                 }
 433                                 Collection<AttributeAssignment> attributeAssignments    = obligation.getAttributeAssignments();
 434                                 if (attributeAssignments.size() > 0) {
 435                                         this.outputFileWriter.println("\t\t\tAttributeAssignments:");
 436                                         for (AttributeAssignment attributeAssignment : attributeAssignments) {
 437                                                 this.dump(attributeAssignment);
 438                                         }
 439                                 }
 440                         }
 441                 }
 442                 Collection<AttributeCategory> listAttributeCategories   = result.getAttributes();
 443                 if (listAttributeCategories.size() > 0) {
 444                         this.outputFileWriter.println("\t\tAttributes:");
 445                         for (AttributeCategory attributeCategory : listAttributeCategories) {
 446                                 this.dump(attributeCategory);
 447                         }
 448                 }
 449                 Collection<IdReference> listIdReferences;
 450                 if ((listIdReferences = result.getPolicyIdentifiers()).size() > 0) {
 451                         this.outputFileWriter.println("\t\tPolicyIds:");
 452                         for (IdReference idReference : listIdReferences) {
 453                                 this.outputFileWriter.println("\t\t\t" + idReference.toString());
 454                         }
 455                 }
 456                 if ((listIdReferences = result.getPolicySetIdentifiers()).size() > 0) {
 457                         this.outputFileWriter.println("\t\tPolicySetIds:");
 458                         for (IdReference idReference : listIdReferences) {
 459                                 this.outputFileWriter.println("\t\t\t" + idReference.toString());
 460                         }
 461                 }
 462         }
 463
 464         private void dump(String label, Response response) {
 465                 this.outputFileWriter.println("\t========== " + label + "==========");
 466                 if (response == null) {
 467                         this.outputFileWriter.println("null");
 468                         return;
 469                 }
 470
 471                 for (Result result : response.getResults()) {
 472                         this.dump(result);
 473                 }
 474         }
 475
 476         private void dump(ConformanceTestResult conformanceTestResult) {
 477
 478                 ResponseMatchResult responseMatchResult = conformanceTestResult.getResponseMatchResult();
 479                 if (this.verbose) {
 480                         this.outputFileWriter.println("========== Test " + conformanceTestResult.getConformanceTest().getTestName() + " ==========");
 481                         this.dump("Expected Response", conformanceTestResult.getExpectedResponse());
 482                         this.dump("Actual Response", conformanceTestResult.getActualResponse());
 483                         if (responseMatchResult != null) {
 484                                 this.outputFileWriter.println("\t========== Matching ==========");
 485                                 this.outputFileWriter.println("\tDecisions Match? " + responseMatchResult.decisionsMatch());
 486                                 this.outputFileWriter.println("\tStatus Codes Match? " + responseMatchResult.statusCodesMatch());
 487                                 this.outputFileWriter.println("\tAttributes Match? " + responseMatchResult.attributesMatch());
 488                                 this.outputFileWriter.println("\tPolicyIds Match? " + responseMatchResult.policyIdentifiersMatch());
 489                                 this.outputFileWriter.println("\tPolicySetIds Match? " + responseMatchResult.policySetIdentifiersMatch());
 490                                 this.outputFileWriter.println("\tAssociated Advice Match? " + responseMatchResult.associatedAdviceMatches());
 491                                 this.outputFileWriter.println("\tObligations Match? " + responseMatchResult.obligationsMatch());
 492                                 this.outputFileWriter.println("========== End ==========");
 493                         }
 494                 } else {
 495                         String testName = conformanceTestResult.getConformanceTest().getTestName();
 496                         if (responseMatchResult != null) {
 497                                 Iterator<ResultMatchResult> iterResultMatches   = responseMatchResult.getResultMatchResults();
 498                                 if (iterResultMatches == null || !iterResultMatches.hasNext()) {
 499                                         this.outputFileWriter.println(testName);
 500                                 } else {
 501                                         while (iterResultMatches.hasNext()) {
 502                                                 ResultMatchResult resultMatchResult     = iterResultMatches.next();
 503                                                 this.outputFileWriter.printf("%s,%s,%s,%s,%s,%s,%s,%s,%d,%d\n",
 504                                                                 testName,
 505                                                                 resultMatchResult.decisionsMatch(),
 506                                                                 resultMatchResult.statusCodesMatch(),
 507                                                                 resultMatchResult.attributesMatch(),
 508                                                                 resultMatchResult.policyIdentifiersMatch(),
 509                                                                 resultMatchResult.policySetIdentifiersMatch(),
 510                                                                 resultMatchResult.associatedAdviceMatches(),
 511                                                                 resultMatchResult.obligationsMatch(),
 512                                                                 conformanceTestResult.getFirstCallTime(),
 513                                                                 conformanceTestResult.getAverageTotalLoopTime()
 514                                                                 );
 515                                         }
 516                                 }
 517                         }
 518                 }
 519                 this.outputFileWriter.flush();
 520         }
 521
 522         private boolean run(ConformanceTest conformanceTest) throws Exception {
 523                 this.testsRun++;
 524                 ConformanceTestResult conformanceTestResult     = this.testEngine.run(conformanceTest);
 525                 boolean bFailed                                                         = true;
 526                 if (conformanceTestResult != null) {
 527                         ResponseMatchResult responseMatchResult = conformanceTestResult.getResponseMatchResult();
 528                         if (responseMatchResult != null) {
 529                                 if (responseMatchResult.decisionsMatch()) {
 530                                         this.decisionsMatch++;
 531                                         this.statusCodesMatch   += (responseMatchResult.statusCodesMatch() ? 1 : 0);
 532                                         this.attributesMatch    += (responseMatchResult.attributesMatch() ? 1 : 0);
 533                                         this.policyIdsMatch             += (responseMatchResult.policyIdentifiersMatch() ? 1 : 0);
 534                                         this.policySetIdsMatch  += (responseMatchResult.policySetIdentifiersMatch() ? 1 : 0);
 535                                         this.associatedAdviceMatch      += (responseMatchResult.associatedAdviceMatches() ? 1 : 0);
 536                                         this.obligationsMatch           += (responseMatchResult.obligationsMatch() ? 1 : 0);
 537                                 }
 538                                 this.unknownFunctions           += (responseMatchResult.unknownFunction() ? 1 : 0);
 539                                 bFailed = this.failed(conformanceTestResult);
 540                                 if (bFailed || !this.failuresOnly) {
 541                                         this.dump(conformanceTestResult);
 542                                 }
 543                         } else if (conformanceTestResult.getError() != null) {
 544                                 this.outputFileWriter.println(conformanceTestResult.getError());
 545                         }
 546                 }
 547                 return (!bFailed || !this.stopOnFirstError);
 548         }
 549
 550         private void run() throws Exception {
 551                 long tStart     = System.currentTimeMillis();
 552
 553                 if (!this.verbose) {
 554                         this.outputFileWriter.println("Test,Decision,Status,Attributes,PolicyIds,PolicySetIds,Advice,Obligations");
 555                 }
 556                 Iterator<ConformanceTest> iterConformanceTests  = this.testSet.getConformanceTests();
 557                 boolean bContinue                                                               = true;
 558                 while (bContinue && iterConformanceTests.hasNext()) {
 559 //                      bContinue       = this.run(iterConformanceTests.next());
 560                         ConformanceTest test = iterConformanceTests.next();
 561                         if (testNamesToRun.size() > 0) {
 562                                 if ( ! testNamesToRun.contains(test.getTestName())) {
 563                                         continue;
 564                                 }
 565                         }
 566                         bContinue       = this.run(test);
 567                 }
 568
 569                 long tElapsed   = System.currentTimeMillis() - tStart;
 570
 571                 if (this.verbose) {
 572                         this.outputFileWriter.println("Tests run = " + this.testsRun);
 573                         this.outputFileWriter.println("Decisions match = " + this.decisionsMatch);
 574                         this.outputFileWriter.println("Status Codes match = " + this.statusCodesMatch);
 575                         this.outputFileWriter.println("Attributes match = " + this.attributesMatch);
 576                         this.outputFileWriter.println("PolicyIds match = " + this.policyIdsMatch);
 577                         this.outputFileWriter.println("PolicySetIds match = " + this.policySetIdsMatch);
 578                         this.outputFileWriter.println("Associated Advice match = " + this.associatedAdviceMatch);
 579                         this.outputFileWriter.println("Obligations match = " + this.obligationsMatch);
 580                         this.outputFileWriter.println("Unknown functions = " + this.unknownFunctions);
 581                 } else {
 582                         this.outputFileWriter.printf("Total (%d),%d,%d,%d,%d,%d,%d,%d,%d\n",
 583                                         this.testsRun,
 584                                         this.decisionsMatch,
 585                                         this.statusCodesMatch,
 586                                         this.attributesMatch,
 587                                         this.policyIdsMatch,
 588                                         this.policySetIdsMatch,
 589                                         this.associatedAdviceMatch,
 590                                         this.obligationsMatch,
 591                                         this.unknownFunctions);
 592                 }
 593
 594                 if (tElapsed > 0) {
 595                         long tHours             = tElapsed / (60*60*1000);
 596                         tElapsed                = tElapsed - tHours * 60 * 60 *1000;
 597                         long tMinutes   = tElapsed / (60*1000);
 598                         tElapsed                = tElapsed - tMinutes * 60 * 1000;
 599                         long tSeconds   = tElapsed / 1000;
 600                         tElapsed                = tElapsed - tSeconds * 1000;
 601
 602                         this.outputFileWriter.printf("Elapsed time = %02d:%02d:%02d.%03d\n", tHours, tMinutes, tSeconds, tElapsed);
 603                         this.outputFileWriter.printf("First decide time in nano-seconds %d\n", this.testEngine.getFirstDecideTime());
 604                         this.outputFileWriter.printf("Total Multiple decide time in nano-seconds %d\n", this.testEngine.getDecideTimeMultiple());
 605
 606                         this.outputFileWriter.printf("\nAverage First decide time in nano-seconds %d\n", this.testEngine.getAvgFirstDecideTime());
 607                         this.outputFileWriter.printf("Average decide time after first call in nano-seconds %d\n", this.testEngine.getAvgDecideTimeMultiple());
 608                 }
 609         }
 610
 611         public Conformance() {
 612         }
 613
 614         public static void main(String[] args) {
 615                 Conformance conformance = new Conformance();
 616                 try {
 617                         if (conformance.init(args)) {
 618                                 conformance.run();
 619                         }
 620
 621                 } catch (Exception ex) {
 622                         ex.printStackTrace(System.err);
 623                         System.exit(1);
 624                 } finally {
 625                         try {
 626                                 conformance.close();
 627                         } catch (IOException ex) {
 628                                 ex.printStackTrace(System.err);
 629                         }
 630                 }
 631                 System.exit(0);
 632         }
 633
 634 }