feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/RepositoryAudit.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * feature-state-management
   4  * ================================================================================
   5  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.policy.drools.statemanagement;
  22
  23 import java.io.File;
  24 import java.io.FileInputStream;
  25 import java.io.FileOutputStream;
  26 import java.io.IOException;
  27 import java.nio.file.FileVisitResult;
  28 import java.nio.file.Files;
  29 import java.nio.file.Path;
  30 import java.nio.file.SimpleFileVisitor;
  31 import java.nio.file.attribute.BasicFileAttributes;
  32 import java.util.LinkedList;
  33 import java.util.Properties;
  34 import java.util.concurrent.TimeUnit;
  35
  36 import org.slf4j.Logger;
  37 import org.slf4j.LoggerFactory;
  38
  39 /**
  40  * This class audits the Maven repository.
  41  */
  42 public class RepositoryAudit extends DroolsPdpIntegrityMonitor.AuditBase {
  43     private static final long DEFAULT_TIMEOUT = 60; //timeout in 60 seconds
  44
  45     // get an instance of logger
  46     private static Logger  logger = LoggerFactory.getLogger(RepositoryAudit.class);
  47     // single global instance of this audit object
  48     private static RepositoryAudit instance = new RepositoryAudit();
  49
  50     /**
  51      * Constructor - set the name to 'Repository'.
  52      */
  53     private RepositoryAudit() {
  54         super("Repository");
  55     }
  56
  57     /**
  58      * Get the integrity monitor instance.
  59      *
  60      * @return the single 'RepositoryAudit' instance
  61      */
  62     public static DroolsPdpIntegrityMonitor.AuditBase getInstance() {
  63         return instance;
  64     }
  65
  66     /**
  67      * Invoke the audit.
  68      *
  69      * @param properties properties to be passed to the audit
  70      */
  71     @Override
  72     public void invoke(Properties properties)
  73             throws IOException, InterruptedException {
  74         logger.debug("Running 'RepositoryAudit.invoke'");
  75
  76         boolean isActive = true;
  77         // ignore errors by default
  78         boolean ignoreErrors = true;
  79         String repoAuditIsActive = StateManagementProperties.getProperty("repository.audit.is.active");
  80         String repoAuditIgnoreErrors =
  81                 StateManagementProperties.getProperty("repository.audit.ignore.errors");
  82         logger.debug("RepositoryAudit.invoke: repoAuditIsActive = {}"
  83                 + ", repoAuditIgnoreErrors = {}",repoAuditIsActive, repoAuditIgnoreErrors);
  84
  85         if (repoAuditIsActive != null) {
  86             try {
  87                 isActive = Boolean.parseBoolean(repoAuditIsActive.trim());
  88             } catch (NumberFormatException e) {
  89                 logger.warn("RepositoryAudit.invoke: Ignoring invalid property: repository.audit.is.active = {}",
  90                         repoAuditIsActive);
  91             }
  92         }
  93
  94         if (!isActive) {
  95             logger.info("RepositoryAudit.invoke: exiting because isActive = {}", isActive);
  96             return;
  97         }
  98
  99         if (repoAuditIgnoreErrors != null) {
 100             try {
 101                 ignoreErrors = Boolean.parseBoolean(repoAuditIgnoreErrors.trim());
 102             } catch (NumberFormatException e) {
 103                 ignoreErrors = true;
 104                 logger.warn("RepositoryAudit.invoke: Ignoring invalid property: repository.audit.ignore.errors = {}",
 105                         repoAuditIgnoreErrors);
 106             }
 107         } else {
 108             ignoreErrors = true;
 109         }
 110
 111         // Fetch repository information from 'IntegrityMonitorProperties'
 112         String repositoryId =
 113                 StateManagementProperties.getProperty("repository.audit.id");
 114         String repositoryUrl =
 115                 StateManagementProperties.getProperty("repository.audit.url");
 116         String repositoryUsername =
 117                 StateManagementProperties.getProperty("repository.audit.username");
 118         String repositoryPassword =
 119                 StateManagementProperties.getProperty("repository.audit.password");
 120         boolean upload =
 121                 repositoryId != null && repositoryUrl != null
 122                 && repositoryUsername != null && repositoryPassword != null;
 123
 124         // used to incrementally construct response as problems occur
 125         // (empty = no problems)
 126         StringBuilder response = new StringBuilder();
 127
 128         long timeoutInSeconds = DEFAULT_TIMEOUT;
 129         String timeoutString =
 130                 StateManagementProperties.getProperty("repository.audit.timeout");
 131         if (timeoutString != null && !timeoutString.isEmpty()) {
 132             try {
 133                 timeoutInSeconds = Long.valueOf(timeoutString);
 134             } catch (NumberFormatException e) {
 135                 logger.error("RepositoryAudit: Invalid 'repository.audit.timeout' value: '{}'",
 136                         timeoutString, e);
 137                 if (!ignoreErrors) {
 138                     response.append("Invalid 'repository.audit.timeout' value: '")
 139                     .append(timeoutString).append("'\n");
 140                     setResponse(response.toString());
 141                 }
 142             }
 143         }
 144
 145         // artifacts to be downloaded
 146         LinkedList<Artifact> artifacts = new LinkedList<>();
 147
 148         /*
 149          * 1) create temporary directory
 150          */
 151         Path dir = Files.createTempDirectory("auditRepo");
 152         logger.info("RepositoryAudit: temporary directory = {}", dir);
 153
 154         // nested 'pom.xml' file and 'repo' directory
 155         final Path pom = dir.resolve("pom.xml");
 156         final Path repo = dir.resolve("repo");
 157
 158         /*
 159          * 2) Create test file, and upload to repository
 160          *    (only if repository information is specified)
 161          */
 162         String groupId = null;
 163         String artifactId = null;
 164         String version = null;
 165         if (upload) {
 166             groupId = "org.onap.policy.audit";
 167             artifactId = "repository-audit";
 168             version = "0." + System.currentTimeMillis();
 169
 170             if (repositoryUrl.toLowerCase().contains("snapshot")) {
 171                 // use SNAPSHOT version
 172                 version += "-SNAPSHOT";
 173             }
 174
 175             // create text file to write
 176             try (FileOutputStream fos =
 177                     new FileOutputStream(dir.resolve("repository-audit.txt").toFile())) {
 178                 fos.write(version.getBytes());
 179             }
 180
 181             // try to install file in repository
 182             if (runProcess(timeoutInSeconds, dir.toFile(), null,
 183                             "mvn", "deploy:deploy-file",
 184                             "-DrepositoryId=" + repositoryId,
 185                             "-Durl=" + repositoryUrl,
 186                             "-Dfile=repository-audit.txt",
 187                             "-DgroupId=" + groupId,
 188                             "-DartifactId=" + artifactId,
 189                             "-Dversion=" + version,
 190                             "-Dpackaging=txt",
 191                             "-DgeneratePom=false") != 0) {
 192                 logger.error("RepositoryAudit: 'mvn deploy:deploy-file' failed");
 193                 if (!ignoreErrors) {
 194                     response.append("'mvn deploy:deploy-file' failed\n");
 195                     setResponse(response.toString());
 196                 }
 197             }
 198             else {
 199                 logger.info("RepositoryAudit: 'mvn deploy:deploy-file succeeded");
 200
 201                 // we also want to include this new artifact in the download
 202                 // test (steps 3 and 4)
 203                 artifacts.add(new Artifact(groupId, artifactId, version, "txt"));
 204             }
 205         }
 206
 207         /*
 208          * 3) create 'pom.xml' file in temporary directory
 209          */
 210         artifacts.add(new Artifact("org.apache.maven/maven-embedder/3.2.2"));
 211
 212         StringBuilder sb = new StringBuilder();
 213         sb.append("<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n"
 214                 + "         xsi:schemaLocation=\"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd\">\n"
 215                 + "\n"
 216                 + "  <modelVersion>4.0.0</modelVersion>\n"
 217                 + "  <groupId>empty</groupId>\n"
 218                 + "  <artifactId>empty</artifactId>\n"
 219                 + "  <version>1.0-SNAPSHOT</version>\n"
 220                 + "  <packaging>pom</packaging>\n"
 221                 + "\n"
 222                 + "  <build>\n"
 223                 + "    <plugins>\n"
 224                 + "      <plugin>\n"
 225                 + "         <groupId>org.apache.maven.plugins</groupId>\n"
 226                 + "         <artifactId>maven-dependency-plugin</artifactId>\n"
 227                 + "         <version>2.10</version>\n"
 228                 + "         <executions>\n"
 229                 + "           <execution>\n"
 230                 + "             <id>copy</id>\n"
 231                 + "             <goals>\n"
 232                 + "               <goal>copy</goal>\n"
 233                 + "             </goals>\n"
 234                 + "             <configuration>\n"
 235                 + "               <localRepositoryDirectory>")
 236             .append(repo)
 237             .append("</localRepositoryDirectory>\n")
 238             .append("               <artifactItems>\n");
 239
 240         for (Artifact artifact : artifacts) {
 241             // each artifact results in an 'artifactItem' element
 242             sb.append("                 <artifactItem>\n"
 243                     + "                   <groupId>")
 244                 .append(artifact.groupId)
 245                 .append("</groupId>\n"
 246                     + "                   <artifactId>")
 247                 .append(artifact.artifactId)
 248                 .append("</artifactId>\n"
 249                     + "                   <version>")
 250                 .append(artifact.version)
 251                 .append("</version>\n"
 252                     + "                   <type>")
 253                 .append(artifact.type)
 254                 .append("</type>\n"
 255                     + "                 </artifactItem>\n");
 256         }
 257         sb.append("               </artifactItems>\n"
 258                 + "             </configuration>\n"
 259                 + "           </execution>\n"
 260                 + "         </executions>\n"
 261                 + "      </plugin>\n"
 262                 + "    </plugins>\n"
 263                 + "  </build>\n"
 264                 + "</project>\n");
 265
 266         try (FileOutputStream fos = new FileOutputStream(pom.toFile())) {
 267             fos.write(sb.toString().getBytes());
 268         }
 269
 270         /*
 271          * 4) Invoke external 'mvn' process to do the downloads
 272          */
 273
 274         // output file = ${dir}/out (this supports step '4a')
 275         File output = dir.resolve("out").toFile();
 276
 277         // invoke process, and wait for response
 278         int rval = runProcess(timeoutInSeconds, dir.toFile(), output, "mvn", "compile");
 279         logger.info("RepositoryAudit: 'mvn' return value = {}", rval);
 280         if (rval != 0) {
 281             logger.error("RepositoryAudit: 'mvn compile' invocation failed");
 282             if (!ignoreErrors) {
 283                 response.append("'mvn compile' invocation failed\n");
 284                 setResponse(response.toString());
 285             }
 286         }
 287
 288         /*
 289          * 4a) Check attempted and successful downloads from output file
 290          *     Note: at present, this step just generates log messages,
 291          *     but doesn't do any verification.
 292          */
 293         if (rval == 0 && output != null) {
 294             // place output in 'fileContents' (replacing the Return characters
 295             // with Newline)
 296             byte[] outputData = new byte[(int)output.length()];
 297             String fileContents;
 298             try (FileInputStream fis = new FileInputStream(output)) {
 299                 //
 300                 // Ideally this should be in a loop or even better use
 301                 // Java 8 nio functionality.
 302                 //
 303                 int bytesRead = fis.read(outputData);
 304                 logger.info("fileContents read {} bytes", bytesRead);
 305                 fileContents = new String(outputData).replace('\r','\n');
 306             }
 307
 308             // generate log messages from 'Downloading' and 'Downloaded'
 309             // messages within the 'mvn' output
 310             int index = 0;
 311             while ((index = fileContents.indexOf("\nDown", index)) > 0) {
 312                 index += 5;
 313                 if (fileContents.regionMatches(index, "loading: ", 0, 9)) {
 314                     index += 9;
 315                     int endIndex = fileContents.indexOf('\n', index);
 316                     logger.info("RepositoryAudit: Attempted download: '{}'",
 317                             fileContents.substring(index, endIndex));
 318                     index = endIndex;
 319                 } else if (fileContents.regionMatches(index, "loaded: ", 0, 8)) {
 320                     index += 8;
 321                     int endIndex = fileContents.indexOf(' ', index);
 322                     logger.info("RepositoryAudit: Successful download: '{}'",fileContents.substring(index, endIndex));
 323                     index = endIndex;
 324                 }
 325             }
 326         }
 327
 328         /*
 329          * 5) Check the contents of the directory to make sure the downloads
 330          *    were successful
 331          */
 332         for (Artifact artifact : artifacts) {
 333             if (repo.resolve(artifact.groupId.replace('.','/'))
 334                     .resolve(artifact.artifactId)
 335                     .resolve(artifact.version)
 336                     .resolve(artifact.artifactId + "-" + artifact.version + "."
 337                             + artifact.type).toFile().exists()) {
 338                 // artifact exists, as expected
 339                 logger.info("RepositoryAudit: {} : exists", artifact.toString());
 340             } else {
 341                 // Audit ERROR: artifact download failed for some reason
 342                 logger.error("RepositoryAudit: {}: does not exist", artifact.toString());
 343                 if (!ignoreErrors) {
 344                     response.append("Failed to download artifact: ")
 345                     .append(artifact).append('\n');
 346                     setResponse(response.toString());
 347                 }
 348             }
 349         }
 350
 351         /*
 352          * 6) Use 'curl' to delete the uploaded test file
 353          *    (only if repository information is specified)
 354          */
 355         if (upload) {
 356             if (runProcess(timeoutInSeconds, dir.toFile(), null,
 357                             "curl",
 358                             "--request", "DELETE",
 359                             "--user", repositoryUsername + ":" + repositoryPassword,
 360                             repositoryUrl + "/" + groupId.replace('.', '/') + "/"
 361                                     + artifactId + "/" + version)
 362                     != 0) {
 363                 logger.error("RepositoryAudit: delete of uploaded artifact failed");
 364                 if (!ignoreErrors) {
 365                     response.append("delete of uploaded artifact failed\n");
 366                     setResponse(response.toString());
 367                 }
 368             } else {
 369                 logger.info("RepositoryAudit: delete of uploaded artifact succeeded");
 370                 artifacts.add(new Artifact(groupId, artifactId, version, "txt"));
 371             }
 372         }
 373
 374         /*
 375          * 7) Remove the temporary directory
 376          */
 377         Files.walkFileTree(dir, new RecursivelyDeleteDirectory());
 378     }
 379
 380     /**
 381      * Run a process, and wait for the response.
 382      *
 383      * @param timeoutInSeconds the number of seconds to wait for the process to terminate
 384      * @param directory the execution directory of the process (null = current directory)
 385      * @param stdout the file to contain the standard output (null = discard standard output)
 386      * @param command command and arguments
 387      * @return the return value of the process
 388      * @throws IOException InterruptedException
 389      */
 390     static int runProcess(long timeoutInSeconds,
 391             File directory, File stdout, String... command)
 392                     throws IOException, InterruptedException {
 393         ProcessBuilder pb = new ProcessBuilder(command);
 394         if (directory != null) {
 395             pb.directory(directory);
 396         }
 397         if (stdout != null) {
 398             pb.redirectOutput(stdout);
 399         }
 400
 401         Process process = pb.start();
 402         if (process.waitFor(timeoutInSeconds, TimeUnit.SECONDS)) {
 403             // process terminated before the timeout
 404             return process.exitValue();
 405         }
 406
 407         // process timed out -- kill it, and return -1
 408         process.destroyForcibly();
 409         return -1;
 410     }
 411
 412     /**
 413      * This class is used to recursively delete a directory and all of its
 414      * contents.
 415      */
 416     private final class RecursivelyDeleteDirectory extends SimpleFileVisitor<Path> {
 417         @Override
 418         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
 419             file.toFile().delete();
 420             return FileVisitResult.CONTINUE;
 421         }
 422
 423         @Override
 424         public FileVisitResult postVisitDirectory(Path file, IOException ex)
 425                 throws IOException {
 426             if (ex == null) {
 427                 file.toFile().delete();
 428                 return FileVisitResult.CONTINUE;
 429             } else {
 430                 throw ex;
 431             }
 432         }
 433     }
 434
 435     /* ============================================================ */
 436
 437     /**
 438      * An instance of this class exists for each artifact that we are trying
 439      * to download.
 440      */
 441     static class Artifact {
 442         String groupId;
 443         String artifactId;
 444         String version;
 445         String type;
 446
 447         /**
 448          * Constructor - populate the 'Artifact' instance.
 449          *
 450          * @param groupId groupId of artifact
 451          * @param artifactId artifactId of artifact
 452          * @param version version of artifact
 453          * @param type type of the artifact (e.g. "jar")
 454          */
 455         Artifact(String groupId, String artifactId, String version, String type) {
 456             this.groupId = groupId;
 457             this.artifactId = artifactId;
 458             this.version = version;
 459             this.type = type;
 460         }
 461
 462         /**
 463          * Constructor - populate an 'Artifact' instance.
 464          *
 465          * @param artifact a string of the form:
 466          * {@code"<groupId>/<artifactId>/<version>[/<type>]"}
 467          * @throws IllegalArgumentException if 'artifact' has the incorrect format
 468          */
 469         Artifact(String artifact) {
 470             String[] segments = artifact.split("/");
 471             if (segments.length != 4 && segments.length != 3) {
 472                 throw new IllegalArgumentException("groupId/artifactId/version/type");
 473             }
 474             groupId = segments[0];
 475             artifactId = segments[1];
 476             version = segments[2];
 477             type = segments.length == 4 ? segments[3] : "jar";
 478         }
 479
 480         /**
 481          * Returns string representation.
 482          *
 483          * @return the artifact id in the form: {@code"<groupId>/<artifactId>/<version>/<type>"}
 484          */
 485         @Override
 486         public String toString() {
 487             return groupId + "/" + artifactId + "/" + version + "/" + type;
 488         }
 489     }
 490 }