common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducer.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 Nordix Foundation
   4  *  ================================================================================
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  *  SPDX-License-Identifier: Apache-2.0
  18  *  ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.csar.storage;
  22
  23 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
  24
  25 import java.io.BufferedOutputStream;
  26 import java.io.IOException;
  27 import java.nio.file.Files;
  28 import java.nio.file.Path;
  29 import java.util.List;
  30 import java.util.Set;
  31 import java.util.UUID;
  32 import java.util.concurrent.atomic.AtomicBoolean;
  33 import java.util.concurrent.atomic.AtomicInteger;
  34 import java.util.function.Consumer;
  35 import java.util.stream.Collectors;
  36 import java.util.zip.ZipEntry;
  37 import java.util.zip.ZipFile;
  38 import java.util.zip.ZipOutputStream;
  39 import lombok.Getter;
  40 import org.apache.commons.collections4.CollectionUtils;
  41 import org.apache.commons.io.FilenameUtils;
  42 import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException;
  43 import org.slf4j.Logger;
  44 import org.slf4j.LoggerFactory;
  45
  46 public class MinIoStorageCsarSizeReducer implements PackageSizeReducer {
  47
  48     private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageCsarSizeReducer.class);
  49     private static final Set<String> ALLOWED_SIGNATURE_EXTENSIONS = Set.of("cms");
  50     private static final Set<String> ALLOWED_CERTIFICATE_EXTENSIONS = Set.of("cert", "crt");
  51     private static final String CSAR_EXTENSION = "csar";
  52     private static final String UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR = "An unexpected problem happened while reading the CSAR '%s'";
  53     @Getter
  54     private final AtomicBoolean reduced = new AtomicBoolean(false);
  55
  56     private final CsarPackageReducerConfiguration configuration;
  57
  58     public MinIoStorageCsarSizeReducer(final CsarPackageReducerConfiguration configuration) {
  59         this.configuration = configuration;
  60     }
  61
  62     @Override
  63     public byte[] reduce(final Path csarPackagePath) {
  64         if (hasSignedPackageStructure(csarPackagePath)) {
  65             return reduce(csarPackagePath, this::signedZipProcessingConsumer);
  66         } else {
  67             return reduce(csarPackagePath, this::unsignedZipProcessingConsumer);
  68         }
  69     }
  70
  71     private byte[] reduce(final Path csarPackagePath, final ZipProcessFunction zipProcessingFunction) {
  72         final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
  73
  74         try (final var zf = new ZipFile(csarPackagePath.toString());
  75             final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) {
  76             zf.entries().asIterator().forEachRemaining(zipProcessingFunction.getProcessZipConsumer(csarPackagePath, zf, zos));
  77         } catch (final IOException ex1) {
  78             rollback(reducedCsarPath);
  79             LOGGER.error("Could not read ZIP stream '{}'", csarPackagePath.toString(), ex1);
  80             final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath);
  81             throw new CsarSizeReducerException(errorMsg, ex1);
  82         }
  83         final byte[] reducedCsarBytes;
  84         try {
  85             if (reduced.get()) {
  86                 reducedCsarBytes = Files.readAllBytes(reducedCsarPath);
  87             } else {
  88                 reducedCsarBytes = Files.readAllBytes(csarPackagePath);
  89             }
  90         } catch (final IOException e) {
  91             LOGGER.error("Could not read bytes of file '{}'", csarPackagePath, e);
  92             final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath);
  93             throw new CsarSizeReducerException(errorMsg, e);
  94         }
  95         try {
  96             Files.delete(reducedCsarPath);
  97         } catch (final IOException e) {
  98             LOGGER.error("Could not delete temporary file '{}'", reducedCsarPath, e);
  99             final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath);
 100             throw new CsarSizeReducerException(errorMsg, e);
 101         }
 102
 103         return reducedCsarBytes;
 104     }
 105
 106     private Consumer<ZipEntry> signedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) {
 107         final var thresholdEntries = configuration.getThresholdEntries();
 108         final var totalEntryArchive = new AtomicInteger(0);
 109         return zipEntry -> {
 110             final var entryName = zipEntry.getName();
 111             try {
 112                 if (totalEntryArchive.getAndIncrement() > thresholdEntries) {
 113                     LOGGER.warn("too many entries in this archive, can lead to inodes exhaustion of the system");
 114                     // too many entries in this archive, can lead to inodes exhaustion of the system
 115                     final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 116                     throw new CsarSizeReducerException(errorMsg);
 117                 }
 118                 zos.putNextEntry(new ZipEntry(entryName));
 119                 if (!zipEntry.isDirectory()) {
 120                     if (entryName.toLowerCase().endsWith(CSAR_EXTENSION)) {
 121                         final var internalCsarExtractPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
 122                         Files.copy(zf.getInputStream(zipEntry), internalCsarExtractPath, REPLACE_EXISTING);
 123                         zos.write(reduce(internalCsarExtractPath, this::unsignedZipProcessingConsumer));
 124                         Files.delete(internalCsarExtractPath);
 125                     } else {
 126                         zos.write(zf.getInputStream(zipEntry).readAllBytes());
 127                     }
 128                 }
 129                 zos.closeEntry();
 130             } catch (final IOException ei) {
 131                 LOGGER.error("Failed to extract '{}' from zip '{}'", entryName, csarPackagePath, ei);
 132                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 133                 throw new CsarSizeReducerException(errorMsg, ei);
 134             }
 135         };
 136     }
 137
 138     private Consumer<ZipEntry> unsignedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) {
 139         final var thresholdEntries = configuration.getThresholdEntries();
 140         final var totalEntryArchive = new AtomicInteger(0);
 141         return zipEntry -> {
 142             final var entryName = zipEntry.getName();
 143             if (totalEntryArchive.getAndIncrement() > thresholdEntries) {
 144                 LOGGER.warn("too many entries in this archive, can lead to inodes exhaustion of the system");
 145                 // too many entries in this archive, can lead to inodes exhaustion of the system
 146                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 147                 throw new CsarSizeReducerException(errorMsg);
 148             }
 149             try {
 150                 zos.putNextEntry(new ZipEntry(entryName));
 151                 if (!zipEntry.isDirectory()) {
 152                     if (isCandidateToRemove(zipEntry)) {
 153                         // replace with EMPTY string to avoid package description inconsistency/validation errors
 154                         zos.write("".getBytes());
 155                         reduced.set(true);
 156                     } else {
 157                         zos.write(zf.getInputStream(zipEntry).readAllBytes());
 158                     }
 159                 }
 160                 zos.closeEntry();
 161             } catch (final IOException ei) {
 162                 LOGGER.error("Failed to extract '{}' from zip '{}'", entryName, csarPackagePath, ei);
 163                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 164                 throw new CsarSizeReducerException(errorMsg, ei);
 165             }
 166         };
 167     }
 168
 169     private void rollback(final Path reducedCsarPath) {
 170         if (Files.exists(reducedCsarPath)) {
 171             try {
 172                 Files.delete(reducedCsarPath);
 173             } catch (final Exception e) {
 174                 LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, e);
 175             }
 176         }
 177     }
 178
 179     private boolean isCandidateToRemove(final ZipEntry zipEntry) {
 180         final String zipEntryName = zipEntry.getName();
 181         return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith)
 182             || zipEntry.getSize() > configuration.getSizeLimit();
 183     }
 184
 185     private boolean hasSignedPackageStructure(final Path csarPackagePath) {
 186         final List<Path> packagePathList;
 187         try (final var zf = new ZipFile(csarPackagePath.toString())) {
 188             packagePathList = zf.stream()
 189                 .filter(zipEntry -> !zipEntry.isDirectory())
 190                 .map(ZipEntry::getName).map(Path::of)
 191                 .collect(Collectors.toList());
 192         } catch (final IOException e) {
 193             LOGGER.error("Failed to read ZipFile '{}'", csarPackagePath.toString(), e);
 194             final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath);
 195             throw new CsarSizeReducerException(errorMsg, e);
 196         }
 197
 198         if (CollectionUtils.isEmpty(packagePathList)) {
 199             return false;
 200         }
 201         final int numberOfFiles = packagePathList.size();
 202         if (numberOfFiles == 2) {
 203             return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList);
 204         }
 205         if (numberOfFiles == 3) {
 206             return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList) && hasOneCertificateFile(packagePathList);
 207         }
 208         return false;
 209     }
 210
 211     private boolean hasOneInternalPackageFile(final List<Path> packagePathList) {
 212         return packagePathList.parallelStream()
 213             .map(Path::toString)
 214             .map(FilenameUtils::getExtension)
 215             .map(String::toLowerCase)
 216             .filter(extension -> extension.endsWith(CSAR_EXTENSION)).count() == 1;
 217     }
 218
 219     private boolean hasOneSignatureFile(final List<Path> packagePathList) {
 220         return packagePathList.parallelStream()
 221             .map(Path::toString)
 222             .map(FilenameUtils::getExtension)
 223             .map(String::toLowerCase)
 224             .filter(ALLOWED_SIGNATURE_EXTENSIONS::contains).count() == 1;
 225     }
 226
 227     private boolean hasOneCertificateFile(final List<Path> packagePathList) {
 228         return packagePathList.parallelStream()
 229             .map(Path::toString)
 230             .map(FilenameUtils::getExtension)
 231             .map(String::toLowerCase)
 232             .filter(ALLOWED_CERTIFICATE_EXTENSIONS::contains).count() == 1;
 233     }
 234
 235     @FunctionalInterface
 236     private interface ZipProcessFunction {
 237
 238         Consumer<ZipEntry> getProcessZipConsumer(Path csarPackagePath, ZipFile zf, ZipOutputStream zos);
 239     }
 240
 241 }