common-be/src/main/java/org/openecomp/sdc/be/csar/storage/MinIoStorageCsarSizeReducer.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 Nordix Foundation
   4  *  ================================================================================
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *  Unless required by applicable law or agreed to in writing, software
  12  *  distributed under the License is distributed on an "AS IS" BASIS,
  13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *  See the License for the specific language governing permissions and
  15  *  limitations under the License.
  16  *
  17  *  SPDX-License-Identifier: Apache-2.0
  18  *  ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.be.csar.storage;
  22
  23 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
  24
  25 import java.io.BufferedOutputStream;
  26 import java.io.IOException;
  27 import java.nio.file.Files;
  28 import java.nio.file.Path;
  29 import java.util.List;
  30 import java.util.Set;
  31 import java.util.UUID;
  32 import java.util.concurrent.atomic.AtomicBoolean;
  33 import java.util.concurrent.atomic.AtomicInteger;
  34 import java.util.function.Consumer;
  35 import java.util.stream.Collectors;
  36 import java.util.zip.ZipEntry;
  37 import java.util.zip.ZipFile;
  38 import java.util.zip.ZipOutputStream;
  39 import lombok.Getter;
  40 import org.apache.commons.collections4.CollectionUtils;
  41 import org.apache.commons.io.FilenameUtils;
  42 import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException;
  43 import org.slf4j.Logger;
  44 import org.slf4j.LoggerFactory;
  45
  46 public class MinIoStorageCsarSizeReducer implements PackageSizeReducer {
  47
  48     private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageCsarSizeReducer.class);
  49     private static final Set<String> ALLOWED_SIGNATURE_EXTENSIONS = Set.of("cms");
  50     private static final Set<String> ALLOWED_CERTIFICATE_EXTENSIONS = Set.of("cert", "crt");
  51     private static final String CSAR_EXTENSION = "csar";
  52     private static final String UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR = "An unexpected problem happened while reading the CSAR '%s'";
  53     @Getter
  54     private final AtomicBoolean reduced = new AtomicBoolean(false);
  55
  56     private final CsarPackageReducerConfiguration configuration;
  57
  58     public MinIoStorageCsarSizeReducer(final CsarPackageReducerConfiguration configuration) {
  59         this.configuration = configuration;
  60     }
  61
  62     @Override
  63     public byte[] reduce(final Path csarPackagePath) {
  64         if (hasSignedPackageStructure(csarPackagePath)) {
  65             return reduce(csarPackagePath, this::signedZipProcessingConsumer);
  66         } else {
  67             return reduce(csarPackagePath, this::unsignedZipProcessingConsumer);
  68         }
  69     }
  70
  71     private byte[] reduce(final Path csarPackagePath, final ZipProcessFunction zipProcessingFunction) {
  72         final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
  73
  74         try (final var zf = new ZipFile(csarPackagePath.toString());
  75             final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) {
  76             zf.entries().asIterator().forEachRemaining(zipProcessingFunction.getProcessZipConsumer(csarPackagePath, zf, zos));
  77         } catch (final IOException ex1) {
  78             rollback(reducedCsarPath);
  79             final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath);
  80             throw new CsarSizeReducerException(errorMsg, ex1);
  81         }
  82         final byte[] reducedCsarBytes;
  83         try {
  84             if (reduced.get()) {
  85                 reducedCsarBytes = Files.readAllBytes(reducedCsarPath);
  86             } else {
  87                 reducedCsarBytes = Files.readAllBytes(csarPackagePath);
  88             }
  89         } catch (final IOException e) {
  90             final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath);
  91             throw new CsarSizeReducerException(errorMsg, e);
  92         }
  93         try {
  94             Files.delete(reducedCsarPath);
  95         } catch (final IOException e) {
  96             final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath);
  97             throw new CsarSizeReducerException(errorMsg, e);
  98         }
  99
 100         return reducedCsarBytes;
 101     }
 102
 103     private Consumer<ZipEntry> signedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) {
 104         final var thresholdEntries = configuration.getThresholdEntries();
 105         final var totalEntryArchive = new AtomicInteger(0);
 106         return zipEntry -> {
 107             final var entryName = zipEntry.getName();
 108             try {
 109                 if (totalEntryArchive.getAndIncrement() > thresholdEntries) {
 110                     // too much entries in this archive, can lead to inodes exhaustion of the system
 111                     final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 112                     throw new CsarSizeReducerException(errorMsg);
 113                 }
 114                 zos.putNextEntry(new ZipEntry(entryName));
 115                 if (!zipEntry.isDirectory()) {
 116                     if (entryName.toLowerCase().endsWith(CSAR_EXTENSION)) {
 117                         final var internalCsarExtractPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
 118                         Files.copy(zf.getInputStream(zipEntry), internalCsarExtractPath, REPLACE_EXISTING);
 119                         zos.write(reduce(internalCsarExtractPath, this::unsignedZipProcessingConsumer));
 120                         Files.delete(internalCsarExtractPath);
 121                     } else {
 122                         zos.write(zf.getInputStream(zipEntry).readAllBytes());
 123                     }
 124                 }
 125                 zos.closeEntry();
 126             } catch (final IOException ei) {
 127                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 128                 throw new CsarSizeReducerException(errorMsg, ei);
 129             }
 130         };
 131     }
 132
 133     private Consumer<ZipEntry> unsignedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) {
 134         final var thresholdEntries = configuration.getThresholdEntries();
 135         final var totalEntryArchive = new AtomicInteger(0);
 136         return zipEntry -> {
 137             final var entryName = zipEntry.getName();
 138             if (totalEntryArchive.getAndIncrement() > thresholdEntries) {
 139                 // too much entries in this archive, can lead to inodes exhaustion of the system
 140                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 141                 throw new CsarSizeReducerException(errorMsg);
 142             }
 143             try {
 144                 zos.putNextEntry(new ZipEntry(entryName));
 145                 if (!zipEntry.isDirectory()) {
 146                     if (isCandidateToRemove(zipEntry)) {
 147                         // replace with EMPTY string to avoid package description inconsistency/validation errors
 148                         zos.write("".getBytes());
 149                         reduced.set(true);
 150                     } else {
 151                         zos.write(zf.getInputStream(zipEntry).readAllBytes());
 152                     }
 153                 }
 154                 zos.closeEntry();
 155             } catch (final IOException ei) {
 156                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
 157                 throw new CsarSizeReducerException(errorMsg, ei);
 158             }
 159         };
 160     }
 161
 162     private void rollback(final Path reducedCsarPath) {
 163         if (Files.exists(reducedCsarPath)) {
 164             try {
 165                 Files.delete(reducedCsarPath);
 166             } catch (final Exception ex2) {
 167                 LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, ex2);
 168             }
 169         }
 170     }
 171
 172     private boolean isCandidateToRemove(final ZipEntry zipEntry) {
 173         final String zipEntryName = zipEntry.getName();
 174         return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith)
 175             || zipEntry.getSize() > configuration.getSizeLimit();
 176     }
 177
 178     private boolean hasSignedPackageStructure(final Path csarPackagePath) {
 179         final List<Path> packagePathList;
 180         try (final var zf = new ZipFile(csarPackagePath.toString())) {
 181             packagePathList = zf.stream()
 182                 .filter(zipEntry -> !zipEntry.isDirectory())
 183                 .map(ZipEntry::getName).map(Path::of)
 184                 .collect(Collectors.toList());
 185         } catch (final IOException e) {
 186             final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath);
 187             throw new CsarSizeReducerException(errorMsg, e);
 188         }
 189
 190         if (CollectionUtils.isEmpty(packagePathList)) {
 191             return false;
 192         }
 193         final int numberOfFiles = packagePathList.size();
 194         if (numberOfFiles == 2) {
 195             return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList);
 196         }
 197         if (numberOfFiles == 3) {
 198             return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList) && hasOneCertificateFile(packagePathList);
 199         }
 200         return false;
 201     }
 202
 203     private boolean hasOneInternalPackageFile(final List<Path> packagePathList) {
 204         return packagePathList.parallelStream()
 205             .map(Path::toString)
 206             .map(FilenameUtils::getExtension)
 207             .map(String::toLowerCase)
 208             .filter(extension -> extension.endsWith(CSAR_EXTENSION)).count() == 1;
 209     }
 210
 211     private boolean hasOneSignatureFile(final List<Path> packagePathList) {
 212         return packagePathList.parallelStream()
 213             .map(Path::toString)
 214             .map(FilenameUtils::getExtension)
 215             .map(String::toLowerCase)
 216             .filter(ALLOWED_SIGNATURE_EXTENSIONS::contains).count() == 1;
 217     }
 218
 219     private boolean hasOneCertificateFile(final List<Path> packagePathList) {
 220         return packagePathList.parallelStream()
 221             .map(Path::toString)
 222             .map(FilenameUtils::getExtension)
 223             .map(String::toLowerCase)
 224             .filter(ALLOWED_CERTIFICATE_EXTENSIONS::contains).count() == 1;
 225     }
 226
 227     @FunctionalInterface
 228     private interface ZipProcessFunction {
 229
 230         Consumer<ZipEntry> getProcessZipConsumer(Path csarPackagePath, ZipFile zf, ZipOutputStream zos);
 231     }
 232
 233 }