3181b088c0f5b00fe35fb841e1ddd91ceca11ced
[sdc.git] / common-be / src / main / java / org / openecomp / sdc / be / csar / storage / MinIoStorageCsarSizeReducer.java
1 /*
2  * ============LICENSE_START=======================================================
3  *  Copyright (C) 2021 Nordix Foundation
4  *  ================================================================================
5  *  Licensed under the Apache License, Version 2.0 (the "License");
6  *  you may not use this file except in compliance with the License.
7  *  You may obtain a copy of the License at
8  *
9  *        http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *  Unless required by applicable law or agreed to in writing, software
12  *  distributed under the License is distributed on an "AS IS" BASIS,
13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *  See the License for the specific language governing permissions and
15  *  limitations under the License.
16  *
17  *  SPDX-License-Identifier: Apache-2.0
18  *  ============LICENSE_END=========================================================
19  */
20
21 package org.openecomp.sdc.be.csar.storage;
22
23 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
24
25 import java.io.BufferedOutputStream;
26 import java.io.IOException;
27 import java.nio.file.Files;
28 import java.nio.file.Path;
29 import java.util.List;
30 import java.util.Set;
31 import java.util.UUID;
32 import java.util.concurrent.atomic.AtomicBoolean;
33 import java.util.concurrent.atomic.AtomicInteger;
34 import java.util.function.Consumer;
35 import java.util.stream.Collectors;
36 import java.util.zip.ZipEntry;
37 import java.util.zip.ZipFile;
38 import java.util.zip.ZipOutputStream;
39 import lombok.Getter;
40 import org.apache.commons.collections4.CollectionUtils;
41 import org.apache.commons.io.FilenameUtils;
42 import org.openecomp.sdc.be.csar.storage.exception.CsarSizeReducerException;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45
46 public class MinIoStorageCsarSizeReducer implements PackageSizeReducer {
47
48     private static final Logger LOGGER = LoggerFactory.getLogger(MinIoStorageCsarSizeReducer.class);
49     private static final Set<String> ALLOWED_SIGNATURE_EXTENSIONS = Set.of("cms");
50     private static final Set<String> ALLOWED_CERTIFICATE_EXTENSIONS = Set.of("cert", "crt");
51     private static final String CSAR_EXTENSION = "csar";
52     private static final String UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR = "An unexpected problem happened while reading the CSAR '%s'";
53     @Getter
54     private final AtomicBoolean reduced = new AtomicBoolean(false);
55
56     private final CsarPackageReducerConfiguration configuration;
57
58     public MinIoStorageCsarSizeReducer(final CsarPackageReducerConfiguration configuration) {
59         this.configuration = configuration;
60     }
61
62     @Override
63     public byte[] reduce(final Path csarPackagePath) {
64         if (hasSignedPackageStructure(csarPackagePath)) {
65             return reduce(csarPackagePath, this::signedZipProcessingConsumer);
66         } else {
67             return reduce(csarPackagePath, this::unsignedZipProcessingConsumer);
68         }
69     }
70
71     private byte[] reduce(final Path csarPackagePath, final ZipProcessFunction zipProcessingFunction) {
72         final var reducedCsarPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
73
74         try (final var zf = new ZipFile(csarPackagePath.toString());
75             final var zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(reducedCsarPath)))) {
76             zf.entries().asIterator().forEachRemaining(zipProcessingFunction.getProcessZipConsumer(csarPackagePath, zf, zos));
77         } catch (final IOException ex1) {
78             rollback(reducedCsarPath);
79             final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath);
80             throw new CsarSizeReducerException(errorMsg, ex1);
81         }
82         final byte[] reducedCsarBytes;
83         try {
84             if (reduced.get()) {
85                 reducedCsarBytes = Files.readAllBytes(reducedCsarPath);
86             } else {
87                 reducedCsarBytes = Files.readAllBytes(csarPackagePath);
88             }
89         } catch (final IOException e) {
90             final var errorMsg = String.format("Could not read bytes of file '%s'", csarPackagePath);
91             throw new CsarSizeReducerException(errorMsg, e);
92         }
93         try {
94             Files.delete(reducedCsarPath);
95         } catch (final IOException e) {
96             final var errorMsg = String.format("Could not delete temporary file '%s'", reducedCsarPath);
97             throw new CsarSizeReducerException(errorMsg, e);
98         }
99
100         return reducedCsarBytes;
101     }
102
103     private Consumer<ZipEntry> signedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) {
104         final var thresholdEntries = configuration.getThresholdEntries();
105         final var totalEntryArchive = new AtomicInteger(0);
106         return zipEntry -> {
107             final var entryName = zipEntry.getName();
108             try {
109                 if (totalEntryArchive.getAndIncrement() > thresholdEntries) {
110                     // too much entries in this archive, can lead to inodes exhaustion of the system
111                     final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
112                     throw new CsarSizeReducerException(errorMsg);
113                 }
114                 zos.putNextEntry(new ZipEntry(entryName));
115                 if (!zipEntry.isDirectory()) {
116                     if (entryName.toLowerCase().endsWith(CSAR_EXTENSION)) {
117                         final var internalCsarExtractPath = Path.of(csarPackagePath + "." + UUID.randomUUID());
118                         Files.copy(zf.getInputStream(zipEntry), internalCsarExtractPath, REPLACE_EXISTING);
119                         zos.write(reduce(internalCsarExtractPath, this::unsignedZipProcessingConsumer));
120                         Files.delete(internalCsarExtractPath);
121                     } else {
122                         zos.write(zf.getInputStream(zipEntry).readAllBytes());
123                     }
124                 }
125                 zos.closeEntry();
126             } catch (final IOException ei) {
127                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
128                 throw new CsarSizeReducerException(errorMsg, ei);
129             }
130         };
131     }
132
133     private Consumer<ZipEntry> unsignedZipProcessingConsumer(final Path csarPackagePath, final ZipFile zf, final ZipOutputStream zos) {
134         final var thresholdEntries = configuration.getThresholdEntries();
135         final var totalEntryArchive = new AtomicInteger(0);
136         return zipEntry -> {
137             final var entryName = zipEntry.getName();
138             if (totalEntryArchive.getAndIncrement() > thresholdEntries) {
139                 // too much entries in this archive, can lead to inodes exhaustion of the system
140                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
141                 throw new CsarSizeReducerException(errorMsg);
142             }
143             try {
144                 zos.putNextEntry(new ZipEntry(entryName));
145                 if (!zipEntry.isDirectory()) {
146                     if (isCandidateToRemove(zipEntry)) {
147                         // replace with EMPTY string to avoid package description inconsistency/validation errors
148                         zos.write("".getBytes());
149                         reduced.set(true);
150                     } else {
151                         zos.write(zf.getInputStream(zipEntry).readAllBytes());
152                     }
153                 }
154                 zos.closeEntry();
155             } catch (final IOException ei) {
156                 final var errorMsg = String.format("Failed to extract '%s' from zip '%s'", entryName, csarPackagePath);
157                 throw new CsarSizeReducerException(errorMsg, ei);
158             }
159         };
160     }
161
162     private void rollback(final Path reducedCsarPath) {
163         if (Files.exists(reducedCsarPath)) {
164             try {
165                 Files.delete(reducedCsarPath);
166             } catch (final Exception ex2) {
167                 LOGGER.warn("Could not delete temporary file '{}'", reducedCsarPath, ex2);
168             }
169         }
170     }
171
172     private boolean isCandidateToRemove(final ZipEntry zipEntry) {
173         final String zipEntryName = zipEntry.getName();
174         return configuration.getFoldersToStrip().stream().anyMatch(Path.of(zipEntryName)::startsWith)
175             || zipEntry.getSize() > configuration.getSizeLimit();
176     }
177
178     private boolean hasSignedPackageStructure(final Path csarPackagePath) {
179         final List<Path> packagePathList;
180         try (final var zf = new ZipFile(csarPackagePath.toString())) {
181             packagePathList = zf.stream()
182                 .filter(zipEntry -> !zipEntry.isDirectory())
183                 .map(ZipEntry::getName).map(Path::of)
184                 .collect(Collectors.toList());
185         } catch (final IOException e) {
186             final var errorMsg = String.format(UNEXPECTED_PROBLEM_HAPPENED_WHILE_READING_THE_CSAR, csarPackagePath);
187             throw new CsarSizeReducerException(errorMsg, e);
188         }
189
190         if (CollectionUtils.isEmpty(packagePathList)) {
191             return false;
192         }
193         final int numberOfFiles = packagePathList.size();
194         if (numberOfFiles == 2) {
195             return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList);
196         }
197         if (numberOfFiles == 3) {
198             return hasOneInternalPackageFile(packagePathList) && hasOneSignatureFile(packagePathList) && hasOneCertificateFile(packagePathList);
199         }
200         return false;
201     }
202
203     private boolean hasOneInternalPackageFile(final List<Path> packagePathList) {
204         return packagePathList.parallelStream()
205             .map(Path::toString)
206             .map(FilenameUtils::getExtension)
207             .map(String::toLowerCase)
208             .filter(extension -> extension.endsWith(CSAR_EXTENSION)).count() == 1;
209     }
210
211     private boolean hasOneSignatureFile(final List<Path> packagePathList) {
212         return packagePathList.parallelStream()
213             .map(Path::toString)
214             .map(FilenameUtils::getExtension)
215             .map(String::toLowerCase)
216             .filter(ALLOWED_SIGNATURE_EXTENSIONS::contains).count() == 1;
217     }
218
219     private boolean hasOneCertificateFile(final List<Path> packagePathList) {
220         return packagePathList.parallelStream()
221             .map(Path::toString)
222             .map(FilenameUtils::getExtension)
223             .map(String::toLowerCase)
224             .filter(ALLOWED_CERTIFICATE_EXTENSIONS::contains).count() == 1;
225     }
226
227     @FunctionalInterface
228     private interface ZipProcessFunction {
229
230         Consumer<ZipEntry> getProcessZipConsumer(Path csarPackagePath, ZipFile zf, ZipOutputStream zos);
231     }
232
233 }