b0b8971e7183662eb7c52c284e4f44df32e63a90
[sdc.git] /
1 /*-
2  * ============LICENSE_START=======================================================
3  *  Copyright (C) 2019 Nordix Foundation.
4  * ================================================================================
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  *      http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  *
17  * SPDX-License-Identifier: Apache-2.0
18  * ============LICENSE_END=========================================================
19  */
20
21 package org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.csar.validation;
22
23
24 import static org.openecomp.sdc.be.config.NonManoArtifactType.ONAP_PM_DICTIONARY;
25 import static org.openecomp.sdc.be.config.NonManoArtifactType.ONAP_VES_EVENTS;
26 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_0;
27 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_1;
28 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_METADATA_LIMIT;
29 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_PNF_METADATA;
30 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_VNF_METADATA;
31 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_MANIFEST_FILE_EXT;
32 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_PNF;
33 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_VNF;
34 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.CREATED_BY_ENTRY;
35 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.CSAR_VERSION_ENTRY;
36 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.ENTRY_DEFINITIONS;
37 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.ETSI_ENTRY_CERTIFICATE;
38 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.ETSI_ENTRY_MANIFEST;
39 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.TOSCA_META_FILE_VERSION;
40 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.TOSCA_META_FILE_VERSION_ENTRY;
41 import static org.openecomp.sdc.tosca.csar.ToscaMetaEntry.TOSCA_META_PATH_FILE_NAME;
42
43 import java.io.IOException;
44 import java.io.InputStream;
45 import java.util.ArrayList;
46 import java.util.Collections;
47 import java.util.HashMap;
48 import java.util.HashSet;
49 import java.util.List;
50 import java.util.Map;
51 import java.util.Optional;
52 import java.util.Set;
53 import java.util.concurrent.CopyOnWriteArrayList;
54 import java.util.stream.Collectors;
55 import org.apache.commons.collections.CollectionUtils;
56 import org.apache.commons.io.FilenameUtils;
57 import org.openecomp.core.impl.ToscaDefinitionImportHandler;
58 import org.openecomp.core.utilities.file.FileContentHandler;
59 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
60 import org.openecomp.sdc.common.errors.Messages;
61 import org.openecomp.sdc.common.utils.SdcCommon;
62 import org.openecomp.sdc.datatypes.error.ErrorLevel;
63 import org.openecomp.sdc.datatypes.error.ErrorMessage;
64 import org.openecomp.sdc.logging.api.Logger;
65 import org.openecomp.sdc.logging.api.LoggerFactory;
66 import org.openecomp.sdc.tosca.csar.Manifest;
67 import org.openecomp.sdc.tosca.csar.OnboardingToscaMetadata;
68 import org.openecomp.sdc.tosca.csar.SOL004ManifestOnboarding;
69 import org.openecomp.sdc.tosca.csar.ToscaMetaEntry;
70 import org.openecomp.sdc.tosca.csar.ToscaMetadata;
71 import org.openecomp.sdc.vendorsoftwareproduct.impl.onboarding.OnboardingPackageContentHandler;
72 import org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.csar.validation.exception.MissingCertificateException;
73 import org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.exceptions.InvalidManifestMetadataException;
74 import org.openecomp.sdc.vendorsoftwareproduct.security.SecurityManager;
75 import org.openecomp.sdc.vendorsoftwareproduct.security.SecurityManagerException;
76 import org.yaml.snakeyaml.Yaml;
77
78 /**
79  * Validates the contents of the package to ensure it complies with the "CSAR with TOSCA-Metadata directory" structure
80  * as defined in ETSI GS NFV-SOL 004 v2.6.1.
81  */
82 class SOL004MetaDirectoryValidator implements Validator {
83
84     private static final Logger LOGGER = LoggerFactory.getLogger(SOL004MetaDirectoryValidator.class);
85
86     private static final String MANIFEST_SOURCE = "Source";
87     private static final String MANIFEST_NON_MANO_SOURCE = "Non-MANO Source";
88     private final List<ErrorMessage> errorsByFile = new CopyOnWriteArrayList<>();
89     private final SecurityManager securityManager = SecurityManager.getInstance();
90     private OnboardingPackageContentHandler contentHandler;
91     private Set<String> folderList;
92     private ToscaMetadata toscaMetadata;
93
94     @Override
95     public Map<String, List<ErrorMessage>> validateContent(final FileContentHandler fileContentHandler) {
96         this.contentHandler = (OnboardingPackageContentHandler) fileContentHandler;
97         this.folderList = contentHandler.getFolderList();
98         parseToscaMetadata();
99         verifyMetadataFile();
100
101         if (packageHasCertificate()) {
102             verifySignedFiles();
103         }
104         return Collections.unmodifiableMap(getAnyValidationErrors());
105     }
106
107     private boolean packageHasCertificate() {
108         final String certificatePath = getCertificatePath().orElse(null);
109         return contentHandler.containsFile(certificatePath);
110     }
111
112     private Optional<String> getCertificatePath() {
113         return toscaMetadata.getEntry(ETSI_ENTRY_CERTIFICATE);
114     }
115
116     /**
117      * Parses the {@link ToscaMetaEntry#TOSCA_META_PATH_FILE_NAME;} file
118      */
119     private void parseToscaMetadata() {
120         try {
121             toscaMetadata =
122                 OnboardingToscaMetadata
123                     .parseToscaMetadataFile(contentHandler.getFileContentAsStream(TOSCA_META_PATH_FILE_NAME.getName()));
124         } catch (final IOException e) {
125             reportError(ErrorLevel.ERROR, Messages.METADATA_PARSER_INTERNAL.getErrorMessage());
126             LOGGER.error(Messages.METADATA_PARSER_INTERNAL.getErrorMessage(), e.getMessage(), e);
127         }
128     }
129
130     private void verifyMetadataFile() {
131         if (toscaMetadata.isValid() && hasETSIMetadata()) {
132             verifyManifestNameAndExtension();
133             handleMetadataEntries();
134         } else {
135             errorsByFile.addAll(toscaMetadata.getErrors());
136         }
137     }
138
139     private void verifySignedFiles() {
140         final Map<String, String> signedFileMap = contentHandler.getFileAndSignaturePathMap(SecurityManager.ALLOWED_SIGNATURE_EXTENSIONS);
141         final String packageCertificatePath = getCertificatePath().orElse(null);
142         final byte[] packageCert = contentHandler.getFileContent(packageCertificatePath);
143         if(packageCert == null) {
144             throw new MissingCertificateException("Expected package certificate");
145         }
146         signedFileMap.entrySet().stream().filter(entry -> entry.getValue() != null).forEach(entry -> {
147             final String filePath = entry.getKey();
148             final String fileSignaturePath = entry.getValue();
149             final byte[] fileBytes = contentHandler.getFileContent(filePath);
150             final byte[] fileSignatureBytes = contentHandler.getFileContent(fileSignaturePath);
151             try {
152                 if (!securityManager.verifySignedData(fileSignatureBytes, packageCert, fileBytes)) {
153                     reportError(ErrorLevel.ERROR,
154                         Messages.ARTIFACT_INVALID_SIGNATURE.formatMessage(fileSignaturePath, filePath));
155                 }
156             } catch (final SecurityManagerException e) {
157                 final String errorMessage = Messages.ARTIFACT_SIGNATURE_VALIDATION_ERROR
158                     .formatMessage(fileSignaturePath, filePath, packageCertificatePath, e.getMessage());
159                 reportError(ErrorLevel.ERROR, errorMessage);
160                 LOGGER.error(errorMessage, e);
161             }
162         });
163     }
164
165     private void verifyManifestNameAndExtension() {
166         final Map<String, String> entries = toscaMetadata.getMetaEntries();
167         final String manifestFileName = getFileName(entries.get(ETSI_ENTRY_MANIFEST.getName()));
168         final String manifestExtension = getFileExtension(entries.get(ETSI_ENTRY_MANIFEST.getName()));
169         final String mainDefinitionFileName = getFileName(entries.get(ENTRY_DEFINITIONS.getName()));
170         if (!(TOSCA_MANIFEST_FILE_EXT).equals(manifestExtension)) {
171             reportError(ErrorLevel.ERROR, Messages.MANIFEST_INVALID_EXT.getErrorMessage());
172         }
173         if (!mainDefinitionFileName.equals(manifestFileName)) {
174             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_NAME.getErrorMessage(),
175                 manifestFileName, mainDefinitionFileName));
176         }
177     }
178
179     private String getFileExtension(final String filePath) {
180         return FilenameUtils.getExtension(filePath);
181     }
182
183     private String getFileName(final String filePath) {
184         return FilenameUtils.getBaseName(filePath);
185     }
186
187     private boolean hasETSIMetadata() {
188         final Map<String, String> entries = toscaMetadata.getMetaEntries();
189         return hasEntry(entries, TOSCA_META_FILE_VERSION_ENTRY.getName())
190             && hasEntry(entries, CSAR_VERSION_ENTRY.getName())
191             && hasEntry(entries, CREATED_BY_ENTRY.getName());
192     }
193
194     private boolean hasEntry(final Map<String, String> entries, final String mandatoryEntry) {
195         if (!entries.containsKey(mandatoryEntry)) {
196             reportError(ErrorLevel.ERROR,
197                 String.format(Messages.METADATA_MISSING_ENTRY.getErrorMessage(), mandatoryEntry));
198             return false;
199         }
200         return true;
201     }
202
203     private void handleMetadataEntries() {
204         toscaMetadata.getMetaEntries().entrySet().parallelStream().forEach(this::handleEntry);
205     }
206
207     private void handleEntry(final Map.Entry<String, String> entry) {
208         final String key = entry.getKey();
209         final ToscaMetaEntry toscaMetaEntry = ToscaMetaEntry.parse(entry.getKey()).orElse(null);
210         if (toscaMetaEntry == null) {
211             reportError(ErrorLevel.ERROR, Messages.METADATA_UNSUPPORTED_ENTRY.formatMessage(key));
212             LOGGER.warn(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), key);
213             return;
214         }
215         final String value = entry.getValue();
216
217         switch (toscaMetaEntry) {
218             case TOSCA_META_FILE_VERSION_ENTRY:
219             case CSAR_VERSION_ENTRY:
220             case CREATED_BY_ENTRY:
221                 verifyMetadataEntryVersions(key, value);
222                 break;
223             case ENTRY_DEFINITIONS:
224                 validateDefinitionFile(value);
225                 break;
226             case ETSI_ENTRY_MANIFEST:
227                 validateManifestFile(value);
228                 break;
229             case ETSI_ENTRY_CHANGE_LOG:
230                 validateChangeLog(value);
231                 break;
232             case ETSI_ENTRY_TESTS:
233             case ETSI_ENTRY_LICENSES:
234                 validateOtherEntries(entry);
235                 break;
236             case ETSI_ENTRY_CERTIFICATE:
237                 validateCertificate(value);
238                 break;
239             default:
240                 reportError(ErrorLevel.ERROR, Messages.METADATA_UNSUPPORTED_ENTRY.formatMessage(key));
241                 LOGGER.warn(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), key);
242                 break;
243         }
244     }
245
246     private void validateOtherEntries(final Map.Entry entry) {
247         final String manifestFile = toscaMetadata.getMetaEntries().get(ETSI_ENTRY_MANIFEST.getName());
248         if (verifyFileExists(contentHandler.getFileList(), manifestFile)) {
249             final Manifest onboardingManifest = new SOL004ManifestOnboarding();
250             onboardingManifest.parse(contentHandler.getFileContentAsStream(manifestFile));
251             final Optional<ResourceTypeEnum> resourceType = onboardingManifest.getType();
252             if (resourceType.isPresent() && resourceType.get() == ResourceTypeEnum.VF) {
253                 final String value = (String) entry.getValue();
254                 validateOtherEntries(value);
255             } else {
256                 final String key = (String) entry.getKey();
257                 reportError(ErrorLevel.ERROR,
258                     String.format(Messages.MANIFEST_INVALID_PNF_METADATA.getErrorMessage(), key));
259             }
260
261         }
262     }
263
264     private void verifyMetadataEntryVersions(final String key, final String version) {
265         if (!(isValidTOSCAVersion(key, version) || isValidCSARVersion(key, version)
266             || CREATED_BY_ENTRY.getName().equals(key))) {
267             errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR,
268                 String.format(Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version)));
269             LOGGER.error("{}: key {} - value {} ", Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version);
270         }
271     }
272
273     private boolean isValidTOSCAVersion(final String key, final String version) {
274         return TOSCA_META_FILE_VERSION_ENTRY.getName().equals(key) && TOSCA_META_FILE_VERSION.getName().equals(version);
275     }
276
277     private boolean isValidCSARVersion(final String value, final String version) {
278         return CSAR_VERSION_ENTRY.getName().equals(value) && (CSAR_VERSION_1_1.equals(version)
279             || CSAR_VERSION_1_0.equals(version));
280     }
281
282     private void validateDefinitionFile(final String filePath) {
283         final Set<String> existingFiles = contentHandler.getFileList();
284
285         if (verifyFileExists(existingFiles, filePath)) {
286             final ToscaDefinitionImportHandler toscaDefinitionImportHandler =
287                 new ToscaDefinitionImportHandler(contentHandler.getFiles(), filePath);
288             final List<ErrorMessage> validationErrorList = toscaDefinitionImportHandler.getErrors();
289             if (CollectionUtils.isNotEmpty(validationErrorList)) {
290                 errorsByFile.addAll(validationErrorList);
291             }
292         } else {
293             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_DEFINITION_FILE.getErrorMessage(), filePath));
294         }
295     }
296
297     private boolean verifyFileExists(final Set<String> existingFiles, final String filePath) {
298         return existingFiles.contains(filePath);
299     }
300
301     private void validateManifestFile(final String filePath) {
302         final Set<String> existingFiles = contentHandler.getFileList();
303         if (verifyFileExists(existingFiles, filePath)) {
304             final Manifest onboardingManifest = new SOL004ManifestOnboarding();
305             onboardingManifest.parse(contentHandler.getFileContentAsStream(filePath));
306             if (onboardingManifest.isValid()) {
307                 try {
308                     verifyManifestMetadata(onboardingManifest.getMetadata());
309                 } catch (final InvalidManifestMetadataException e) {
310                     reportError(ErrorLevel.ERROR, e.getMessage());
311                     LOGGER.error(e.getMessage(), e);
312                 }
313                 verifyManifestSources(onboardingManifest);
314             } else {
315                 final List<String> manifestErrors = onboardingManifest.getErrors();
316                 manifestErrors.forEach(error -> reportError(ErrorLevel.ERROR, error));
317             }
318         } else {
319             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_NOT_FOUND.getErrorMessage(), filePath));
320         }
321     }
322
323     private void verifyManifestMetadata(final Map<String, String> metadata) {
324         if (metadata.size() != MANIFEST_METADATA_LIMIT) {
325             reportError(ErrorLevel.ERROR,
326                 String.format(Messages.MANIFEST_METADATA_DOES_NOT_MATCH_LIMIT.getErrorMessage(),
327                     MANIFEST_METADATA_LIMIT));
328         }
329         if (isPnfMetadata(metadata)) {
330             handleMetadataEntries(metadata, MANIFEST_PNF_METADATA);
331         } else {
332             handleMetadataEntries(metadata, MANIFEST_VNF_METADATA);
333         }
334     }
335
336     private boolean isPnfMetadata(final Map<String, String> metadata) {
337         final String firstMetadataDefinition = metadata.keySet().iterator().next();
338         final String expectedMetadataType =
339             firstMetadataDefinition.contains(TOSCA_TYPE_PNF) ? TOSCA_TYPE_PNF : TOSCA_TYPE_VNF;
340         if (metadata.keySet().stream()
341             .anyMatch((final String metadataEntry) -> !metadataEntry.contains(expectedMetadataType))) {
342             throw new InvalidManifestMetadataException(Messages.MANIFEST_METADATA_INVALID_ENTRY.getErrorMessage());
343         }
344
345         return TOSCA_TYPE_PNF.equals(expectedMetadataType);
346     }
347
348     private void handleMetadataEntries(final Map<String, String> metadata, final Set<String> manifestMetadata) {
349         manifestMetadata.stream()
350             .filter(requiredEntry -> !metadata.containsKey(requiredEntry))
351             .forEach(requiredEntry ->
352                 reportError(ErrorLevel.ERROR,
353                     String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredEntry)));
354     }
355
356     /**
357      * Checks if all manifest sources exists within the package and if all package files are being referred.
358      *
359      * @param onboardingManifest The manifest
360      */
361     private void verifyManifestSources(final Manifest onboardingManifest) {
362         final Set<String> packageFiles = contentHandler.getFileList();
363         final List<String> sources = filterSources(onboardingManifest.getSources());
364         verifyFilesExist(packageFiles, sources, MANIFEST_SOURCE);
365
366         final Map<String, List<String>> nonManoArtifacts = onboardingManifest.getNonManoSources();
367
368         final List<String> nonManoValidFilePaths = new ArrayList<>();
369         nonManoArtifacts.forEach((nonManoType, files) -> {
370             final List<String> internalNonManoFileList = filterSources(files);
371             nonManoValidFilePaths.addAll(internalNonManoFileList);
372             if (ONAP_PM_DICTIONARY.getType().equals(nonManoType) || ONAP_VES_EVENTS.getType().equals(nonManoType)) {
373                 internalNonManoFileList.forEach(this::validateYaml);
374             }
375         });
376
377         verifyFilesExist(packageFiles, nonManoValidFilePaths, MANIFEST_NON_MANO_SOURCE);
378
379         final Set<String> allReferredFiles = new HashSet<>();
380         allReferredFiles.addAll(sources);
381         allReferredFiles.addAll(nonManoValidFilePaths);
382         verifyFilesBeingReferred(allReferredFiles, packageFiles);
383     }
384
385     /**
386      * Validates if a YAML file has the correct extension, is not empty and the content is a valid YAML. Reports each
387      * error found.
388      *
389      * @param filePath the file path inside the package
390      */
391     private void validateYaml(final String filePath) {
392         if (!contentHandler.containsFile(filePath)) {
393             return;
394         }
395         final String fileExtension = getFileExtension(filePath);
396         if (!"yaml".equalsIgnoreCase(fileExtension) && !"yml".equalsIgnoreCase(fileExtension)) {
397             reportError(ErrorLevel.ERROR, Messages.INVALID_YAML_EXTENSION.formatMessage(filePath));
398             return;
399         }
400
401         final InputStream fileContent = contentHandler.getFileContentAsStream(filePath);
402         if (fileContent == null) {
403             reportError(ErrorLevel.ERROR, Messages.EMPTY_YAML_FILE_1.formatMessage(filePath));
404             return;
405         }
406         try {
407             new Yaml().loadAll(fileContent).iterator().next();
408         } catch (final Exception e) {
409             reportError(ErrorLevel.ERROR, Messages.INVALID_YAML_FORMAT_1.formatMessage(filePath, e.getMessage()));
410         }
411     }
412
413     /**
414      * Checks if all package files are referred in manifest. Reports missing references.
415      *
416      * @param referredFileSet the list of referred files path
417      * @param packageFileSet  the list of package file path
418      */
419     private void verifyFilesBeingReferred(final Set<String> referredFileSet, final Set<String> packageFileSet) {
420         packageFileSet.forEach(filePath -> {
421             if (!referredFileSet.contains(filePath)) {
422                 reportError(ErrorLevel.ERROR,
423                     String.format(Messages.MISSING_MANIFEST_REFERENCE.getErrorMessage(), filePath));
424             }
425         });
426     }
427
428     private List<String> filterSources(final List<String> source) {
429         return source.stream()
430             .filter(this::externalFileReferences)
431             .collect(Collectors.toList());
432     }
433
434     private boolean externalFileReferences(final String filePath) {
435         return !filePath.contains("://");
436     }
437
438     private void validateOtherEntries(final String folderPath) {
439         if (!verifyFoldersExist(folderList, folderPath)) {
440             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_OPTIONAL_FOLDERS.getErrorMessage(),
441                 folderPath));
442         }
443     }
444
445     private void validateCertificate(final String file) {
446         final Set<String> packageFiles = contentHandler.getFileList();
447         if (!verifyFileExist(packageFiles, file)) {
448             reportError(ErrorLevel.ERROR,
449                 String.format(Messages.MISSING_METADATA_FILES.getErrorMessage(), file, file));
450         }
451     }
452
453     private boolean verifyFoldersExist(final Set<String> folderList, final String folderPath) {
454         return folderList.contains(folderPath + "/");
455     }
456
457     private void verifyFilesExist(final Set<String> existingFiles, final List<String> sources, final String type) {
458         sources.forEach(file -> {
459             if (!existingFiles.contains(file)) {
460                 reportError(ErrorLevel.ERROR,
461                     String.format(Messages.MISSING_MANIFEST_SOURCE.getErrorMessage(), type, file));
462             }
463         });
464     }
465
466     private boolean verifyFileExist(final Set<String> existingFiles, final String file) {
467         return existingFiles.contains(file);
468     }
469
470     private void validateChangeLog(final String filePath) {
471         if (!verifyFileExists(contentHandler.getFileList(), filePath)) {
472             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_METADATA_FILES.getErrorMessage(), filePath));
473         }
474     }
475
476     private void reportError(final ErrorLevel errorLevel, final String errorMessage) {
477         errorsByFile.add(new ErrorMessage(errorLevel, errorMessage));
478     }
479
480     private Map<String, List<ErrorMessage>> getAnyValidationErrors() {
481         if (errorsByFile.isEmpty()) {
482             return Collections.emptyMap();
483         }
484         final Map<String, List<ErrorMessage>> errors = new HashMap<>();
485         errors.put(SdcCommon.UPLOAD_FILE, errorsByFile);
486         return errors;
487     }
488 }