gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2019 Nordix Foundation.
   4  * ================================================================================
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *      http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  * SPDX-License-Identifier: Apache-2.0
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.csar.validation;
  22
  23 import org.apache.commons.collections.CollectionUtils;
  24 import org.openecomp.core.impl.ToscaDefinitionImportHandler;
  25 import org.openecomp.core.utilities.file.FileContentHandler;
  26 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  27 import org.openecomp.sdc.common.errors.Messages;
  28 import org.openecomp.sdc.common.utils.SdcCommon;
  29 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  30 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  31 import org.openecomp.sdc.logging.api.Logger;
  32 import org.openecomp.sdc.logging.api.LoggerFactory;
  33 import org.openecomp.sdc.tosca.csar.Manifest;
  34 import org.openecomp.sdc.tosca.csar.OnboardingToscaMetadata;
  35 import org.openecomp.sdc.tosca.csar.SOL004ManifestOnboarding;
  36 import org.openecomp.sdc.tosca.csar.ToscaMetadata;
  37 import org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.exceptions.InvalidManifestMetadataException;
  38 import java.io.IOException;
  39 import java.util.ArrayList;
  40 import java.util.Collection;
  41 import java.util.Collections;
  42 import java.util.HashMap;
  43 import java.util.HashSet;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Optional;
  47 import java.util.Set;
  48 import java.util.stream.Collectors;
  49
  50 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_0;
  51 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_1;
  52 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_METADATA_LIMIT;
  53 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_PNF_METADATA;
  54 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_VNF_METADATA;
  55 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_MANIFEST_FILE_EXT;
  56 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_CERTIFICATE;
  57 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION_ENTRY;
  58 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CREATED_BY_ENTRY;
  59 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CSAR_VERSION_ENTRY;
  60 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_CHANGE_LOG;
  61 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_DEFINITIONS;
  62 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_LICENSES;
  63 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_MANIFEST;
  64 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_TESTS;
  65 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION;
  66 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_PATH_FILE_NAME;
  67 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_PNF;
  68 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_VNF;
  69
  70 /**
  71  * Validates the contents of the package to ensure it complies with the "CSAR with TOSCA-Metadata directory" structure
  72  * as defined in ETSI GS NFV-SOL 004 v2.6.1.
  73  */
  74 class SOL004MetaDirectoryValidator implements Validator {
  75
  76     private static final Logger LOGGER = LoggerFactory.getLogger(SOL004MetaDirectoryValidator.class);
  77
  78     private static final String MANIFEST_SOURCE = "Source";
  79     private static final String MANIFEST_NON_MANO_SOURCE = "Non-MANO Source";
  80     private final List<ErrorMessage> errorsByFile = new ArrayList<>();
  81
  82     @Override
  83     public Map<String, List<ErrorMessage>> validateContent(FileContentHandler contentHandler, List<String> folderList) {
  84             validateMetaFile(contentHandler, folderList);
  85             return Collections.unmodifiableMap(getAnyValidationErrors());
  86     }
  87
  88     private void validateMetaFile(FileContentHandler contentHandler, List<String> folderList) {
  89         try {
  90             ToscaMetadata toscaMetadata = OnboardingToscaMetadata.parseToscaMetadataFile(contentHandler.getFileContent(TOSCA_META_PATH_FILE_NAME));
  91             if(toscaMetadata.isValid() && hasETSIMetadata(toscaMetadata)) {
  92                 verifyManifestNameAndExtension(toscaMetadata);
  93                 handleMetadataEntries(contentHandler, folderList, toscaMetadata);
  94             }else {
  95                 errorsByFile.addAll(toscaMetadata.getErrors());
  96             }
  97         }catch (IOException e){
  98             reportError(ErrorLevel.ERROR, Messages.METADATA_PARSER_INTERNAL.getErrorMessage());
  99             LOGGER.error(Messages.METADATA_PARSER_INTERNAL.getErrorMessage(), e.getMessage(), e);
 100         }
 101     }
 102
 103     private void verifyManifestNameAndExtension(ToscaMetadata toscaMetadata) {
 104         Map<String, String> entries = toscaMetadata.getMetaEntries();
 105         String manifestFileName = getFileName(entries.get(TOSCA_META_ETSI_ENTRY_MANIFEST));
 106         String manifestExtension = getFileExtension(entries.get(TOSCA_META_ETSI_ENTRY_MANIFEST));
 107         String mainDefinitionFileName= getFileName(entries.get(TOSCA_META_ENTRY_DEFINITIONS));
 108         if(!(TOSCA_MANIFEST_FILE_EXT).equals(manifestExtension)){
 109             reportError(ErrorLevel.ERROR, Messages.MANIFEST_INVALID_EXT.getErrorMessage());
 110         }
 111         if(!mainDefinitionFileName.equals(manifestFileName)){
 112             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_NAME.getErrorMessage(),
 113                     manifestFileName, mainDefinitionFileName));
 114         }
 115     }
 116
 117     public String getFileExtension(String filePath){
 118         return filePath.substring(filePath.lastIndexOf('.') + 1);
 119     }
 120
 121     private String getFileName(String filePath){
 122         return filePath.substring(filePath.lastIndexOf('/') + 1, filePath.lastIndexOf('.'));
 123     }
 124
 125     private boolean hasETSIMetadata(ToscaMetadata toscaMetadata){
 126         Map<String, String> entries = toscaMetadata.getMetaEntries();
 127         return hasEntry(entries, TOSCA_META_FILE_VERSION_ENTRY)
 128                 && hasEntry(entries, TOSCA_META_CSAR_VERSION_ENTRY)
 129                 && hasEntry(entries, TOSCA_META_CREATED_BY_ENTRY);
 130     }
 131
 132     private boolean hasEntry(Map<String, String> entries, String mandatoryEntry) {
 133         if (!entries.containsKey(mandatoryEntry)) {
 134             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_ENTRY.getErrorMessage(),mandatoryEntry));
 135             return false;
 136         }
 137         return true;
 138     }
 139
 140     private void handleMetadataEntries(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata) {
 141         for(Map.Entry entry: toscaMetadata.getMetaEntries().entrySet()){
 142             handleEntry(contentHandler, folderList, toscaMetadata, entry);
 143         }
 144     }
 145
 146     private void handleEntry(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata, Map.Entry entry) {
 147         String key = (String) entry.getKey();
 148         String value = (String) entry.getValue();
 149         switch (key){
 150             case TOSCA_META_FILE_VERSION_ENTRY:
 151             case TOSCA_META_CSAR_VERSION_ENTRY:
 152             case TOSCA_META_CREATED_BY_ENTRY:
 153                 verifyMetadataEntryVersions(key, value);
 154                 break;
 155             case TOSCA_META_ENTRY_DEFINITIONS:
 156                 validateDefinitionFile(contentHandler, value);
 157                 break;
 158             case TOSCA_META_ETSI_ENTRY_MANIFEST:
 159                 validateManifestFile(contentHandler, value);
 160                 break;
 161             case TOSCA_META_ETSI_ENTRY_CHANGE_LOG:
 162                 validateChangeLog(contentHandler, value);
 163                 break;
 164             case TOSCA_META_ETSI_ENTRY_TESTS:
 165             case TOSCA_META_ETSI_ENTRY_LICENSES:
 166                 validateOtherEntries(folderList, entry, contentHandler, toscaMetadata);
 167                 break;
 168             case TOSCA_META_ETSI_ENTRY_CERTIFICATE:
 169                 validateOtherEntries(folderList, value);
 170                 break;
 171             default:
 172                 errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry)));
 173                 LOGGER.warn(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry);
 174                 break;
 175         }
 176     }
 177
 178     private void validateOtherEntries(List<String> folderList, Map.Entry entry, FileContentHandler contentHandler, ToscaMetadata toscaMetadata) {
 179         String manifestFile = toscaMetadata.getMetaEntries().get(TOSCA_META_ETSI_ENTRY_MANIFEST);
 180         if(verifyFileExists(contentHandler.getFileList(), manifestFile)){
 181             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 182             onboardingManifest.parse(contentHandler.getFileContent(manifestFile));
 183             Optional<ResourceTypeEnum> resourceType = onboardingManifest.getType();
 184             if(resourceType.isPresent() && resourceType.get() == ResourceTypeEnum.VF){
 185                 String value = (String) entry.getValue();
 186                 validateOtherEntries(folderList, value);
 187             }else{
 188                 String key = (String) entry.getKey();
 189                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_PNF_METADATA.getErrorMessage(), key));
 190             }
 191
 192         }
 193     }
 194
 195
 196     private void verifyMetadataEntryVersions(String key, String version) {
 197         if(!(isValidTOSCAVersion(key,version) || isValidCSARVersion(key, version) || TOSCA_META_CREATED_BY_ENTRY.equals(key))) {
 198             errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version)));
 199             LOGGER.error("{}: key {} - value {} ", Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version);
 200         }
 201     }
 202
 203     private boolean isValidTOSCAVersion(String key, String version){
 204         return TOSCA_META_FILE_VERSION_ENTRY.equals(key) && TOSCA_META_FILE_VERSION.equals(version);
 205     }
 206
 207     private boolean isValidCSARVersion(String value, String version){
 208         return TOSCA_META_CSAR_VERSION_ENTRY.equals(value) && (CSAR_VERSION_1_1.equals(version)
 209                 || CSAR_VERSION_1_0.equals(version));
 210     }
 211
 212     private void validateDefinitionFile(final FileContentHandler contentHandler, final String filePath) {
 213         final Set<String> existingFiles = contentHandler.getFileList();
 214
 215         if (verifyFileExists(existingFiles, filePath)) {
 216             final ToscaDefinitionImportHandler toscaDefinitionImportHandler =
 217                 new ToscaDefinitionImportHandler(contentHandler.getFiles(), filePath);
 218             final List<ErrorMessage> validationErrorList = toscaDefinitionImportHandler.getErrors();
 219             if (CollectionUtils.isNotEmpty(validationErrorList)) {
 220                 errorsByFile.addAll(validationErrorList);
 221             }
 222         } else {
 223             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_DEFINITION_FILE.getErrorMessage(), filePath));
 224         }
 225     }
 226
 227     private boolean verifyFileExists(Set<String> existingFiles, String filePath){
 228         return existingFiles.contains(filePath);
 229     }
 230
 231     private void validateManifestFile(FileContentHandler contentHandler, String filePath){
 232         final Set<String> existingFiles = contentHandler.getFileList();
 233         if (verifyFileExists(existingFiles, filePath)) {
 234             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 235             onboardingManifest.parse(contentHandler.getFileContent(filePath));
 236             if(onboardingManifest.isValid()){
 237                 try {
 238                     verifyManifestMetadata(onboardingManifest.getMetadata());
 239                 }catch (InvalidManifestMetadataException e){
 240                    reportError(ErrorLevel.ERROR, e.getMessage());
 241                    LOGGER.error(e.getMessage(), e);
 242                 }
 243                 verifyManifestSources(existingFiles, onboardingManifest);
 244             }else{
 245                 List<String> manifestErrors = onboardingManifest.getErrors();
 246                 for(String error: manifestErrors){
 247                     reportError(ErrorLevel.ERROR, error);
 248                 }
 249             }
 250         }else {
 251             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_NOT_FOUND.getErrorMessage(), filePath));
 252         }
 253     }
 254
 255     private void verifyManifestMetadata(Map<String, String> metadata) {
 256         if(metadata.size() != MANIFEST_METADATA_LIMIT){
 257             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_DOES_NOT_MATCH_LIMIT.getErrorMessage(),
 258                     MANIFEST_METADATA_LIMIT));
 259         }
 260         if(isPnfMetadata(metadata)){
 261             handlePnfMetadataEntries(metadata);
 262         }else {
 263             handleVnfMetadataEntries(metadata);
 264         }
 265     }
 266
 267     private boolean isPnfMetadata(Map<String, String> metadata) {
 268         String metadataType = "";
 269         for(String key: metadata.keySet()) {
 270             if(metadataType.isEmpty()){
 271                  metadataType = key.contains(TOSCA_TYPE_PNF) ? TOSCA_TYPE_PNF : TOSCA_TYPE_VNF;
 272             }else if(!key.contains(metadataType)){
 273                 throw new InvalidManifestMetadataException(Messages.MANIFEST_METADATA_INVALID_ENTRY.getErrorMessage());
 274             }
 275         }
 276         return TOSCA_TYPE_PNF.equals(metadataType);
 277     }
 278
 279     private void handleVnfMetadataEntries(Map<String, String> metadata) {
 280         for (String requiredVnfEntry : MANIFEST_VNF_METADATA) {
 281             if (!metadata.containsKey(requiredVnfEntry)) {
 282                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredVnfEntry));
 283             }
 284         }
 285     }
 286
 287     private void handlePnfMetadataEntries(Map<String, String> metadata) {
 288         for (String requiredPnfEntry : MANIFEST_PNF_METADATA) {
 289             if (!metadata.containsKey(requiredPnfEntry)) {
 290                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredPnfEntry));
 291             }
 292         }
 293     }
 294
 295     /**
 296      * Checks if all manifest sources exists within the package and if all package files are being referred.
 297      *
 298      * @param packageFiles          The package file path list
 299      * @param onboardingManifest    The manifest
 300      */
 301     private void verifyManifestSources(final Set<String> packageFiles, final Manifest onboardingManifest) {
 302         final List<String> sources = filterSources(onboardingManifest.getSources());
 303         verifyFilesExist(packageFiles, sources, MANIFEST_SOURCE);
 304
 305         final Map<String, List<String>> nonManoArtifacts = onboardingManifest.getNonManoSources();
 306         final List<String> nonManoFiles = nonManoArtifacts.values().stream()
 307             .map(this::filterSources)
 308             .flatMap(Collection::stream)
 309             .collect(Collectors.toList());
 310         verifyFilesExist(packageFiles, nonManoFiles, MANIFEST_NON_MANO_SOURCE);
 311
 312         final Set<String> allReferredFiles = new HashSet<>();
 313         allReferredFiles.addAll(sources);
 314         allReferredFiles.addAll(nonManoFiles);
 315         verifyFilesBeingReferred(allReferredFiles, packageFiles);
 316     }
 317
 318     /**
 319      * Checks if all package files are referred in manifest.
 320      * Reports missing references.
 321      *
 322      * @param referredFileSet   the list of referred files path
 323      * @param packageFileSet    the list of package file path
 324      */
 325     private void verifyFilesBeingReferred(final Set<String> referredFileSet, final Set<String> packageFileSet) {
 326         packageFileSet.forEach(filePath -> {
 327             if (!referredFileSet.contains(filePath)) {
 328                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_MANIFEST_REFERENCE.getErrorMessage(), filePath));
 329             }
 330         });
 331     }
 332
 333     private List<String> filterSources(List<String> source){
 334         return source.stream()
 335                 .filter(this::externalFileReferences)
 336                 .collect(Collectors.toList());
 337     }
 338
 339     private boolean externalFileReferences(String filePath){
 340         return !filePath.contains("://");
 341     }
 342
 343     private void validateOtherEntries(List<String> folderList, String folderPath){
 344         if(!verifyFoldersExist(folderList, folderPath))
 345             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_OPTIONAL_FOLDERS.getErrorMessage(),
 346                     folderPath));
 347     }
 348
 349     private boolean verifyFoldersExist(List<String> folderList, String folderPath){
 350         return folderList.contains(folderPath + "/");
 351     }
 352
 353     private void verifyFilesExist(Set<String> existingFiles, List<String> sources, String type){
 354         for(String file: sources){
 355             if(!verifyFileExists(existingFiles, file)){
 356                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_MANIFEST_SOURCE.getErrorMessage(), type, file));
 357             }
 358
 359         }
 360     }
 361
 362     private void validateChangeLog(FileContentHandler contentHandler, String filePath){
 363         if(!verifyFileExists(contentHandler.getFileList(), filePath)){
 364             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_METADATA_FILES.getErrorMessage(), filePath));
 365         }
 366     }
 367
 368     private void reportError(final ErrorLevel errorLevel, final String errorMessage) {
 369         errorsByFile.add(new ErrorMessage(errorLevel, errorMessage));
 370     }
 371
 372     private Map<String, List<ErrorMessage>> getAnyValidationErrors(){
 373
 374         if(errorsByFile.isEmpty()){
 375             return Collections.emptyMap();
 376         }
 377         Map<String, List<ErrorMessage>> errors = new HashMap<>();
 378         errors.put(SdcCommon.UPLOAD_FILE, errorsByFile);
 379         return errors;
 380     }
 381 }