gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2019 Nordix Foundation.
   4  * ================================================================================
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *      http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  * SPDX-License-Identifier: Apache-2.0
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.csar.validation;
  22
  23 import java.util.Collection;
  24 import org.openecomp.core.converter.ServiceTemplateReaderService;
  25 import org.openecomp.core.impl.services.ServiceTemplateReaderServiceImpl;
  26 import org.openecomp.core.utilities.file.FileContentHandler;
  27 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  28 import org.openecomp.sdc.common.errors.Messages;
  29 import org.openecomp.sdc.common.utils.SdcCommon;
  30 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  31 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  32 import org.openecomp.sdc.logging.api.Logger;
  33 import org.openecomp.sdc.logging.api.LoggerFactory;
  34 import org.openecomp.sdc.tosca.csar.Manifest;
  35 import org.openecomp.sdc.tosca.csar.OnboardingToscaMetadata;
  36 import org.openecomp.sdc.tosca.csar.SOL004ManifestOnboarding;
  37 import org.openecomp.sdc.tosca.csar.ToscaMetadata;
  38 import org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.exceptions.InvalidManifestMetadataException;
  39 import java.io.IOException;
  40 import java.util.ArrayList;
  41 import java.util.Collections;
  42 import java.util.HashMap;
  43 import java.util.HashSet;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Optional;
  47 import java.util.Set;
  48 import java.util.stream.Collectors;
  49
  50 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_0;
  51 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_1;
  52 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_METADATA_LIMIT;
  53 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_PNF_METADATA;
  54 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_VNF_METADATA;
  55 import static org.openecomp.sdc.tosca.csar.CSARConstants.NON_FILE_IMPORT_ATTRIBUTES;
  56 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_MANIFEST_FILE_EXT;
  57 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_CERTIFICATE;
  58 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION_ENTRY;
  59 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CREATED_BY_ENTRY;
  60 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CSAR_VERSION_ENTRY;
  61 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_CHANGE_LOG;
  62 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_DEFINITIONS;
  63 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_LICENSES;
  64 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_MANIFEST;
  65 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_TESTS;
  66 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION;
  67 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_PATH_FILE_NAME;
  68 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_PNF;
  69 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_VNF;
  70
  71 /**
  72  * Validates the contents of the package to ensure it complies with the "CSAR with TOSCA-Metadata directory" structure
  73  * as defined in ETSI GS NFV-SOL 004 v2.6.1.
  74  *
  75  */
  76
  77 class SOL004MetaDirectoryValidator implements Validator{
  78
  79     private static final Logger LOGGER = LoggerFactory.getLogger(SOL004MetaDirectoryValidator.class);
  80
  81     private static final String MANIFEST_SOURCE = "Source";
  82     private static final String MANIFEST_NON_MANO_SOURCE = "Non-MANO Source";
  83     private final List<ErrorMessage> errorsByFile = new ArrayList<>();
  84     private final Set<String> verifiedImports = new HashSet<>();
  85
  86     @Override
  87     public Map<String, List<ErrorMessage>> validateContent(FileContentHandler contentHandler, List<String> folderList) {
  88             validateMetaFile(contentHandler, folderList);
  89             return Collections.unmodifiableMap(getAnyValidationErrors());
  90     }
  91
  92     private void validateMetaFile(FileContentHandler contentHandler, List<String> folderList) {
  93         try {
  94             ToscaMetadata toscaMetadata = OnboardingToscaMetadata.parseToscaMetadataFile(contentHandler.getFileContent(TOSCA_META_PATH_FILE_NAME));
  95             if(toscaMetadata.isValid() && hasETSIMetadata(toscaMetadata)) {
  96                 verifyManifestNameAndExtension(toscaMetadata);
  97                 handleMetadataEntries(contentHandler, folderList, toscaMetadata);
  98             }else {
  99                 errorsByFile.addAll(toscaMetadata.getErrors());
 100             }
 101         }catch (IOException e){
 102             reportError(ErrorLevel.ERROR, Messages.METADATA_PARSER_INTERNAL.getErrorMessage());
 103             LOGGER.error(Messages.METADATA_PARSER_INTERNAL.getErrorMessage(), e.getMessage(), e);
 104         }
 105     }
 106
 107     private void verifyManifestNameAndExtension(ToscaMetadata toscaMetadata) {
 108         Map<String, String> entries = toscaMetadata.getMetaEntries();
 109         String manifestFileName = getFileName(entries.get(TOSCA_META_ETSI_ENTRY_MANIFEST));
 110         String manifestExtension = getFileExtension(entries.get(TOSCA_META_ETSI_ENTRY_MANIFEST));
 111         String mainDefinitionFileName= getFileName(entries.get(TOSCA_META_ENTRY_DEFINITIONS));
 112         if(!(TOSCA_MANIFEST_FILE_EXT).equals(manifestExtension)){
 113             reportError(ErrorLevel.ERROR, Messages.MANIFEST_INVALID_EXT.getErrorMessage());
 114         }
 115         if(!mainDefinitionFileName.equals(manifestFileName)){
 116             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_NAME.getErrorMessage(),
 117                     manifestFileName, mainDefinitionFileName));
 118         }
 119     }
 120
 121     public String getFileExtension(String filePath){
 122         return filePath.substring(filePath.lastIndexOf(".") + 1);
 123     }
 124
 125     private String getFileName(String filePath){
 126         return filePath.substring(filePath.lastIndexOf("/") + 1, filePath.lastIndexOf("."));
 127     }
 128
 129     private boolean hasETSIMetadata(ToscaMetadata toscaMetadata){
 130         Map<String, String> entries = toscaMetadata.getMetaEntries();
 131         return hasEntry(entries, TOSCA_META_FILE_VERSION_ENTRY)
 132                 && hasEntry(entries, TOSCA_META_CSAR_VERSION_ENTRY)
 133                 && hasEntry(entries, TOSCA_META_CREATED_BY_ENTRY);
 134     }
 135
 136     private boolean hasEntry(Map<String, String> entries, String mandatoryEntry) {
 137         if (!entries.containsKey(mandatoryEntry)) {
 138             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_ENTRY.getErrorMessage(),mandatoryEntry));
 139             return false;
 140         }
 141         return true;
 142     }
 143
 144     private void handleMetadataEntries(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata) {
 145         for(Map.Entry entry: toscaMetadata.getMetaEntries().entrySet()){
 146             handleEntry(contentHandler, folderList, toscaMetadata, entry);
 147         }
 148     }
 149
 150     private void handleEntry(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata, Map.Entry entry) {
 151         String key = (String) entry.getKey();
 152         String value = (String) entry.getValue();
 153         switch (key){
 154             case TOSCA_META_FILE_VERSION_ENTRY:
 155             case TOSCA_META_CSAR_VERSION_ENTRY:
 156             case TOSCA_META_CREATED_BY_ENTRY:
 157                 verifyMetadataEntryVersions(key, value);
 158                 break;
 159             case TOSCA_META_ENTRY_DEFINITIONS:
 160                 validateDefinitionFile(contentHandler, value);
 161                 break;
 162             case TOSCA_META_ETSI_ENTRY_MANIFEST:
 163                 validateManifestFile(contentHandler, value);
 164                 break;
 165             case TOSCA_META_ETSI_ENTRY_CHANGE_LOG:
 166                 validateChangeLog(contentHandler, value);
 167                 break;
 168             case TOSCA_META_ETSI_ENTRY_TESTS:
 169             case TOSCA_META_ETSI_ENTRY_LICENSES:
 170                 validateOtherEntries(folderList, entry, contentHandler, toscaMetadata);
 171                 break;
 172             case TOSCA_META_ETSI_ENTRY_CERTIFICATE:
 173                 validateOtherEntries(folderList, value);
 174                 break;
 175             default:
 176                 errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry)));
 177                 LOGGER.warn(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry);
 178                 break;
 179         }
 180     }
 181
 182     private void validateOtherEntries(List<String> folderList, Map.Entry entry, FileContentHandler contentHandler, ToscaMetadata toscaMetadata) {
 183         String manifestFile = toscaMetadata.getMetaEntries().get(TOSCA_META_ETSI_ENTRY_MANIFEST);
 184         if(verifyFileExists(contentHandler.getFileList(), manifestFile)){
 185             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 186             onboardingManifest.parse(contentHandler.getFileContent(manifestFile));
 187             Optional<ResourceTypeEnum> resourceType = onboardingManifest.getType();
 188             if(resourceType.isPresent() && resourceType.get() == ResourceTypeEnum.VF){
 189                 String value = (String) entry.getValue();
 190                 validateOtherEntries(folderList, value);
 191             }else{
 192                 String key = (String) entry.getKey();
 193                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_PNF_METADATA.getErrorMessage(), key));
 194             }
 195
 196         }
 197     }
 198
 199
 200     private void verifyMetadataEntryVersions(String key, String version) {
 201         if(!(isValidTOSCAVersion(key,version) || isValidCSARVersion(key, version) || TOSCA_META_CREATED_BY_ENTRY.equals(key))) {
 202             errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version)));
 203             LOGGER.error("{}: key {} - value {} ", Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version);
 204         }
 205     }
 206
 207     private boolean isValidTOSCAVersion(String key, String version){
 208         return TOSCA_META_FILE_VERSION_ENTRY.equals(key) && TOSCA_META_FILE_VERSION.equals(version);
 209     }
 210
 211     private boolean isValidCSARVersion(String value, String version){
 212         return TOSCA_META_CSAR_VERSION_ENTRY.equals(value) && (CSAR_VERSION_1_1.equals(version)
 213                 || CSAR_VERSION_1_0.equals(version));
 214     }
 215
 216     private void validateDefinitionFile(FileContentHandler contentHandler, String filePath) {
 217         Set<String> existingFiles = contentHandler.getFileList();
 218
 219         if (verifyFileExists(existingFiles, filePath)) {
 220             byte[] definitionFile = getFileContent(filePath, contentHandler);
 221             handleImports(contentHandler, filePath, existingFiles, definitionFile);
 222         }else{
 223             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_DEFINITION_FILE.getErrorMessage(), filePath));
 224         }
 225     }
 226
 227     private void handleImports(FileContentHandler contentHandler, String filePath, Set<String> existingFiles,
 228                                byte[] definitionFile) {
 229         try {
 230             ServiceTemplateReaderService readerService = new ServiceTemplateReaderServiceImpl(definitionFile);
 231             List<Object> imports = (readerService).getImports();
 232             for (Object o : imports) {
 233                 String rootDir = "/";
 234                 if (filePath.contains("/")) {
 235                     rootDir = filePath.substring(0, filePath.lastIndexOf("/"));
 236                 }
 237                 String verifiedFile = verifyImport(existingFiles, o, rootDir);
 238                 if (verifiedFile != null && !verifiedImports.contains(verifiedFile)) {
 239                     verifiedImports.add(verifiedFile);
 240                     handleImports(contentHandler, verifiedFile, existingFiles, getFileContent(verifiedFile,
 241                             contentHandler));
 242                 }
 243             }
 244         }
 245         catch (Exception  e){
 246             reportError(ErrorLevel.ERROR, String.format(Messages.INVALID_YAML_FORMAT.getErrorMessage(), e.getMessage()));
 247             LOGGER.error("{}", Messages.INVALID_YAML_FORMAT_REASON, e.getMessage(), e);
 248         }
 249     }
 250
 251     private String verifyImport(Set<String> existingFiles, Object o, String parentDir) {
 252         if(o instanceof String){
 253             String filePath = ((String) o);
 254             if(!filePath.contains("/")){
 255                 filePath = parentDir + "/" + filePath;
 256             }
 257             if(!verifyFileExists(existingFiles, filePath)){
 258                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_IMPORT_FILE.getErrorMessage(), filePath,
 259                         parentDir));
 260                 return null;
 261             }
 262             return filePath;
 263         } else if(o instanceof Map){
 264             Map<String, Object> o1 = (Map)o;
 265             for(Map.Entry<String, Object> entry: o1.entrySet()){
 266                 if(NON_FILE_IMPORT_ATTRIBUTES.stream().noneMatch(attr -> entry.getKey().equals(attr))){
 267                     verifyImport(existingFiles, entry.getValue(), parentDir);
 268                 }
 269             }
 270         }else {
 271             reportError(ErrorLevel.ERROR, String.format(Messages.INVALID_IMPORT_STATEMENT.getErrorMessage(), parentDir));
 272         }
 273         return null;
 274     }
 275
 276     private boolean verifyFileExists(Set<String> existingFiles, String filePath){
 277         return existingFiles.contains(filePath);
 278     }
 279
 280     private byte[] getFileContent(String filename, FileContentHandler contentHandler){
 281         Map<String, byte[]> files = contentHandler.getFiles();
 282         return files.get(filename);
 283     }
 284
 285     private void validateManifestFile(FileContentHandler contentHandler, String filePath){
 286         final Set<String> existingFiles = contentHandler.getFileList();
 287         if(verifyFileExists(existingFiles, filePath)) {
 288             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 289             onboardingManifest.parse(contentHandler.getFileContent(filePath));
 290             if(onboardingManifest.isValid()){
 291                 try {
 292                     verifyManifestMetadata(onboardingManifest.getMetadata());
 293                 }catch (InvalidManifestMetadataException e){
 294                    reportError(ErrorLevel.ERROR, e.getMessage());
 295                    LOGGER.error(e.getMessage(), e);
 296                 }
 297                 verifyManifestSources(existingFiles, onboardingManifest);
 298             }else{
 299                 List<String> manifestErrors = onboardingManifest.getErrors();
 300                 for(String error: manifestErrors){
 301                     reportError(ErrorLevel.ERROR, error);
 302                 }
 303             }
 304         }else {
 305             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_NOT_FOUND.getErrorMessage(), filePath));
 306         }
 307     }
 308
 309     private void verifyManifestMetadata(Map<String, String> metadata) {
 310         if(metadata.size() != MANIFEST_METADATA_LIMIT){
 311             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_DOES_NOT_MATCH_LIMIT.getErrorMessage(),
 312                     MANIFEST_METADATA_LIMIT));
 313         }
 314         if(isPnfMetadata(metadata)){
 315             handlePnfMetadataEntries(metadata);
 316         }else {
 317             handleVnfMetadataEntries(metadata);
 318         }
 319     }
 320
 321     private boolean isPnfMetadata(Map<String, String> metadata) {
 322         String metadataType = "";
 323         for(String key: metadata.keySet()) {
 324             if(metadataType.isEmpty()){
 325                  metadataType = key.contains(TOSCA_TYPE_PNF) ? TOSCA_TYPE_PNF : TOSCA_TYPE_VNF;
 326             }else if(!key.contains(metadataType)){
 327                 throw new InvalidManifestMetadataException(Messages.MANIFEST_METADATA_INVALID_ENTRY.getErrorMessage());
 328             }
 329         }
 330         return TOSCA_TYPE_PNF.equals(metadataType);
 331     }
 332
 333     private void handleVnfMetadataEntries(Map<String, String> metadata) {
 334         for (String requiredVnfEntry : MANIFEST_VNF_METADATA) {
 335             if (!metadata.containsKey(requiredVnfEntry)) {
 336                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredVnfEntry));
 337             }
 338         }
 339     }
 340
 341     private void handlePnfMetadataEntries(Map<String, String> metadata) {
 342         for (String requiredPnfEntry : MANIFEST_PNF_METADATA) {
 343             if (!metadata.containsKey(requiredPnfEntry)) {
 344                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredPnfEntry));
 345             }
 346         }
 347     }
 348
 349     /**
 350      * Checks if all manifest sources exists within the package and if all package files are being referred.
 351      *
 352      * @param packageFiles          The package file path list
 353      * @param onboardingManifest    The manifest
 354      */
 355     private void verifyManifestSources(final Set<String> packageFiles, final Manifest onboardingManifest) {
 356         final List<String> sources = filterSources(onboardingManifest.getSources());
 357         verifyFilesExist(packageFiles, sources, MANIFEST_SOURCE);
 358
 359         final Map<String, List<String>> nonManoArtifacts = onboardingManifest.getNonManoSources();
 360         final List<String> nonManoFiles = nonManoArtifacts.values().stream()
 361             .map(this::filterSources)
 362             .flatMap(Collection::stream)
 363             .collect(Collectors.toList());
 364         verifyFilesExist(packageFiles, nonManoFiles, MANIFEST_NON_MANO_SOURCE);
 365
 366         final Set<String> allReferredFiles = new HashSet<>();
 367         allReferredFiles.addAll(sources);
 368         allReferredFiles.addAll(nonManoFiles);
 369         verifyFilesBeingReferred(allReferredFiles, packageFiles);
 370     }
 371
 372     /**
 373      * Checks if all package files are referred in manifest.
 374      * Reports missing references.
 375      *
 376      * @param referredFileSet   the list of referred files path
 377      * @param packageFileSet    the list of package file path
 378      */
 379     private void verifyFilesBeingReferred(final Set<String> referredFileSet, final Set<String> packageFileSet) {
 380         packageFileSet.forEach(filePath -> {
 381             if (!referredFileSet.contains(filePath)) {
 382                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_MANIFEST_REFERENCE.getErrorMessage(), filePath));
 383             }
 384         });
 385     }
 386
 387     private List<String> filterSources(List<String> source){
 388         return source.stream()
 389                 .filter(this::externalFileReferences)
 390                 .collect(Collectors.toList());
 391     }
 392
 393     private boolean externalFileReferences(String filePath){
 394         return !filePath.contains("://");
 395     }
 396
 397     private void validateOtherEntries(List<String> folderList, String folderPath){
 398         if(!verifyFoldersExist(folderList, folderPath))
 399             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_OPTIONAL_FOLDERS.getErrorMessage(),
 400                     folderPath));
 401     }
 402
 403     private boolean verifyFoldersExist(List<String> folderList, String folderPath){
 404         return folderList.contains(folderPath + "/");
 405     }
 406
 407     private void verifyFilesExist(Set<String> existingFiles, List<String> sources, String type){
 408         for(String file: sources){
 409             if(!verifyFileExists(existingFiles, file)){
 410                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_MANIFEST_SOURCE.getErrorMessage(), type, file));
 411             }
 412
 413         }
 414     }
 415
 416     private void validateChangeLog(FileContentHandler contentHandler, String filePath){
 417         if(!verifyFileExists(contentHandler.getFileList(), filePath)){
 418             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_METADATA_FILES.getErrorMessage(), filePath));
 419         }
 420     }
 421
 422     private void reportError(ErrorLevel errorLevel, String errorMessage){
 423         errorsByFile.add(new ErrorMessage(errorLevel, errorMessage));
 424     }
 425
 426     private Map<String, List<ErrorMessage>> getAnyValidationErrors(){
 427
 428         if(errorsByFile.isEmpty()){
 429             return Collections.emptyMap();
 430         }
 431         Map<String, List<ErrorMessage>> errors = new HashMap<>();
 432         errors.put(SdcCommon.UPLOAD_FILE, errorsByFile);
 433         return errors;
 434     }
 435 }