gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2019 Nordix Foundation.
   4  * ================================================================================
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *      http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  * SPDX-License-Identifier: Apache-2.0
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.csar.validation;
  22
  23 import org.openecomp.core.converter.ServiceTemplateReaderService;
  24 import org.openecomp.core.impl.services.ServiceTemplateReaderServiceImpl;
  25 import org.openecomp.core.utilities.file.FileContentHandler;
  26 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  27 import org.openecomp.sdc.common.errors.Messages;
  28 import org.openecomp.sdc.common.utils.SdcCommon;
  29 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  30 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  31 import org.openecomp.sdc.logging.api.Logger;
  32 import org.openecomp.sdc.logging.api.LoggerFactory;
  33 import org.openecomp.sdc.tosca.csar.Manifest;
  34 import org.openecomp.sdc.tosca.csar.OnboardingToscaMetadata;
  35 import org.openecomp.sdc.tosca.csar.SOL004ManifestOnboarding;
  36 import org.openecomp.sdc.tosca.csar.ToscaMetadata;
  37 import org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.exceptions.InvalidManifestMetadataException;
  38 import java.io.IOException;
  39 import java.util.ArrayList;
  40 import java.util.Collections;
  41 import java.util.HashMap;
  42 import java.util.HashSet;
  43 import java.util.List;
  44 import java.util.Map;
  45 import java.util.Optional;
  46 import java.util.Set;
  47 import java.util.stream.Collectors;
  48
  49 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_0;
  50 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_1;
  51 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_METADATA_LIMIT;
  52 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_PNF_METADATA;
  53 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_VNF_METADATA;
  54 import static org.openecomp.sdc.tosca.csar.CSARConstants.NON_FILE_IMPORT_ATTRIBUTES;
  55 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_MANIFEST_FILE_EXT;
  56 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_CERTIFICATE;
  57 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION_ENTRY;
  58 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CREATED_BY_ENTRY;
  59 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CSAR_VERSION_ENTRY;
  60 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_CHANGE_LOG;
  61 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_DEFINITIONS;
  62 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_LICENSES;
  63 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_MANIFEST;
  64 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ETSI_ENTRY_TESTS;
  65 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION;
  66 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_PATH_FILE_NAME;
  67 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_PNF;
  68 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_VNF;
  69
  70 /**
  71  * Validates the contents of the package to ensure it complies with the "CSAR with TOSCA-Metadata directory" structure
  72  * as defined in ETSI GS NFV-SOL 004 v2.6.1.
  73  *
  74  */
  75
  76 class SOL004MetaDirectoryValidator implements Validator{
  77
  78     private static final Logger LOGGER = LoggerFactory.getLogger(SOL004MetaDirectoryValidator.class);
  79
  80     private static final String MANIFEST_SOURCE = "Source";
  81     private static final String MANIFEST_NON_MANO_SOURCE = "Non-MANO Source";
  82     private final List<ErrorMessage> errorsByFile = new ArrayList<>();
  83     private final Set<String> verifiedImports = new HashSet<>();
  84
  85     @Override
  86     public Map<String, List<ErrorMessage>> validateContent(FileContentHandler contentHandler, List<String> folderList) {
  87             validateMetaFile(contentHandler, folderList);
  88             return Collections.unmodifiableMap(getAnyValidationErrors());
  89     }
  90
  91     private void validateMetaFile(FileContentHandler contentHandler, List<String> folderList) {
  92         try {
  93             ToscaMetadata toscaMetadata = OnboardingToscaMetadata.parseToscaMetadataFile(contentHandler.getFileContent(TOSCA_META_PATH_FILE_NAME));
  94             if(toscaMetadata.isValid() && hasETSIMetadata(toscaMetadata)) {
  95                 verifyManifestNameAndExtension(toscaMetadata);
  96                 handleMetadataEntries(contentHandler, folderList, toscaMetadata);
  97             }else {
  98                 errorsByFile.addAll(toscaMetadata.getErrors());
  99             }
 100         }catch (IOException e){
 101             reportError(ErrorLevel.ERROR, Messages.METADATA_PARSER_INTERNAL.getErrorMessage());
 102             LOGGER.error(Messages.METADATA_PARSER_INTERNAL.getErrorMessage(), e.getMessage(), e);
 103         }
 104     }
 105
 106     private void verifyManifestNameAndExtension(ToscaMetadata toscaMetadata) {
 107         Map<String, String> entries = toscaMetadata.getMetaEntries();
 108         String manifestFileName = getFileName(entries.get(TOSCA_META_ETSI_ENTRY_MANIFEST));
 109         String manifestExtension = getFileExtension(entries.get(TOSCA_META_ETSI_ENTRY_MANIFEST));
 110         String mainDefinitionFileName= getFileName(entries.get(TOSCA_META_ENTRY_DEFINITIONS));
 111         if(!(TOSCA_MANIFEST_FILE_EXT).equals(manifestExtension)){
 112             reportError(ErrorLevel.ERROR, Messages.MANIFEST_INVALID_EXT.getErrorMessage());
 113         }
 114         if(!mainDefinitionFileName.equals(manifestFileName)){
 115             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_NAME.getErrorMessage(),
 116                     manifestFileName, mainDefinitionFileName));
 117         }
 118     }
 119
 120     public String getFileExtension(String filePath){
 121         return filePath.substring(filePath.lastIndexOf(".") + 1);
 122     }
 123
 124     private String getFileName(String filePath){
 125         return filePath.substring(filePath.lastIndexOf("/") + 1, filePath.lastIndexOf("."));
 126     }
 127
 128     private boolean hasETSIMetadata(ToscaMetadata toscaMetadata){
 129         Map<String, String> entries = toscaMetadata.getMetaEntries();
 130         return hasEntry(entries, TOSCA_META_FILE_VERSION_ENTRY)
 131                 && hasEntry(entries, TOSCA_META_CSAR_VERSION_ENTRY)
 132                 && hasEntry(entries, TOSCA_META_CREATED_BY_ENTRY);
 133     }
 134
 135     private boolean hasEntry(Map<String, String> entries, String mandatoryEntry) {
 136         if (!entries.containsKey(mandatoryEntry)) {
 137             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_ENTRY.getErrorMessage(),mandatoryEntry));
 138             return false;
 139         }
 140         return true;
 141     }
 142
 143     private void handleMetadataEntries(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata) {
 144         for(Map.Entry entry: toscaMetadata.getMetaEntries().entrySet()){
 145             handleEntry(contentHandler, folderList, toscaMetadata, entry);
 146         }
 147     }
 148
 149     private void handleEntry(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata, Map.Entry entry) {
 150         String key = (String) entry.getKey();
 151         String value = (String) entry.getValue();
 152         switch (key){
 153             case TOSCA_META_FILE_VERSION_ENTRY:
 154             case TOSCA_META_CSAR_VERSION_ENTRY:
 155             case TOSCA_META_CREATED_BY_ENTRY:
 156                 verifyMetadataEntryVersions(key, value);
 157                 break;
 158             case TOSCA_META_ENTRY_DEFINITIONS:
 159                 validateDefinitionFile(contentHandler, value);
 160                 break;
 161             case TOSCA_META_ETSI_ENTRY_MANIFEST:
 162                 validateManifestFile(contentHandler, value);
 163                 break;
 164             case TOSCA_META_ETSI_ENTRY_CHANGE_LOG:
 165                 validateChangeLog(contentHandler, value);
 166                 break;
 167             case TOSCA_META_ETSI_ENTRY_TESTS:
 168             case TOSCA_META_ETSI_ENTRY_LICENSES:
 169                 validateOtherEntries(folderList, entry, contentHandler, toscaMetadata);
 170                 break;
 171             case TOSCA_META_ETSI_ENTRY_CERTIFICATE:
 172                 validateOtherEntries(folderList, value);
 173                 break;
 174             default:
 175                 errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry)));
 176                 LOGGER.warn(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry);
 177                 break;
 178         }
 179     }
 180
 181     private void validateOtherEntries(List<String> folderList, Map.Entry entry, FileContentHandler contentHandler, ToscaMetadata toscaMetadata) {
 182         String manifestFile = toscaMetadata.getMetaEntries().get(TOSCA_META_ETSI_ENTRY_MANIFEST);
 183         if(verifyFileExists(contentHandler.getFileList(), manifestFile)){
 184             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 185             onboardingManifest.parse(contentHandler.getFileContent(manifestFile));
 186             Optional<ResourceTypeEnum> resourceType = onboardingManifest.getType();
 187             if(resourceType.isPresent() && resourceType.get() == ResourceTypeEnum.VF){
 188                 String value = (String) entry.getValue();
 189                 validateOtherEntries(folderList, value);
 190             }else{
 191                 String key = (String) entry.getKey();
 192                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_INVALID_PNF_METADATA.getErrorMessage(), key));
 193             }
 194
 195         }
 196     }
 197
 198
 199     private void verifyMetadataEntryVersions(String key, String version) {
 200         if(!(isValidTOSCAVersion(key,version) || isValidCSARVersion(key, version) || TOSCA_META_CREATED_BY_ENTRY.equals(key))) {
 201             errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version)));
 202             LOGGER.error("{}: key {} - value {} ", Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version);
 203         }
 204     }
 205
 206     private boolean isValidTOSCAVersion(String key, String version){
 207         return TOSCA_META_FILE_VERSION_ENTRY.equals(key) && TOSCA_META_FILE_VERSION.equals(version);
 208     }
 209
 210     private boolean isValidCSARVersion(String value, String version){
 211         return TOSCA_META_CSAR_VERSION_ENTRY.equals(value) && (CSAR_VERSION_1_1.equals(version)
 212                 || CSAR_VERSION_1_0.equals(version));
 213     }
 214
 215     private void validateDefinitionFile(FileContentHandler contentHandler, String filePath) {
 216         Set<String> existingFiles = contentHandler.getFileList();
 217
 218         if (verifyFileExists(existingFiles, filePath)) {
 219             byte[] definitionFile = getFileContent(filePath, contentHandler);
 220             handleImports(contentHandler, filePath, existingFiles, definitionFile);
 221         }else{
 222             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_DEFINITION_FILE.getErrorMessage(), filePath));
 223         }
 224     }
 225
 226     private void handleImports(FileContentHandler contentHandler, String filePath, Set<String> existingFiles,
 227                                byte[] definitionFile) {
 228         try {
 229             ServiceTemplateReaderService readerService = new ServiceTemplateReaderServiceImpl(definitionFile);
 230             List<Object> imports = (readerService).getImports();
 231             for (Object o : imports) {
 232                 String rootDir = "/";
 233                 if (filePath.contains("/")) {
 234                     rootDir = filePath.substring(0, filePath.lastIndexOf("/"));
 235                 }
 236                 String verifiedFile = verifyImport(existingFiles, o, rootDir);
 237                 if (verifiedFile != null && !verifiedImports.contains(verifiedFile)) {
 238                     verifiedImports.add(verifiedFile);
 239                     handleImports(contentHandler, verifiedFile, existingFiles, getFileContent(verifiedFile,
 240                             contentHandler));
 241                 }
 242             }
 243         }
 244         catch (Exception  e){
 245             reportError(ErrorLevel.ERROR, String.format(Messages.INVALID_YAML_FORMAT.getErrorMessage(), e.getMessage()));
 246             LOGGER.error("{}", Messages.INVALID_YAML_FORMAT_REASON, e.getMessage(), e);
 247         }
 248     }
 249
 250     private String verifyImport(Set<String> existingFiles, Object o, String parentDir) {
 251         if(o instanceof String){
 252             String filePath = ((String) o);
 253             if(!filePath.contains("/")){
 254                 filePath = parentDir + "/" + filePath;
 255             }
 256             if(!verifyFileExists(existingFiles, filePath)){
 257                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_IMPORT_FILE.getErrorMessage(), filePath,
 258                         parentDir));
 259                 return null;
 260             }
 261             return filePath;
 262         } else if(o instanceof Map){
 263             Map<String, Object> o1 = (Map)o;
 264             for(Map.Entry<String, Object> entry: o1.entrySet()){
 265                 if(NON_FILE_IMPORT_ATTRIBUTES.stream().noneMatch(attr -> entry.getKey().equals(attr))){
 266                     verifyImport(existingFiles, entry.getValue(), parentDir);
 267                 }
 268             }
 269         }else {
 270             reportError(ErrorLevel.ERROR, String.format(Messages.INVALID_IMPORT_STATEMENT.getErrorMessage(), parentDir));
 271         }
 272         return null;
 273     }
 274
 275     private boolean verifyFileExists(Set<String> existingFiles, String filePath){
 276         return existingFiles.contains(filePath);
 277     }
 278
 279     private byte[] getFileContent(String filename, FileContentHandler contentHandler){
 280         Map<String, byte[]> files = contentHandler.getFiles();
 281         return files.get(filename);
 282     }
 283
 284     private void validateManifestFile(FileContentHandler contentHandler, String filePath){
 285         final Set<String> exitingFiles = contentHandler.getFileList();
 286         if(verifyFileExists(exitingFiles, filePath)) {
 287             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 288             onboardingManifest.parse(contentHandler.getFileContent(filePath));
 289             if(onboardingManifest.isValid()){
 290                 try {
 291                     verifyManifestMetadata(onboardingManifest.getMetadata());
 292                 }catch (InvalidManifestMetadataException e){
 293                    reportError(ErrorLevel.ERROR, e.getMessage());
 294                    LOGGER.error(e.getMessage(), e);
 295                 }
 296                 verifySourcesExists(exitingFiles, onboardingManifest);
 297             }else{
 298                 List<String> manifestErrors = onboardingManifest.getErrors();
 299                 for(String error: manifestErrors){
 300                     reportError(ErrorLevel.ERROR, error);
 301                 }
 302             }
 303         }else {
 304             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_NOT_FOUND.getErrorMessage(), filePath));
 305         }
 306     }
 307
 308     private void verifyManifestMetadata(Map<String, String> metadata) {
 309         if(metadata.size() != MANIFEST_METADATA_LIMIT){
 310             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_DOES_NOT_MATCH_LIMIT.getErrorMessage(),
 311                     MANIFEST_METADATA_LIMIT));
 312         }
 313         if(isPnfMetadata(metadata)){
 314             handlePnfMetadataEntries(metadata);
 315         }else {
 316             handleVnfMetadataEntries(metadata);
 317         }
 318     }
 319
 320     private boolean isPnfMetadata(Map<String, String> metadata) {
 321         String metadataType = "";
 322         for(String key: metadata.keySet()) {
 323             if(metadataType.isEmpty()){
 324                  metadataType = key.contains(TOSCA_TYPE_PNF) ? TOSCA_TYPE_PNF : TOSCA_TYPE_VNF;
 325             }else if(!key.contains(metadataType)){
 326                 throw new InvalidManifestMetadataException(Messages.MANIFEST_METADATA_INVALID_ENTRY.getErrorMessage());
 327             }
 328         }
 329         return TOSCA_TYPE_PNF.equals(metadataType);
 330     }
 331
 332     private void handleVnfMetadataEntries(Map<String, String> metadata) {
 333         for (String requiredVnfEntry : MANIFEST_VNF_METADATA) {
 334             if (!metadata.containsKey(requiredVnfEntry)) {
 335                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredVnfEntry));
 336             }
 337         }
 338     }
 339
 340     private void handlePnfMetadataEntries(Map<String, String> metadata) {
 341         for (String requiredPnfEntry : MANIFEST_PNF_METADATA) {
 342             if (!metadata.containsKey(requiredPnfEntry)) {
 343                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredPnfEntry));
 344             }
 345         }
 346     }
 347
 348     private void verifySourcesExists(Set<String> exitingFiles, Manifest onboardingManifest) {
 349         List<String> sources = filterSources(onboardingManifest.getSources());
 350         Map<String, List<String>> nonManoArtifacts = onboardingManifest.getNonManoSources();
 351         verifyFilesExist(exitingFiles, sources, MANIFEST_SOURCE);
 352         for (Map.Entry entry : nonManoArtifacts.entrySet()) {
 353             verifyFilesExist(exitingFiles, filterSources((List)entry.getValue()), MANIFEST_NON_MANO_SOURCE);
 354         }
 355     }
 356
 357     private List<String> filterSources(List<String> source){
 358         return source.stream()
 359                 .filter(this::externalFileReferences)
 360                 .collect(Collectors.toList());
 361     }
 362
 363     private boolean externalFileReferences(String filePath){
 364         return !filePath.contains("://");
 365     }
 366
 367     private void validateOtherEntries(List<String> folderList, String folderPath){
 368         if(!verifyFoldersExist(folderList, folderPath))
 369             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_OPTIONAL_FOLDERS.getErrorMessage(),
 370                     folderPath));
 371     }
 372
 373     private boolean verifyFoldersExist(List<String> folderList, String folderPath){
 374         return folderList.contains(folderPath + "/");
 375     }
 376
 377     private void verifyFilesExist(Set<String> existingFiles, List<String> sources, String type){
 378         for(String file: sources){
 379             if(!verifyFileExists(existingFiles, file)){
 380                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_MANIFEST_SOURCE.getErrorMessage(), type, file));
 381             }
 382
 383         }
 384     }
 385
 386     private void validateChangeLog(FileContentHandler contentHandler, String filePath){
 387         if(!verifyFileExists(contentHandler.getFileList(), filePath)){
 388             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_METADATA_FILES.getErrorMessage(), filePath));
 389         }
 390     }
 391
 392     private void reportError(ErrorLevel errorLevel, String errorMessage){
 393         errorsByFile.add(new ErrorMessage(errorLevel, errorMessage));
 394     }
 395
 396     private Map<String, List<ErrorMessage>> getAnyValidationErrors(){
 397
 398         if(errorsByFile.isEmpty()){
 399             return Collections.emptyMap();
 400         }
 401         Map<String, List<ErrorMessage>> errors = new HashMap<>();
 402         errors.put(SdcCommon.UPLOAD_FILE, errorsByFile);
 403         return errors;
 404     }
 405 }