gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2019 Nordix Foundation.
   4  * ================================================================================
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *      http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  * SPDX-License-Identifier: Apache-2.0
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.csar.validation;
  22
  23 import org.openecomp.core.converter.ServiceTemplateReaderService;
  24 import org.openecomp.core.impl.services.ServiceTemplateReaderServiceImpl;
  25 import org.openecomp.core.utilities.file.FileContentHandler;
  26 import org.openecomp.sdc.common.errors.Messages;
  27 import org.openecomp.sdc.common.utils.SdcCommon;
  28 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  29 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  30 import org.openecomp.sdc.logging.api.Logger;
  31 import org.openecomp.sdc.logging.api.LoggerFactory;
  32 import org.openecomp.sdc.tosca.csar.Manifest;
  33 import org.openecomp.sdc.tosca.csar.OnboardingToscaMetadata;
  34 import org.openecomp.sdc.tosca.csar.SOL004ManifestOnboarding;
  35 import org.openecomp.sdc.tosca.csar.ToscaMetadata;
  36 import org.openecomp.sdc.vendorsoftwareproduct.impl.orchestration.exceptions.InvalidManifestMetadataException;
  37 import java.io.IOException;
  38 import java.util.ArrayList;
  39 import java.util.Collections;
  40 import java.util.HashMap;
  41 import java.util.HashSet;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Set;
  45 import java.util.stream.Collectors;
  46
  47 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_0;
  48 import static org.openecomp.sdc.tosca.csar.CSARConstants.CSAR_VERSION_1_1;
  49 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_METADATA_LIMIT;
  50 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_PNF_METADATA;
  51 import static org.openecomp.sdc.tosca.csar.CSARConstants.MANIFEST_VNF_METADATA;
  52 import static org.openecomp.sdc.tosca.csar.CSARConstants.NON_FILE_IMPORT_ATTRIBUTES;
  53 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_MANIFEST_FILE_EXT;
  54 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION_ENTRY;
  55 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CREATED_BY_ENTRY;
  56 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_CSAR_VERSION_ENTRY;
  57 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_CHANGE_LOG;
  58 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_DEFINITIONS;
  59 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_LICENSES;
  60 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_MANIFEST;
  61 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_ENTRY_TESTS;
  62 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_FILE_VERSION;
  63 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_META_PATH_FILE_NAME;
  64 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_PNF;
  65 import static org.openecomp.sdc.tosca.csar.CSARConstants.TOSCA_TYPE_VNF;
  66
  67 /**
  68  * Validates the contents of the package to ensure it complies with the "CSAR with TOSCA-Metadata directory" structure
  69  * as defined in ETSI GS NFV-SOL 004 v2.5.1.
  70  *
  71  */
  72
  73 class SOL004MetaDirectoryValidator implements Validator{
  74
  75     private static final Logger LOGGER = LoggerFactory.getLogger(SOL004MetaDirectoryValidator.class);
  76
  77     private final List<ErrorMessage> errorsByFile = new ArrayList<>();
  78     private final Set<String> verifiedImports = new HashSet<>();
  79
  80     @Override
  81     public Map<String, List<ErrorMessage>> validateContent(FileContentHandler contentHandler, List<String> folderList) {
  82             validateMetaFile(contentHandler, folderList);
  83             return Collections.unmodifiableMap(getAnyValidationErrors());
  84     }
  85
  86     private void validateMetaFile(FileContentHandler contentHandler, List<String> folderList) {
  87         try {
  88             ToscaMetadata toscaMetadata = OnboardingToscaMetadata.parseToscaMetadataFile(contentHandler.getFileContent(TOSCA_META_PATH_FILE_NAME));
  89             if(toscaMetadata.isValid() && hasETSIMetadata(toscaMetadata)) {
  90                 verifyManifestNameAndExtension(toscaMetadata);
  91                 handleMetadataEntries(contentHandler, folderList, toscaMetadata);
  92             }else {
  93                 errorsByFile.addAll(toscaMetadata.getErrors());
  94             }
  95         }catch (IOException e){
  96             reportError(ErrorLevel.ERROR, Messages.METADATA_PARSER_INTERNAL.getErrorMessage());
  97             LOGGER.error(Messages.METADATA_PARSER_INTERNAL.getErrorMessage(), e.getMessage(), e);
  98         }
  99     }
 100
 101     private void verifyManifestNameAndExtension(ToscaMetadata toscaMetadata) {
 102         Map<String, String> entries = toscaMetadata.getMetaEntries();
 103         String manifestFileName = getFileName(entries.get(TOSCA_META_ENTRY_MANIFEST));
 104         String manifestExtension = getFileExtension(entries.get(TOSCA_META_ENTRY_MANIFEST));
 105         String mainDefinitionFileName= getFileName(entries.get(TOSCA_META_ENTRY_DEFINITIONS));
 106         if(!(TOSCA_MANIFEST_FILE_EXT).equals(manifestExtension)){
 107             reportError(ErrorLevel.ERROR, Messages.MANIFEST_INVALID_EXT.getErrorMessage());
 108         }
 109         if(!mainDefinitionFileName.equals(manifestFileName)){
 110             reportError(ErrorLevel.ERROR, Messages.MANIFEST_INVALID_NAME.getErrorMessage());
 111         }
 112     }
 113
 114     public String getFileExtension(String filePath){
 115         return filePath.substring(filePath.lastIndexOf(".") + 1);
 116     }
 117
 118     private String getFileName(String filePath){
 119         return filePath.substring(filePath.lastIndexOf("/") + 1, filePath.lastIndexOf("."));
 120     }
 121
 122     private boolean hasETSIMetadata(ToscaMetadata toscaMetadata){
 123         Map<String, String> entries = toscaMetadata.getMetaEntries();
 124         return hasEntry(entries, TOSCA_META_FILE_VERSION_ENTRY)
 125                 && hasEntry(entries, TOSCA_META_CSAR_VERSION_ENTRY)
 126                 && hasEntry(entries, TOSCA_META_CREATED_BY_ENTRY);
 127     }
 128
 129     private boolean hasEntry(Map<String, String> entries, String mandatoryEntry) {
 130         if (!entries.containsKey(mandatoryEntry)) {
 131             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_ENTRY.getErrorMessage(),mandatoryEntry));
 132             return false;
 133         }
 134         return true;
 135     }
 136
 137     private void handleMetadataEntries(FileContentHandler contentHandler, List<String> folderList, ToscaMetadata toscaMetadata) {
 138         for(Map.Entry entry: toscaMetadata.getMetaEntries().entrySet()){
 139             String key = (String) entry.getKey();
 140             String value = (String) entry.getValue();
 141             switch (key){
 142                 case TOSCA_META_FILE_VERSION_ENTRY:
 143                 case TOSCA_META_CSAR_VERSION_ENTRY:
 144                 case TOSCA_META_CREATED_BY_ENTRY:
 145                     verifyMetadataEntryVersions(key, value);
 146                     break;
 147                 case TOSCA_META_ENTRY_DEFINITIONS:
 148                     validateDefinitionFile(contentHandler, value);
 149                     break;
 150                 case TOSCA_META_ENTRY_MANIFEST:
 151                     validateManifestFile(contentHandler, value);
 152                     break;
 153                 case TOSCA_META_ENTRY_CHANGE_LOG:
 154                     validateChangeLog(contentHandler, value);
 155                     break;
 156                 case TOSCA_META_ENTRY_TESTS:
 157                 case TOSCA_META_ENTRY_LICENSES:
 158                     validateOtherEntries(folderList, value);
 159                     break;
 160                 default:
 161                     errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry)));
 162                     LOGGER.warn(Messages.METADATA_UNSUPPORTED_ENTRY.getErrorMessage(), entry);
 163                     break;
 164             }
 165
 166         }
 167     }
 168
 169
 170     private void verifyMetadataEntryVersions(String key, String version) {
 171         if(!(isValidTOSCAVersion(key,version) || isValidCSARVersion(key, version) || TOSCA_META_CREATED_BY_ENTRY.equals(key))) {
 172             errorsByFile.add(new ErrorMessage(ErrorLevel.ERROR, String.format(Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version)));
 173             LOGGER.error("{}: key {} - value {} ", Messages.METADATA_INVALID_VERSION.getErrorMessage(), key, version);
 174         }
 175     }
 176
 177     private boolean isValidTOSCAVersion(String key, String version){
 178         return TOSCA_META_FILE_VERSION_ENTRY.equals(key) && TOSCA_META_FILE_VERSION.equals(version);
 179     }
 180
 181     private boolean isValidCSARVersion(String value, String version){
 182         return TOSCA_META_CSAR_VERSION_ENTRY.equals(value) && (CSAR_VERSION_1_1.equals(version)
 183                 || CSAR_VERSION_1_0.equals(version));
 184     }
 185
 186     private void validateDefinitionFile(FileContentHandler contentHandler, String filePath) {
 187         Set<String> existingFiles = contentHandler.getFileList();
 188
 189         if (verifyFileExists(existingFiles, filePath)) {
 190             byte[] definitionFile = getFileContent(filePath, contentHandler);
 191             handleImports(contentHandler, filePath, existingFiles, definitionFile);
 192         }else{
 193             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_DEFINITION_FILE.getErrorMessage(), filePath));
 194         }
 195     }
 196
 197     private void handleImports(FileContentHandler contentHandler, String filePath, Set<String> existingFiles,
 198                                byte[] definitionFile) {
 199         try {
 200             ServiceTemplateReaderService readerService = new ServiceTemplateReaderServiceImpl(definitionFile);
 201             List<Object> imports = (readerService).getImports();
 202             for (Object o : imports) {
 203                 String rootDir = "/";
 204                 if (filePath.contains("/")) {
 205                     rootDir = filePath.substring(0, filePath.lastIndexOf("/"));
 206                 }
 207                 String verifiedFile = verifyImport(existingFiles, o, rootDir);
 208                 if (verifiedFile != null && !verifiedImports.contains(verifiedFile)) {
 209                     verifiedImports.add(verifiedFile);
 210                     handleImports(contentHandler, verifiedFile, existingFiles, getFileContent(verifiedFile,
 211                             contentHandler));
 212                 }
 213             }
 214         }
 215         catch (Exception  e){
 216             reportError(ErrorLevel.ERROR, String.format(Messages.INVALID_YAML_FORMAT.getErrorMessage(), e.getMessage()));
 217             LOGGER.error("{}", Messages.INVALID_YAML_FORMAT_REASON, e.getMessage(), e);
 218         }
 219     }
 220
 221     private String verifyImport(Set<String> existingFiles, Object o, String parentDir) {
 222         if(o instanceof String){
 223             String filePath = ((String) o);
 224             if(!filePath.contains("/")){
 225                 filePath = parentDir + "/" + filePath;
 226             }
 227             if(!verifyFileExists(existingFiles, filePath)){
 228                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_IMPORT_FILE.getErrorMessage(), (String) o));
 229                 return null;
 230             }
 231             return filePath;
 232         } else if(o instanceof Map){
 233             Map<String, Object> o1 = (Map)o;
 234             for(Map.Entry<String, Object> entry: o1.entrySet()){
 235                 if(NON_FILE_IMPORT_ATTRIBUTES.stream().noneMatch(attr -> entry.getKey().equals(attr))){
 236                     verifyImport(existingFiles, entry.getValue(), parentDir);
 237                 }
 238             }
 239         }else {
 240             reportError(ErrorLevel.ERROR, Messages.INVALID_IMPORT_STATEMENT.getErrorMessage());
 241         }
 242         return null;
 243     }
 244
 245     private boolean verifyFileExists(Set<String> existingFiles, String filePath){
 246         return existingFiles.contains(filePath);
 247     }
 248
 249     private byte[] getFileContent(String filename, FileContentHandler contentHandler){
 250         Map<String, byte[]> files = contentHandler.getFiles();
 251         return files.get(filename);
 252     }
 253
 254     private void validateManifestFile(FileContentHandler contentHandler, String filePath){
 255         final Set<String> exitingFiles = contentHandler.getFileList();
 256         if(verifyFileExists(exitingFiles, filePath)) {
 257             Manifest onboardingManifest = new SOL004ManifestOnboarding();
 258             onboardingManifest.parse(contentHandler.getFileContent(filePath));
 259             if(onboardingManifest.isValid()){
 260                 try {
 261                     verifyManifestMetadata(onboardingManifest.getMetadata());
 262                 }catch (InvalidManifestMetadataException e){
 263                    reportError(ErrorLevel.ERROR, e.getMessage());
 264                    LOGGER.error(e.getMessage(), e);
 265                 }
 266                 verifySourcesExists(exitingFiles, onboardingManifest);
 267             }else{
 268                 List<String> manifestErrors = onboardingManifest.getErrors();
 269                 for(String error: manifestErrors){
 270                     reportError(ErrorLevel.ERROR, error);
 271                 }
 272             }
 273         }else {
 274             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_NOT_EXIST.getErrorMessage(), filePath));
 275         }
 276     }
 277
 278     private void verifyManifestMetadata(Map<String, String> metadata) {
 279         if(metadata.size() != MANIFEST_METADATA_LIMIT){
 280             reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_DOES_NOT_MATCH_LIMIT.getErrorMessage(),
 281                     MANIFEST_METADATA_LIMIT));
 282         }
 283         if(isPnfMetadata(metadata)){
 284             handlePnfMetadataEntries(metadata);
 285         }else {
 286             handleVnfMetadataEntries(metadata);
 287         }
 288     }
 289
 290     private boolean isPnfMetadata(Map<String, String> metadata) {
 291         String metadataType = "";
 292         for(String key: metadata.keySet()) {
 293             if(metadataType.isEmpty()){
 294                  metadataType = key.contains(TOSCA_TYPE_PNF) ? TOSCA_TYPE_PNF : TOSCA_TYPE_VNF;
 295             }else if(!key.contains(metadataType)){
 296                 throw new InvalidManifestMetadataException(Messages.MANIFEST_METADATA_INVALID_ENTRY.getErrorMessage());
 297             }
 298         }
 299         return TOSCA_TYPE_PNF.equals(metadataType);
 300     }
 301
 302     private void handleVnfMetadataEntries(Map<String, String> metadata) {
 303         for (String requiredVnfEntry : MANIFEST_VNF_METADATA) {
 304             if (!metadata.containsKey(requiredVnfEntry)) {
 305                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredVnfEntry));
 306             }
 307         }
 308     }
 309
 310     private void handlePnfMetadataEntries(Map<String, String> metadata) {
 311         for (String requiredPnfEntry : MANIFEST_PNF_METADATA) {
 312             if (!metadata.containsKey(requiredPnfEntry)) {
 313                 reportError(ErrorLevel.ERROR, String.format(Messages.MANIFEST_METADATA_MISSING_ENTRY.getErrorMessage(), requiredPnfEntry));
 314             }
 315         }
 316     }
 317
 318     private void verifySourcesExists(Set<String> exitingFiles, Manifest onboardingManifest) {
 319         List<String> sources = filterSources(onboardingManifest.getSources());
 320         Map<String, List<String>> nonManoArtifacts = onboardingManifest.getNonManoSources();
 321         verifyFilesExist(exitingFiles, sources);
 322         for (Map.Entry entry : nonManoArtifacts.entrySet()) {
 323             verifyFilesExist(exitingFiles, filterSources((List)entry.getValue()));
 324         }
 325     }
 326
 327     private List<String> filterSources(List<String> source){
 328         return source.stream()
 329                 .filter(this::externalFileReferences)
 330                 .collect(Collectors.toList());
 331     }
 332
 333     private boolean externalFileReferences(String filePath){
 334         return !filePath.contains("://");
 335     }
 336
 337     private void validateOtherEntries(List<String> folderList, String folderPath){
 338         if(!verifyFoldersExist(folderList, folderPath))
 339             reportError(ErrorLevel.ERROR, String.format(Messages.METADATA_MISSING_OPTIONAL_FOLDERS.getErrorMessage(),
 340                     folderPath));
 341     }
 342
 343     private boolean verifyFoldersExist(List<String> folderList, String folderPath){
 344         return folderList.contains(folderPath + "/");
 345     }
 346
 347     private void verifyFilesExist(Set<String> existingFiles, List<String> sources){
 348         for(String file: sources){
 349             if(!verifyFileExists(existingFiles, file)){
 350                 reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_ARTIFACT.getErrorMessage(), file));
 351             }
 352
 353         }
 354     }
 355
 356     private void validateChangeLog(FileContentHandler contentHandler, String filePath){
 357         if(!verifyFileExists(contentHandler.getFileList(), filePath)){
 358             reportError(ErrorLevel.ERROR, String.format(Messages.MISSING_ARTIFACT.getErrorMessage(), filePath));
 359         }
 360     }
 361
 362     private void reportError(ErrorLevel errorLevel, String errorMessage){
 363         errorsByFile.add(new ErrorMessage(errorLevel, errorMessage));
 364     }
 365
 366     private Map<String, List<ErrorMessage>> getAnyValidationErrors(){
 367
 368         if(errorsByFile.isEmpty()){
 369             return Collections.emptyMap();
 370         }
 371         Map<String, List<ErrorMessage>> errors = new HashMap<>();
 372         errors.put(SdcCommon.UPLOAD_FILE, errorsByFile);
 373         return errors;
 374     }
 375 }