gerrit.onap Code Review - sdc.git/blob

   1 /*
   2  * Copyright © 2016-2018 European Support Limited
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package org.openecomp.sdc.vendorsoftwareproduct.services.impl.filedatastructuremodule;
  18
  19 import static org.openecomp.core.validation.errors.ErrorMessagesFormatBuilder.getErrorWithParameters;
  20
  21 import java.io.ByteArrayInputStream;
  22 import java.io.ByteArrayOutputStream;
  23 import java.io.IOException;
  24 import java.io.InputStream;
  25 import java.nio.ByteBuffer;
  26 import java.nio.charset.StandardCharsets;
  27 import java.util.ArrayList;
  28 import java.util.HashSet;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Objects;
  32 import java.util.Optional;
  33 import java.util.Set;
  34 import java.util.stream.Collectors;
  35 import java.util.zip.ZipEntry;
  36 import java.util.zip.ZipInputStream;
  37 import java.util.zip.ZipOutputStream;
  38 import org.apache.commons.collections4.CollectionUtils;
  39 import org.openecomp.core.utilities.file.FileContentHandler;
  40 import org.openecomp.core.utilities.json.JsonUtil;
  41 import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
  42 import org.openecomp.sdc.common.errors.CoreException;
  43 import org.openecomp.sdc.common.errors.ErrorCategory;
  44 import org.openecomp.sdc.common.errors.ErrorCode;
  45 import org.openecomp.sdc.common.errors.Messages;
  46 import org.openecomp.sdc.common.utils.SdcCommon;
  47 import org.openecomp.sdc.common.zip.ZipUtils;
  48 import org.openecomp.sdc.common.zip.exception.ZipSlipException;
  49 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  50 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  51 import org.openecomp.sdc.heat.datatypes.manifest.FileData;
  52 import org.openecomp.sdc.heat.datatypes.manifest.ManifestContent;
  53 import org.openecomp.sdc.heat.datatypes.structure.Artifact;
  54 import org.openecomp.sdc.heat.datatypes.structure.HeatStructureTree;
  55 import org.openecomp.sdc.heat.datatypes.structure.ValidationStructureList;
  56 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDao;
  57 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDaoFactory;
  58 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.OrchestrationTemplateCandidateData;
  59 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.VspDetails;
  60 import org.openecomp.sdc.vendorsoftwareproduct.errors.utils.ErrorsUtil;
  61 import org.openecomp.sdc.vendorsoftwareproduct.services.HeatFileAnalyzer;
  62 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.CandidateService;
  63 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.ManifestCreator;
  64 import org.openecomp.sdc.vendorsoftwareproduct.services.utils.CandidateServiceValidator;
  65 import org.openecomp.sdc.vendorsoftwareproduct.types.CandidateDataEntityTo;
  66 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.AnalyzedZipHeatFiles;
  67 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.FilesDataStructure;
  68 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.Module;
  69 import org.openecomp.sdc.versioning.dao.types.Version;
  70 import org.slf4j.Logger;
  71 import org.slf4j.LoggerFactory;
  72
  73 public class CandidateServiceImpl implements CandidateService {
  74   private static final Logger logger = LoggerFactory.getLogger(CandidateServiceImpl.class);
  75   private CandidateServiceValidator candidateServiceValidator = new CandidateServiceValidator();
  76   private ManifestCreator manifestCreator;
  77   private OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDao;
  78
  79   public CandidateServiceImpl(ManifestCreator manifestCreator,
  80                               OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDao) {
  81     this.manifestCreator = manifestCreator;
  82     this.orchestrationTemplateCandidateDao = orchestrationTemplateCandidateDao;
  83   }
  84
  85   public CandidateServiceImpl() {
  86   }
  87
  88   @Override
  89   public Optional<ErrorMessage> validateNonEmptyFileToUpload(InputStream fileToUpload,
  90                                                              String fileSuffix) {
  91     String errorMessage =
  92         getErrorWithParameters(Messages.NO_FILE_WAS_UPLOADED_OR_FILE_NOT_EXIST.getErrorMessage(),
  93             fileSuffix);
  94
  95     if (Objects.isNull(fileToUpload)) {
  96       return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
  97           errorMessage));
  98     } else {
  99       try {
 100         int available = fileToUpload.available();
 101         if (available == 0) {
 102           return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 103               errorMessage));
 104         }
 105       } catch (IOException e) {
 106         logger.debug(e.getMessage(), e);
 107         return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 108             errorMessage));
 109       }
 110     }
 111     return Optional.empty();
 112   }
 113
 114   @Override
 115   public Optional<ErrorMessage> validateRawZipData(String fileSuffix,
 116                                                    byte[] uploadedFileData) {
 117     if (Objects.isNull(uploadedFileData)) {
 118       return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 119           getErrorWithParameters(Messages.NO_FILE_WAS_UPLOADED_OR_FILE_NOT_EXIST.getErrorMessage(),
 120               fileSuffix)));
 121     }
 122     return Optional.empty();
 123   }
 124
 125   private String heatStructureTreeToFileDataStructure(HeatStructureTree tree,
 126                                                       FileContentHandler zipContentMap,
 127                                                       Map<String, List<ErrorMessage>> uploadErrors,
 128                                                       AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 129     FilesDataStructure structure = new FilesDataStructure();
 130     Set<String> usedEnvFiles = new HashSet<>();
 131     addHeatsToFileDataStructure(tree, usedEnvFiles, structure, uploadErrors,
 132         analyzedZipHeatFiles);
 133     handleOtherResources(tree, usedEnvFiles, structure);
 134     FilesDataStructure fileDataStructureFromManifest =
 135         createFileDataStructureFromManifest(zipContentMap.getFileContentAsStream(SdcCommon.MANIFEST_NAME));
 136     List<String> structureArtifacts = structure.getArtifacts();
 137     structureArtifacts.addAll(fileDataStructureFromManifest.getArtifacts().stream().filter
 138         (artifact -> isNotStrctureArtifact(structureArtifacts, artifact))
 139         .collect(Collectors.toList()));
 140     handleArtifactsFromTree(tree, structure);
 141
 142     return JsonUtil.object2Json(structure);
 143   }
 144
 145   private boolean isNotStrctureArtifact(List<String> structureArtifacts, String artifact) {
 146     return !structureArtifacts.contains(artifact);
 147   }
 148
 149   @Override
 150   public OrchestrationTemplateCandidateData createCandidateDataEntity(
 151       CandidateDataEntityTo candidateDataEntityTo, InputStream zipFileManifest,
 152       AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 153     FileContentHandler zipContentMap = candidateDataEntityTo.getContentMap();
 154     FilesDataStructure filesDataStructure;
 155     String dataStructureJson;
 156
 157     if (zipFileManifest != null) {
 158       // create data structure from manifest
 159       filesDataStructure = createFileDataStructureFromManifest(zipFileManifest);
 160       Set<String> zipFileList = zipContentMap.getFileList();
 161       balanceManifestFilesWithZipFiles(filesDataStructure,
 162           zipContentMap, analyzedZipHeatFiles);
 163       Set<String> filesDataStructureFiles = getFlatFileNames(filesDataStructure);
 164       filesDataStructure.getUnassigned().addAll(zipFileList.stream()
 165           .filter(fileName -> (!filesDataStructureFiles.contains(fileName)
 166               && !filesDataStructure.getNested().contains(fileName)
 167               && !fileName.equals(SdcCommon.MANIFEST_NAME)))
 168           .collect(Collectors.toList()));
 169       dataStructureJson = JsonUtil.object2Json(filesDataStructure);
 170     } else {
 171       // create data structure from based on naming convention
 172       dataStructureJson =
 173           heatStructureTreeToFileDataStructure(candidateDataEntityTo.getTree(), zipContentMap,
 174               candidateDataEntityTo.getErrors(), analyzedZipHeatFiles);
 175     }
 176
 177     OrchestrationTemplateCandidateData candidateData = new OrchestrationTemplateCandidateData();
 178     candidateData.setContentData(ByteBuffer.wrap(candidateDataEntityTo.getUploadedFileData()));
 179     candidateData.setFilesDataStructure(dataStructureJson);
 180     return candidateData;
 181   }
 182
 183   private void balanceManifestFilesWithZipFiles(
 184       FilesDataStructure filesDataStructure,
 185       FileContentHandler fileContentHandler, AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 186     Set<String> zipFileList = fileContentHandler.getFileList();
 187     filesDataStructure.getNested().addAll(analyzedZipHeatFiles.getNestedFiles());
 188     List<Module> modules = filesDataStructure.getModules();
 189     if (CollectionUtils.isEmpty(modules)) {
 190       return;
 191     }
 192
 193     for (int i = 0; i < modules.size(); i++) {
 194       Module module = modules.get(i);
 195       if (!isFileExistInZipContains(zipFileList, module.getYaml())) {
 196         addFileToUnassigned(filesDataStructure, zipFileList, module.getEnv());
 197         addFileToUnassigned(filesDataStructure, zipFileList, module.getVol());
 198         addFileToUnassigned(filesDataStructure, zipFileList, module.getVolEnv());
 199         modules.remove(i--);
 200       } else if (Objects.nonNull(module.getVol()) && !zipFileList.contains(module.getVol())) {
 201         module.setVol(null);
 202         CollectionUtils
 203             .addIgnoreNull(filesDataStructure.getUnassigned(), module.getVolEnv());
 204       } else {
 205         if (filesDataStructure.getNested().contains(module.getYaml())) {
 206           moveModuleFileToNested(filesDataStructure, i--, module);
 207         }
 208       }
 209     }
 210   }
 211
 212   private void addFileToUnassigned(FilesDataStructure filesDataStructure, Set<String> zipFileList,
 213                                    String fileName) {
 214     if (isFileExistInZipContains(zipFileList, fileName)) {
 215       filesDataStructure.getUnassigned().add(fileName);
 216     }
 217   }
 218
 219   private boolean isFileExistInZipContains(Set<String> zipFileList, String fileName) {
 220     return Objects.nonNull(fileName) && zipFileList.contains(fileName);
 221   }
 222
 223   private void moveModuleFileToNested(FilesDataStructure filesDataStructure, int i,
 224                                       Module module) {
 225     if (!filesDataStructure.getNested().contains(module.getYaml())) {
 226       filesDataStructure.getNested().add(module.getYaml());
 227     }
 228     if (Objects.nonNull(module.getEnv())) {
 229       filesDataStructure.getNested().add(module.getEnv());
 230     }
 231     if (Objects.nonNull(module.getVol())) {
 232       filesDataStructure.getNested().add(module.getVol());
 233     }
 234     if (Objects.nonNull(module.getVolEnv())) {
 235       filesDataStructure.getNested().add(module.getVolEnv());
 236     }
 237     filesDataStructure.getModules().remove(i);
 238   }
 239
 240   private Set<String> getFlatFileNames(FilesDataStructure filesDataStructure) {
 241     Set<String> fileNames = new HashSet<>();
 242     if (!CollectionUtils.isEmpty(filesDataStructure.getModules())) {
 243       for (Module module : filesDataStructure.getModules()) {
 244         CollectionUtils.addIgnoreNull(fileNames, module.getEnv());
 245         CollectionUtils.addIgnoreNull(fileNames, module.getVol());
 246         CollectionUtils.addIgnoreNull(fileNames, module.getVolEnv());
 247         CollectionUtils.addIgnoreNull(fileNames, module.getYaml());
 248       }
 249     }
 250     fileNames.addAll(filesDataStructure.getArtifacts());
 251     fileNames.addAll(filesDataStructure.getNested());
 252     fileNames.addAll(filesDataStructure.getUnassigned());
 253
 254     return fileNames;
 255   }
 256
 257   private FilesDataStructure createFileDataStructureFromManifest(InputStream isManifestContent) {
 258     ManifestContent manifestContent =
 259         JsonUtil.json2Object(isManifestContent, ManifestContent.class);
 260     FilesDataStructure structure = new FilesDataStructure();
 261     for (FileData fileData : manifestContent.getData()) {
 262       if (Objects.nonNull(fileData.getType()) &&
 263           fileData.getType().equals(FileData.Type.HEAT)) {
 264         Module module = new Module();
 265         module.setType(FileData.Type.HEAT);
 266         module.setYaml(fileData.getFile());
 267         module.setIsBase(fileData.getBase());
 268         addHeatDependenciesToModule(module, fileData.getData());
 269         structure.getModules().add(module);
 270       }else if (Objects.nonNull(fileData.getType()) &&
 271               fileData.getType().equals(FileData.Type.HELM)) {
 272         Module module = new Module();
 273         module.setType(FileData.Type.HELM);
 274         module.setYaml(fileData.getFile());
 275         module.setIsBase(fileData.getBase());
 276         structure.getModules().add(module);
 277       }
 278       else if (HeatFileAnalyzer.isYamlOrEnvFile(fileData.getFile()) &&
 279           !FileData.Type.isArtifact(fileData.getType())) {
 280         structure.getUnassigned().add(fileData.getFile());
 281       } else {
 282         structure.getArtifacts().add(fileData.getFile());
 283       }
 284     }
 285     return structure;
 286   }
 287
 288   private void addHeatDependenciesToModule(Module module, List<FileData> data) {
 289     if (CollectionUtils.isEmpty(data)) {
 290       return;
 291     }
 292
 293     for (FileData fileData : data) {
 294       if (fileData.getType().equals(FileData.Type.HEAT_ENV)) {
 295         module.setEnv(fileData.getFile());
 296       } else if (fileData.getType().equals(FileData.Type.HEAT_VOL)) { // must be volume
 297         module.setVol(fileData.getFile());
 298         if (!CollectionUtils.isEmpty(fileData.getData())) {
 299           FileData volEnv = fileData.getData().get(0);
 300           if (volEnv.getType().equals(FileData.Type.HEAT_ENV)) {
 301             module.setVolEnv(volEnv.getFile());
 302           } else {
 303             throw new CoreException((new ErrorCode.ErrorCodeBuilder())
 304                 .withMessage(Messages.ILLEGAL_MANIFEST.getErrorMessage())
 305                 .withId(Messages.ILLEGAL_MANIFEST.getErrorMessage())
 306                 .withCategory(ErrorCategory.APPLICATION).build());
 307           }
 308         }
 309       } else {
 310         throw new CoreException((new ErrorCode.ErrorCodeBuilder())
 311             .withMessage(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
 312             .withId(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
 313             .withCategory(ErrorCategory.APPLICATION).build());
 314       }
 315     }
 316   }
 317
 318   @Override
 319   public void updateCandidateUploadData(final String vspId, final Version version,
 320                                         final OrchestrationTemplateCandidateData uploadData) {
 321     orchestrationTemplateCandidateDao.update(vspId, version, uploadData);
 322   }
 323
 324   @Override
 325   public Optional<FilesDataStructure> getOrchestrationTemplateCandidateFileDataStructure(
 326       String vspId, Version version) {
 327     Optional<String> jsonFileDataStructure =
 328         orchestrationTemplateCandidateDao.getStructure(vspId, version);
 329
 330     if (jsonFileDataStructure.isPresent() && JsonUtil.isValidJson(jsonFileDataStructure.get())) {
 331       return Optional
 332           .of(JsonUtil.json2Object(jsonFileDataStructure.get(), FilesDataStructure.class));
 333     } else {
 334       return Optional.empty();
 335     }
 336   }
 337
 338   @Override
 339   public void updateOrchestrationTemplateCandidateFileDataStructure(String vspId, Version version,
 340                                                                     FilesDataStructure fileDataStructure) {
 341     OrchestrationTemplateCandidateDaoFactory.getInstance().createInterface()
 342         .updateStructure(vspId, version, fileDataStructure);
 343   }
 344
 345   @Override
 346   public Optional<OrchestrationTemplateCandidateData> getOrchestrationTemplateCandidate(String vspId,
 347                                                                                         Version version) {
 348     return orchestrationTemplateCandidateDao.get(vspId, version);
 349   }
 350
 351   @Override
 352   public Optional<OrchestrationTemplateCandidateData> getOrchestrationTemplateCandidateInfo(
 353       String vspId,
 354       Version version) {
 355     return orchestrationTemplateCandidateDao.getInfo(vspId, version);
 356   }
 357
 358   @Override
 359   public String createManifest(VspDetails vspDetails, FilesDataStructure structure) {
 360     return JsonUtil.object2Json(manifestCreator.createManifest(vspDetails, structure)
 361         .orElseThrow(() -> new CoreException(new ErrorCode.ErrorCodeBuilder()
 362             .withMessage(Messages.CREATE_MANIFEST_FROM_ZIP.getErrorMessage()).build())));
 363   }
 364
 365   @Override
 366   public Optional<ManifestContent> createManifest(VspDetails vspDetails,
 367                                                   FileContentHandler fileContentHandler,
 368                                                   AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 369     return manifestCreator.createManifest(vspDetails, fileContentHandler, analyzedZipHeatFiles);
 370   }
 371
 372   @Override
 373   public Optional<ByteArrayInputStream> fetchZipFileByteArrayInputStream(String vspId,
 374                                                                          OrchestrationTemplateCandidateData candidateDataEntity,
 375                                                                          String manifest,
 376                                                                          OnboardingTypesEnum type,
 377                                                                          Map<String, List<ErrorMessage>> uploadErrors) {
 378     byte[] file;
 379     ByteArrayInputStream byteArrayInputStream = null;
 380     try {
 381       file = replaceManifestInZip(candidateDataEntity.getContentData(), manifest, type);
 382       byteArrayInputStream = new ByteArrayInputStream(
 383           Objects.isNull(file) ? candidateDataEntity.getContentData().array()
 384               : file);
 385     } catch (IOException e) {
 386       ErrorMessage errorMessage =
 387           new ErrorMessage(ErrorLevel.ERROR,
 388               Messages.CANDIDATE_PROCESS_FAILED.getErrorMessage());
 389       logger.error(errorMessage.getMessage(), e);
 390       ErrorsUtil
 391           .addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE, errorMessage, uploadErrors);
 392     }
 393     return Optional.ofNullable(byteArrayInputStream);
 394   }
 395
 396   @Override
 397   public byte[] replaceManifestInZip(ByteBuffer contentData, String manifest,
 398                                      OnboardingTypesEnum type)
 399       throws IOException {
 400     ByteArrayOutputStream baos = new ByteArrayOutputStream();
 401
 402     try (final ZipOutputStream zos = new ZipOutputStream(baos);
 403          ZipInputStream zipStream = new ZipInputStream(
 404              new ByteArrayInputStream(contentData.array()))) {
 405       ZipEntry zipEntry;
 406       boolean manifestWritten = false;
 407       while ((zipEntry = zipStream.getNextEntry()) != null) {
 408         if (!zipEntry.getName().equalsIgnoreCase(SdcCommon.MANIFEST_NAME)) {
 409           ZipEntry loc_ze = new ZipEntry(zipEntry.getName());
 410           zos.putNextEntry(loc_ze);
 411           byte[] buf = new byte[1024];
 412           int len;
 413           while ((len = zipStream.read(buf)) > 0) {
 414             zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 415           }
 416         } else {
 417           manifestWritten = true;
 418           writeManifest(manifest, type, zos);
 419         }
 420         zos.closeEntry();
 421       }
 422       if (!manifestWritten) {
 423         writeManifest(manifest, type, zos);
 424         zos.closeEntry();
 425       }
 426     }
 427     return baos.toByteArray();
 428   }
 429
 430   @Override
 431   public byte[] getZipData(ByteBuffer contentData)
 432       throws IOException {
 433     ByteArrayOutputStream baos = new ByteArrayOutputStream();
 434
 435     try (final ZipOutputStream zos = new ZipOutputStream(baos);
 436          ZipInputStream zipStream = new ZipInputStream(
 437              new ByteArrayInputStream(contentData.array()))) {
 438       ZipEntry zipEntry;
 439       while ((zipEntry = zipStream.getNextEntry()) != null) {
 440         try {
 441           ZipUtils.checkForZipSlipInRead(zipEntry);
 442         } catch (ZipSlipException e) {
 443           throw new IOException(e);
 444         }
 445         ZipEntry locZipEntry = new ZipEntry(zipEntry.getName());
 446         zos.putNextEntry(locZipEntry);
 447         byte[] buf = new byte[1024];
 448         int len;
 449         while ((len = zipStream.read(buf)) > 0) {
 450           zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 451         }
 452         zos.closeEntry();
 453       }
 454     }
 455     return baos.toByteArray();
 456   }
 457
 458   @Override
 459   public Optional<List<ErrorMessage>> validateFileDataStructure(
 460       FilesDataStructure filesDataStructure) {
 461     return candidateServiceValidator.validateFileDataStructure(filesDataStructure);
 462   }
 463
 464   @Override
 465   public void deleteOrchestrationTemplateCandidate(String vspId, Version versionId) {
 466     orchestrationTemplateCandidateDao.delete(vspId, versionId);
 467   }
 468
 469   @Override
 470   public void updateValidationData(String vspId, Version version, ValidationStructureList
 471       validationData) {
 472     orchestrationTemplateCandidateDao.updateValidationData(vspId, version, validationData);
 473   }
 474
 475   private void writeManifest(String manifest,
 476                              OnboardingTypesEnum type,
 477                              ZipOutputStream zos) throws IOException {
 478
 479     if (isManifestNeedsToGetWritten(type)) {
 480       return;
 481     }
 482
 483     zos.putNextEntry(new ZipEntry(SdcCommon.MANIFEST_NAME));
 484     try (InputStream manifestStream = new ByteArrayInputStream(
 485         manifest.getBytes(StandardCharsets.UTF_8))) {
 486       byte[] buf = new byte[1024];
 487       int len;
 488       while ((len = (manifestStream.read(buf))) > 0) {
 489         zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 490       }
 491     }
 492   }
 493
 494   private boolean isManifestNeedsToGetWritten(OnboardingTypesEnum type) {
 495     return type.equals(OnboardingTypesEnum.CSAR);
 496   }
 497
 498   private void handleArtifactsFromTree(HeatStructureTree tree, FilesDataStructure structure) {
 499
 500     if (Objects.isNull(tree) || Objects.isNull(tree.getArtifacts())) {
 501       return;
 502     }
 503
 504     if (CollectionUtils.isNotEmpty(tree.getArtifacts())) {
 505       structure.getArtifacts().addAll(
 506           tree.getArtifacts()
 507               .stream()
 508               .map(Artifact::getFileName)
 509               .filter(fileName -> !structure.getArtifacts().contains(fileName))
 510               .collect(Collectors.toList()));
 511     }
 512   }
 513
 514   private void handleOtherResources(HeatStructureTree tree, Set<String> usedEnvFiles,
 515                                     FilesDataStructure structure) {
 516     Set<HeatStructureTree> others = tree.getOther();
 517     if (Objects.isNull(others)) {
 518       return;
 519     }
 520
 521     List<String> artifacts = new ArrayList<>();
 522     List<String> unassigned = new ArrayList<>();
 523     for (HeatStructureTree other : others) {
 524       if (HeatFileAnalyzer.isYamlOrEnvFile(other.getFileName())) {
 525         if (isEnvFileUsedByHeatFile(usedEnvFiles, other)) {
 526           continue;
 527         }
 528         unassigned.add(other.getFileName());
 529       } else {
 530         artifacts.add(other.getFileName());
 531       }
 532       handleArtifactsFromTree(other, structure);
 533     }
 534     structure.getArtifacts().addAll(artifacts);
 535     structure.getUnassigned().addAll(unassigned);
 536   }
 537
 538   private boolean isEnvFileUsedByHeatFile(Set<String> usedEnvFiles, HeatStructureTree other) {
 539     return HeatFileAnalyzer.isEnvFile(other.getFileName()) &&
 540         usedEnvFiles.contains(other.getFileName());
 541   }
 542
 543   private void addHeatsToFileDataStructure(HeatStructureTree tree, Set<String> usedEnvFiles,
 544                                            FilesDataStructure structure,
 545                                            Map<String, List<ErrorMessage>> uploadErrors,
 546                                            AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 547     List<Module> modules = new ArrayList<>();
 548     Set<HeatStructureTree> heatsSet = tree.getHeat();
 549     if (Objects.isNull(heatsSet)) {
 550       return;
 551     }
 552     for (HeatStructureTree heat : heatsSet) {
 553       if (isFileBaseFile(heat.getFileName())) {
 554         handleSingleHeat(structure, modules, heat, uploadErrors);
 555       } else if (isFileModuleFile(heat.getFileName(),
 556           analyzedZipHeatFiles.getModuleFiles())) {
 557         handleSingleHeat(structure, modules, heat, uploadErrors);
 558       } else {
 559         structure.getUnassigned().add(heat.getFileName());
 560         addNestedToFileDataStructure(heat, structure);
 561       }
 562       if (!Objects.isNull(heat.getEnv())) {
 563         usedEnvFiles.add(heat.getEnv() == null ? null : heat.getEnv().getFileName());
 564       }
 565     }
 566     structure.setModules(modules);
 567
 568   }
 569
 570   private boolean isFileModuleFile(String fileName, Set<String> modulesFileNames) {
 571     return modulesFileNames.contains(fileName);
 572   }
 573
 574   private boolean isFileBaseFile(String fileName) {
 575     return manifestCreator.isFileBaseFile(fileName);
 576   }
 577
 578   private void handleSingleHeat(FilesDataStructure structure, List<Module> modules,
 579                                 HeatStructureTree heat,
 580                                 Map<String, List<ErrorMessage>> uploadErrors) {
 581     Module module = new Module();
 582     module.setYaml(heat.getFileName());
 583     module.setIsBase(heat.getBase());
 584     addNestedToFileDataStructure(heat, structure);
 585     Set<HeatStructureTree> volumeSet = heat.getVolume();
 586     int inx = 0;
 587     if (Objects.nonNull(volumeSet)) {
 588       handleVolumes(module, volumeSet, structure, inx, uploadErrors);
 589     }
 590     handleEnv(module, heat, false, structure);
 591     modules.add(module);
 592   }
 593
 594   private void handleVolumes(Module module, Set<HeatStructureTree> volumeSet,
 595                              FilesDataStructure structure, int inx,
 596                              Map<String, List<ErrorMessage>> uploadErrors) {
 597     for (HeatStructureTree volume : volumeSet) {
 598       Objects.requireNonNull(volume, "volume cannot be null!");
 599       if (inx++ > 0) {
 600         ErrorsUtil.addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE,
 601             new ErrorMessage(ErrorLevel.WARNING,
 602                 Messages.MORE_THEN_ONE_VOL_FOR_HEAT.getErrorMessage()), uploadErrors);
 603         break;
 604       }
 605       handleArtifactsFromTree(volume, structure);
 606       module.setVol(volume.getFileName());
 607       handleEnv(module, volume, true, structure);
 608       addNestedToFileDataStructure(volume, structure);
 609     }
 610   }
 611
 612   private void handleEnv(Module module, HeatStructureTree tree, boolean isVolEnv,
 613                          FilesDataStructure structure) {
 614     if (Objects.nonNull(tree.getEnv())) {
 615       if (isVolEnv) {
 616         module.setVolEnv(tree.getEnv().getFileName());
 617       } else {
 618         module.setEnv(tree.getEnv().getFileName());
 619       }
 620       handleArtifactsFromTree(tree.getEnv(), structure);
 621     }
 622   }
 623
 624   private void addNestedToFileDataStructure(HeatStructureTree heat,
 625                                             FilesDataStructure structure) {
 626     Set<HeatStructureTree> nestedSet = heat.getNested();
 627     if (Objects.isNull(nestedSet)) {
 628       return;
 629     }
 630     for (HeatStructureTree nested : nestedSet) {
 631       if (structure.getNested().contains(nested.getFileName())) {
 632         continue;
 633       }
 634       structure.getNested().add(nested.getFileName());
 635       if (CollectionUtils.isNotEmpty(nested.getArtifacts())) {
 636         handleArtifactsFromTree(nested, structure);
 637       }
 638       addNestedToFileDataStructure(nested, structure);
 639     }
 640   }
 641 }