gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.vendorsoftwareproduct.services.impl.filedatastructuremodule;
  22
  23 import org.apache.commons.collections4.CollectionUtils;
  24 import org.openecomp.core.utilities.file.FileContentHandler;
  25 import org.openecomp.core.utilities.json.JsonUtil;
  26 import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
  27 import org.openecomp.sdc.common.errors.CoreException;
  28 import org.openecomp.sdc.common.errors.ErrorCategory;
  29 import org.openecomp.sdc.common.errors.ErrorCode;
  30 import org.openecomp.sdc.common.errors.Messages;
  31 import org.openecomp.sdc.common.utils.SdcCommon;
  32 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  33 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  34 import org.openecomp.sdc.heat.datatypes.manifest.FileData;
  35 import org.openecomp.sdc.heat.datatypes.manifest.ManifestContent;
  36 import org.openecomp.sdc.heat.datatypes.structure.Artifact;
  37 import org.openecomp.sdc.heat.datatypes.structure.HeatStructureTree;
  38 import org.openecomp.sdc.logging.context.impl.MdcDataDebugMessage;
  39 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDao;
  40 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDaoFactory;
  41 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.OrchestrationTemplateCandidateData;
  42 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.VspDetails;
  43 import org.openecomp.sdc.vendorsoftwareproduct.errors.utils.ErrorsUtil;
  44 import org.openecomp.sdc.vendorsoftwareproduct.services.HeatFileAnalyzer;
  45 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.CandidateService;
  46 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.ManifestCreator;
  47 import org.openecomp.sdc.vendorsoftwareproduct.services.utils.CandidateServiceValidator;
  48 import org.openecomp.sdc.vendorsoftwareproduct.types.CandidateDataEntityTo;
  49 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.AnalyzedZipHeatFiles;
  50 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.FilesDataStructure;
  51 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.Module;
  52 import org.openecomp.sdc.versioning.dao.types.Version;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 import java.io.ByteArrayInputStream;
  57 import java.io.ByteArrayOutputStream;
  58 import java.io.IOException;
  59 import java.io.InputStream;
  60 import java.nio.ByteBuffer;
  61 import java.nio.charset.StandardCharsets;
  62 import java.util.ArrayList;
  63 import java.util.HashSet;
  64 import java.util.List;
  65 import java.util.Map;
  66 import java.util.Objects;
  67 import java.util.Optional;
  68 import java.util.Set;
  69 import java.util.stream.Collectors;
  70 import java.util.zip.ZipEntry;
  71 import java.util.zip.ZipInputStream;
  72 import java.util.zip.ZipOutputStream;
  73
  74 public class CandidateServiceImpl implements CandidateService {
  75   protected static final Logger logger = LoggerFactory.getLogger(CandidateServiceImpl.class);
  76   private static MdcDataDebugMessage mdcDataDebugMessage = new MdcDataDebugMessage();
  77
  78   private CandidateServiceValidator candidateServiceValidator = new CandidateServiceValidator();
  79   private ManifestCreator manifestCreator;
  80   private OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDao;
  81
  82   public CandidateServiceImpl(ManifestCreator manifestCreator,
  83                               OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDao) {
  84     this.manifestCreator = manifestCreator;
  85     this.orchestrationTemplateCandidateDao = orchestrationTemplateCandidateDao;
  86
  87   }
  88
  89   public CandidateServiceImpl() {
  90   }
  91
  92   @Override
  93   public Optional<ErrorMessage> validateNonEmptyFileToUpload(InputStream fileToUpload) {
  94
  95
  96     mdcDataDebugMessage.debugEntryMessage(null);
  97
  98     if (Objects.isNull(fileToUpload)) {
  99       return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 100           Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 101     } else {
 102       try {
 103         int available = fileToUpload.available();
 104         if (available == 0) {
 105           mdcDataDebugMessage.debugExitMessage(null);
 106           return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 107               Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 108         }
 109       } catch (IOException e) {
 110         logger.debug(e.getMessage(), e);
 111         mdcDataDebugMessage.debugExitMessage(null);
 112         return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 113             Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 114       }
 115     }
 116
 117     mdcDataDebugMessage.debugExitMessage(null);
 118     return Optional.empty();
 119   }
 120
 121   @Override
 122   public Optional<ErrorMessage> validateRawZipData(byte[] uploadedFileData) {
 123     if (Objects.isNull(uploadedFileData)) {
 124       return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 125           Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 126     }
 127     return Optional.empty();
 128   }
 129
 130   private String heatStructureTreeToFileDataStructure(HeatStructureTree tree,
 131                                                       FileContentHandler zipContentMap,
 132                                                       Map<String, List<ErrorMessage>> uploadErrors,
 133                                                       AnalyzedZipHeatFiles analyzedZipHeatFiles)
 134       throws Exception {
 135     FilesDataStructure structure = new FilesDataStructure();
 136     Set<String> usedEnvFiles = new HashSet<>();
 137     addHeatsToFileDataStructure(tree, usedEnvFiles, structure, uploadErrors,
 138         analyzedZipHeatFiles);
 139     handleOtherResources(tree, usedEnvFiles, structure);
 140     FilesDataStructure fileDataStructureFromManifest =
 141         createFileDataStructureFromManifest(zipContentMap.getFileContent
 142             (SdcCommon.MANIFEST_NAME));
 143     List<String> structureArtifacts = structure.getArtifacts();
 144     structureArtifacts.addAll(fileDataStructureFromManifest.getArtifacts().stream().filter
 145         (artifact -> isNotStrctureArtifact(structureArtifacts, artifact))
 146         .collect((Collectors.toList())));
 147     handleArtifactsFromTree(tree, structure);
 148
 149     return JsonUtil.object2Json(structure);
 150   }
 151
 152   private boolean isNotStrctureArtifact(List<String> structureArtifacts, String artifact) {
 153     return !structureArtifacts.contains(artifact);
 154   }
 155
 156   @Override
 157   public OrchestrationTemplateCandidateData createCandidateDataEntity(
 158       CandidateDataEntityTo candidateDataEntityTo, InputStream zipFileManifest,
 159       AnalyzedZipHeatFiles analyzedZipHeatFiles) throws Exception {
 160
 161
 162     mdcDataDebugMessage.debugEntryMessage(null);
 163
 164     FileContentHandler zipContentMap = candidateDataEntityTo.getContentMap();
 165     FilesDataStructure filesDataStructure;
 166     String dataStructureJson;
 167
 168     if (zipFileManifest != null) {
 169       // create data structure from manifest
 170       filesDataStructure = createFileDataStructureFromManifest(zipFileManifest);
 171       Set<String> zipFileList = zipContentMap.getFileList();
 172       balanceManifestFilesWithZipFiles(filesDataStructure,
 173           zipContentMap, analyzedZipHeatFiles);
 174       Set<String> filesDataStructureFiles = getFlatFileNames(filesDataStructure);
 175       filesDataStructure.getUnassigned().addAll(zipFileList.stream()
 176           .filter(fileName -> (!filesDataStructureFiles.contains(fileName) &&
 177               !filesDataStructure.getNested().contains(fileName) &&
 178               !fileName.equals(SdcCommon.MANIFEST_NAME)))
 179           .collect(Collectors.toList()));
 180       dataStructureJson = JsonUtil.object2Json(filesDataStructure);
 181     } else {
 182       // create data structure from based on naming convention
 183       dataStructureJson =
 184           heatStructureTreeToFileDataStructure(candidateDataEntityTo.getTree(), zipContentMap,
 185               candidateDataEntityTo.getErrors(), analyzedZipHeatFiles);
 186     }
 187
 188     OrchestrationTemplateCandidateData candidateData = new OrchestrationTemplateCandidateData();
 189     candidateData.setContentData(ByteBuffer.wrap(candidateDataEntityTo.getUploadedFileData()));
 190     candidateData.setFilesDataStructure(dataStructureJson);
 191
 192     mdcDataDebugMessage.debugExitMessage(null);
 193     return candidateData;
 194   }
 195
 196   private void balanceManifestFilesWithZipFiles(
 197       FilesDataStructure filesDataStructure,
 198       FileContentHandler fileContentHandler, AnalyzedZipHeatFiles analyzedZipHeatFiles)
 199       throws Exception {
 200     Set<String> zipFileList = fileContentHandler.getFileList();
 201     filesDataStructure.getNested().addAll(analyzedZipHeatFiles.getNestedFiles());
 202     List<Module> modules = filesDataStructure.getModules();
 203     if (CollectionUtils.isEmpty(modules)) {
 204       return;
 205     }
 206
 207     for (int i = 0; i < modules.size(); i++) {
 208       Module module = modules.get(i);
 209       if (!isFileExistInZipContains(zipFileList, module.getYaml())) {
 210         addFileToUnassigned(filesDataStructure, zipFileList, module.getEnv());
 211         addFileToUnassigned(filesDataStructure, zipFileList, module.getVol());
 212         addFileToUnassigned(filesDataStructure, zipFileList, module.getVolEnv());
 213         modules.remove(i--);
 214       } else if (Objects.nonNull(module.getVol()) && !zipFileList.contains(module.getVol())) {
 215         module.setVol(null);
 216         CollectionUtils
 217             .addIgnoreNull(filesDataStructure.getUnassigned(), module.getVolEnv());
 218       } else {
 219         if (filesDataStructure.getNested().contains(module.getYaml())) {
 220           moveModuleFileToNested(filesDataStructure, i--, module);
 221         }
 222       }
 223     }
 224   }
 225
 226   private void addFileToUnassigned(FilesDataStructure filesDataStructure, Set<String> zipFileList,
 227                                    String fileName) {
 228     if (isFileExistInZipContains(zipFileList, fileName)) {
 229       filesDataStructure.getUnassigned().add(fileName);
 230     }
 231   }
 232
 233   private boolean isFileExistInZipContains(Set<String> zipFileList, String fileName) {
 234     return Objects.nonNull(fileName) && zipFileList.contains(fileName);
 235   }
 236
 237   private void moveModuleFileToNested(FilesDataStructure filesDataStructure, int i,
 238                                       Module module) {
 239     if (!filesDataStructure.getNested().contains(module.getYaml())) {
 240       filesDataStructure.getNested().add(module.getYaml());
 241     }
 242     if (Objects.nonNull(module.getEnv())) {
 243       filesDataStructure.getNested().add(module.getEnv());
 244     }
 245     if (Objects.nonNull(module.getVol())) {
 246       filesDataStructure.getNested().add(module.getVol());
 247     }
 248     if (Objects.nonNull(module.getVolEnv())) {
 249       filesDataStructure.getNested().add(module.getVolEnv());
 250     }
 251     filesDataStructure.getModules().remove(i);
 252   }
 253
 254   private Set<String> getFlatFileNames(FilesDataStructure filesDataStructure) {
 255     Set<String> fileNames = new HashSet<>();
 256     if (!CollectionUtils.isEmpty(filesDataStructure.getModules())) {
 257       for (Module module : filesDataStructure.getModules()) {
 258         CollectionUtils.addIgnoreNull(fileNames, module.getEnv());
 259         CollectionUtils.addIgnoreNull(fileNames, module.getVol());
 260         CollectionUtils.addIgnoreNull(fileNames, module.getVolEnv());
 261         CollectionUtils.addIgnoreNull(fileNames, module.getYaml());
 262       }
 263     }
 264     fileNames.addAll(filesDataStructure.getArtifacts().stream().collect(Collectors.toSet()));
 265     fileNames.addAll(filesDataStructure.getNested().stream().collect(Collectors.toSet()));
 266     fileNames.addAll(filesDataStructure.getUnassigned().stream().collect(Collectors.toSet()));
 267
 268     return fileNames;
 269   }
 270
 271   private FilesDataStructure createFileDataStructureFromManifest(InputStream isManifestContent) {
 272
 273
 274     mdcDataDebugMessage.debugEntryMessage(null);
 275
 276     ManifestContent manifestContent =
 277         JsonUtil.json2Object(isManifestContent, ManifestContent.class);
 278     FilesDataStructure structure = new FilesDataStructure();
 279     for (FileData fileData : manifestContent.getData()) {
 280       if (Objects.nonNull(fileData.getType()) &&
 281           fileData.getType().equals(FileData.Type.HEAT)) {
 282         Module module = new Module();
 283         module.setYaml(fileData.getFile());
 284         module.setIsBase(fileData.getBase());
 285         addHeatDependenciesToModule(module, fileData.getData());
 286         structure.getModules().add(module);
 287       } else if (HeatFileAnalyzer.isYamlOrEnvFile(fileData.getFile()) &&
 288           !FileData.Type.isArtifact(fileData.getType())) {
 289         structure.getUnassigned().add(fileData.getFile());
 290       } else {
 291         structure.getArtifacts().add(fileData.getFile());
 292       }
 293     }
 294
 295     mdcDataDebugMessage.debugExitMessage(null);
 296     return structure;
 297   }
 298
 299   private void addHeatDependenciesToModule(Module module, List<FileData> data) {
 300     if (CollectionUtils.isEmpty(data)) {
 301       return;
 302     }
 303
 304     for (FileData fileData : data) {
 305       if (fileData.getType().equals(FileData.Type.HEAT_ENV)) {
 306         module.setEnv(fileData.getFile());
 307       } else if (fileData.getType().equals(FileData.Type.HEAT_VOL))// must be volume
 308       {
 309         module.setVol(fileData.getFile());
 310         if (!CollectionUtils.isEmpty(fileData.getData())) {
 311           FileData volEnv = fileData.getData().get(0);
 312           if (volEnv.getType().equals(FileData.Type.HEAT_ENV)) {
 313             module.setVolEnv(volEnv.getFile());
 314           } else {
 315             throw new CoreException((new ErrorCode.ErrorCodeBuilder())
 316                 .withMessage(Messages.ILLEGAL_MANIFEST.getErrorMessage())
 317                 .withId(Messages.ILLEGAL_MANIFEST.getErrorMessage())
 318                 .withCategory(ErrorCategory.APPLICATION).build());
 319           }
 320         }
 321       } else {
 322         throw new CoreException((new ErrorCode.ErrorCodeBuilder())
 323             .withMessage(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
 324             .withId(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
 325             .withCategory(ErrorCategory.APPLICATION).build());
 326       }
 327     }
 328   }
 329
 330   @Override
 331   public void updateCandidateUploadData(String vspId, Version version,
 332                                         OrchestrationTemplateCandidateData uploadData) {
 333     mdcDataDebugMessage.debugEntryMessage(null);
 334
 335     orchestrationTemplateCandidateDao.update(vspId, version, uploadData);
 336
 337     mdcDataDebugMessage.debugExitMessage(null);
 338   }
 339
 340   @Override
 341   public Optional<FilesDataStructure> getOrchestrationTemplateCandidateFileDataStructure(
 342       String vspId, Version version) {
 343
 344     mdcDataDebugMessage.debugEntryMessage("VSP Id", vspId);
 345
 346     Optional<String> jsonFileDataStructure =
 347         orchestrationTemplateCandidateDao.getStructure(vspId, version);
 348
 349     if (jsonFileDataStructure.isPresent() && JsonUtil.isValidJson(jsonFileDataStructure.get())) {
 350       mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 351       return Optional
 352           .of(JsonUtil.json2Object(jsonFileDataStructure.get(), FilesDataStructure.class));
 353     } else {
 354       mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 355       return Optional.empty();
 356     }
 357   }
 358
 359   @Override
 360   public void updateOrchestrationTemplateCandidateFileDataStructure(String vspId, Version version,
 361                                                                     FilesDataStructure fileDataStructure) {
 362     OrchestrationTemplateCandidateDaoFactory.getInstance().createInterface()
 363         .updateStructure(vspId, version, fileDataStructure);
 364   }
 365
 366   @Override
 367   public OrchestrationTemplateCandidateData getOrchestrationTemplateCandidate(String vspId,
 368                                                                               Version version) {
 369     mdcDataDebugMessage.debugEntryMessage("VSP Id", vspId);
 370     mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 371
 372     return orchestrationTemplateCandidateDao.get(vspId, version);
 373   }
 374
 375   @Override
 376   public OrchestrationTemplateCandidateData getOrchestrationTemplateCandidateInfo(String vspId,
 377                                                                                   Version version) {
 378     mdcDataDebugMessage.debugEntryMessage("VSP Id", vspId);
 379     mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 380
 381     return orchestrationTemplateCandidateDao.getInfo(vspId, version);
 382   }
 383
 384   @Override
 385   public String createManifest(VspDetails vspDetails, FilesDataStructure structure) {
 386
 387     mdcDataDebugMessage.debugEntryMessage("VSP Id", vspDetails.getId());
 388
 389     Optional<ManifestContent> manifest = manifestCreator.createManifest(vspDetails, structure);
 390     if (!manifest.isPresent()) {
 391       throw new RuntimeException(Messages.CREATE_MANIFEST_FROM_ZIP.getErrorMessage());
 392     }
 393
 394     mdcDataDebugMessage.debugExitMessage("VSP Id", vspDetails.getId());
 395     return JsonUtil.object2Json(manifest.get());
 396   }
 397
 398   @Override
 399   public Optional<ManifestContent> createManifest(VspDetails vspDetails,
 400                                                   FileContentHandler fileContentHandler,
 401                                                   AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 402
 403
 404     mdcDataDebugMessage.debugEntryMessage("VSP Id", vspDetails.getId());
 405
 406     mdcDataDebugMessage.debugExitMessage("VSP Id", vspDetails.getId());
 407     return manifestCreator.createManifest(vspDetails, fileContentHandler, analyzedZipHeatFiles);
 408   }
 409
 410   @Override
 411   public Optional<ByteArrayInputStream> fetchZipFileByteArrayInputStream(String vspId,
 412                                                                          OrchestrationTemplateCandidateData candidateDataEntity,
 413                                                                          String manifest,
 414                                                                          OnboardingTypesEnum type,
 415                                                                          Map<String, List<ErrorMessage>> uploadErrors) {
 416     byte[] file;
 417     ByteArrayInputStream byteArrayInputStream = null;
 418     try {
 419       file = replaceManifestInZip(candidateDataEntity.getContentData(), manifest, vspId, type);
 420       byteArrayInputStream = new ByteArrayInputStream(
 421           Objects.isNull(file) ? candidateDataEntity.getContentData().array()
 422               : file);
 423     } catch (IOException e) {
 424       ErrorMessage errorMessage =
 425           new ErrorMessage(ErrorLevel.ERROR,
 426               Messages.CANDIDATE_PROCESS_FAILED.getErrorMessage());
 427       logger.error(errorMessage.getMessage(), e);
 428       ErrorsUtil
 429           .addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE, errorMessage, uploadErrors);
 430     }
 431     return Optional.ofNullable(byteArrayInputStream);
 432   }
 433
 434   @Override
 435   public byte[] replaceManifestInZip(ByteBuffer contentData, String manifest, String vspId,
 436                                      OnboardingTypesEnum type)
 437       throws IOException {
 438     ByteArrayOutputStream baos = new ByteArrayOutputStream();
 439
 440     try (final ZipOutputStream zos = new ZipOutputStream(baos);
 441          ZipInputStream zipStream = new ZipInputStream(
 442              new ByteArrayInputStream(contentData.array()))) {
 443       ZipEntry zipEntry;
 444       boolean manifestWritten = false;
 445       while ((zipEntry = zipStream.getNextEntry()) != null) {
 446         if (!zipEntry.getName().equalsIgnoreCase(SdcCommon.MANIFEST_NAME)) {
 447           ZipEntry loc_ze = new ZipEntry(zipEntry.getName());
 448           zos.putNextEntry(loc_ze);
 449           byte[] buf = new byte[1024];
 450           int len;
 451           while ((len = zipStream.read(buf)) > 0) {
 452             zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 453           }
 454         } else {
 455           manifestWritten = true;
 456           writeManifest(manifest, type, zos);
 457         }
 458         zos.closeEntry();
 459       }
 460       if (!manifestWritten) {
 461         writeManifest(manifest, type, zos);
 462         zos.closeEntry();
 463       }
 464     }
 465     return baos.toByteArray();
 466   }
 467
 468   @Override
 469   public Optional<List<ErrorMessage>> validateFileDataStructure(
 470       FilesDataStructure filesDataStructure) {
 471     return candidateServiceValidator.validateFileDataStructure(filesDataStructure);
 472   }
 473
 474   private void writeManifest(String manifest,
 475                              OnboardingTypesEnum type,
 476                              ZipOutputStream zos) throws IOException {
 477
 478     if (isManifestNeedsToGetWritten(type)) {
 479       return;
 480     }
 481
 482     zos.putNextEntry(new ZipEntry(SdcCommon.MANIFEST_NAME));
 483     try (InputStream manifestStream = new ByteArrayInputStream(
 484         manifest.getBytes(StandardCharsets.UTF_8))) {
 485       byte[] buf = new byte[1024];
 486       int len;
 487       while ((len = (manifestStream.read(buf))) > 0) {
 488         zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 489       }
 490     }
 491   }
 492
 493   private boolean isManifestNeedsToGetWritten(OnboardingTypesEnum type) {
 494     return type.equals(OnboardingTypesEnum.CSAR);
 495   }
 496
 497   private void handleArtifactsFromTree(HeatStructureTree tree, FilesDataStructure structure) {
 498
 499     if (Objects.isNull(tree) || Objects.isNull(tree.getArtifacts())) {
 500       return;
 501     }
 502
 503     if (CollectionUtils.isNotEmpty(tree.getArtifacts())) {
 504       structure.getArtifacts().addAll(
 505           tree.getArtifacts()
 506               .stream()
 507               .map(Artifact::getFileName)
 508               .filter(fileName -> !structure.getArtifacts().contains(fileName))
 509               .collect(Collectors.toList()));
 510     }
 511   }
 512
 513   private void handleOtherResources(HeatStructureTree tree, Set<String> usedEnvFiles,
 514                                     FilesDataStructure structure) {
 515     Set<HeatStructureTree> others = tree.getOther();
 516     if (Objects.isNull(others)) {
 517       return;
 518     }
 519
 520     List<String> artifacts = new ArrayList<>();
 521     List<String> unassigned = new ArrayList<>();
 522     for (HeatStructureTree other : others) {
 523       if (HeatFileAnalyzer.isYamlOrEnvFile(other.getFileName())) {
 524         if (isEnvFileUsedByHeatFile(usedEnvFiles, other)) {
 525           continue;
 526         }
 527         unassigned.add(other.getFileName());
 528       } else {
 529         artifacts.add(other.getFileName());
 530       }
 531       handleArtifactsFromTree(other, structure);
 532     }
 533     structure.getArtifacts().addAll(artifacts);
 534     structure.getUnassigned().addAll(unassigned);
 535   }
 536
 537   private boolean isEnvFileUsedByHeatFile(Set<String> usedEnvFiles, HeatStructureTree other) {
 538     if (HeatFileAnalyzer.isEnvFile(other.getFileName())) {
 539       if (usedEnvFiles.contains(other.getFileName())) {
 540         return true;
 541       }
 542     }
 543     return false;
 544   }
 545
 546   private void addHeatsToFileDataStructure(HeatStructureTree tree, Set<String> usedEnvFiles,
 547                                            FilesDataStructure structure,
 548                                            Map<String, List<ErrorMessage>> uploadErrors,
 549                                            AnalyzedZipHeatFiles analyzedZipHeatFiles)
 550       throws Exception {
 551     List<Module> modules = new ArrayList<>();
 552     Set<HeatStructureTree> heatsSet = tree.getHeat();
 553     if (Objects.isNull(heatsSet)) {
 554       return;
 555     }
 556     for (HeatStructureTree heat : heatsSet) {
 557       if (isFileBaseFile(heat.getFileName())) {
 558         handleSingleHeat(structure, modules, heat, uploadErrors);
 559       } else if (isFileModuleFile(heat.getFileName(),
 560           analyzedZipHeatFiles.getModuleFiles())) {
 561         handleSingleHeat(structure, modules, heat, uploadErrors);
 562       } else {
 563         structure.getUnassigned().add(heat.getFileName());
 564         addNestedToFileDataStructure(heat, structure);
 565       }
 566       if (!Objects.isNull(heat.getEnv())) {
 567         usedEnvFiles.add(heat.getEnv() == null ? null : heat.getEnv().getFileName());
 568       }
 569     }
 570     structure.setModules(modules);
 571
 572   }
 573
 574   private boolean isFileModuleFile(String fileName, Set<String> modulesFileNames) {
 575     return modulesFileNames.contains(fileName);
 576   }
 577
 578   private boolean isFileBaseFile(String fileName) {
 579     return manifestCreator.isFileBaseFile(fileName);
 580   }
 581
 582   private void handleSingleHeat(FilesDataStructure structure, List<Module> modules,
 583                                 HeatStructureTree heat,
 584                                 Map<String, List<ErrorMessage>> uploadErrors) {
 585
 586
 587     mdcDataDebugMessage.debugEntryMessage(null);
 588
 589     Module module = new Module();
 590     module.setYaml(heat.getFileName());
 591     module.setIsBase(heat.getBase());
 592     addNestedToFileDataStructure(heat, structure);
 593     Set<HeatStructureTree> volumeSet = heat.getVolume();
 594     int inx = 0;
 595     if (Objects.nonNull(volumeSet)) {
 596       handleVolumes(module, volumeSet, structure, inx, uploadErrors);
 597     }
 598     handleEnv(module, heat, false, structure);
 599     modules.add(module);
 600
 601     mdcDataDebugMessage.debugExitMessage(null);
 602   }
 603
 604   private void handleVolumes(Module module, Set<HeatStructureTree> volumeSet,
 605                              FilesDataStructure structure, int inx,
 606                              Map<String, List<ErrorMessage>> uploadErrors) {
 607
 608
 609     mdcDataDebugMessage.debugEntryMessage(null);
 610
 611     for (HeatStructureTree volume : volumeSet) {
 612       if (inx++ > 0) {
 613         ErrorsUtil.addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE,
 614             new ErrorMessage(ErrorLevel.WARNING,
 615                 Messages.MORE_THEN_ONE_VOL_FOR_HEAT.getErrorMessage()), uploadErrors);
 616         break;
 617       }
 618       handleArtifactsFromTree(volume, structure);
 619       module.setVol(volume.getFileName());
 620       handleEnv(module, volume, true, structure);
 621       addNestedToFileDataStructure(volume, structure);
 622     }
 623
 624     mdcDataDebugMessage.debugExitMessage(null);
 625   }
 626
 627   private void handleEnv(Module module, HeatStructureTree tree, boolean isVolEnv,
 628                          FilesDataStructure structure) {
 629
 630
 631     mdcDataDebugMessage.debugEntryMessage(null);
 632
 633     if (Objects.nonNull(tree.getEnv())) {
 634       if (isVolEnv) {
 635         module.setVolEnv(tree.getEnv().getFileName());
 636       } else {
 637         module.setEnv(tree.getEnv().getFileName());
 638       }
 639       handleArtifactsFromTree(tree.getEnv(), structure);
 640     }
 641
 642     mdcDataDebugMessage.debugExitMessage(null);
 643   }
 644
 645   private void addNestedToFileDataStructure(HeatStructureTree heat,
 646                                             FilesDataStructure structure) {
 647     Set<HeatStructureTree> nestedSet = heat.getNested();
 648     if (Objects.isNull(nestedSet)) {
 649       return;
 650     }
 651     for (HeatStructureTree nested : nestedSet) {
 652       if (structure.getNested().contains(nested.getFileName())) {
 653         continue;
 654       }
 655       structure.getNested().add(nested.getFileName());
 656       if (CollectionUtils.isNotEmpty(nested.getArtifacts())) {
 657         handleArtifactsFromTree(nested, structure);
 658       }
 659       addNestedToFileDataStructure(nested, structure);
 660     }
 661   }
 662 }