gerrit.onap Code Review - sdc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.vendorsoftwareproduct.services.impl.filedatastructuremodule;
  22
  23 import org.apache.commons.collections4.CollectionUtils;
  24 import org.openecomp.core.utilities.file.FileContentHandler;
  25 import org.openecomp.core.utilities.json.JsonUtil;
  26 import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
  27 import org.openecomp.sdc.common.errors.CoreException;
  28 import org.openecomp.sdc.common.errors.ErrorCategory;
  29 import org.openecomp.sdc.common.errors.ErrorCode;
  30 import org.openecomp.sdc.common.errors.Messages;
  31 import org.openecomp.sdc.common.utils.SdcCommon;
  32 import org.openecomp.sdc.datatypes.error.ErrorLevel;
  33 import org.openecomp.sdc.datatypes.error.ErrorMessage;
  34 import org.openecomp.sdc.heat.datatypes.manifest.FileData;
  35 import org.openecomp.sdc.heat.datatypes.manifest.ManifestContent;
  36 import org.openecomp.sdc.heat.datatypes.structure.Artifact;
  37 import org.openecomp.sdc.heat.datatypes.structure.HeatStructureTree;
  38 import org.openecomp.sdc.logging.context.impl.MdcDataDebugMessage;
  39 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDao;
  40 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDaoFactory;
  41 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.OrchestrationTemplateCandidateData;
  42 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.VspDetails;
  43 import org.openecomp.sdc.vendorsoftwareproduct.errors.utils.ErrorsUtil;
  44 import org.openecomp.sdc.vendorsoftwareproduct.services.HeatFileAnalyzer;
  45 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.CandidateService;
  46 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.ManifestCreator;
  47 import org.openecomp.sdc.vendorsoftwareproduct.services.utils.CandidateServiceValidator;
  48 import org.openecomp.sdc.vendorsoftwareproduct.types.CandidateDataEntityTo;
  49 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.AnalyzedZipHeatFiles;
  50 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.FilesDataStructure;
  51 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.Module;
  52 import org.openecomp.sdc.versioning.dao.types.Version;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 import java.io.ByteArrayInputStream;
  57 import java.io.ByteArrayOutputStream;
  58 import java.io.IOException;
  59 import java.io.InputStream;
  60 import java.nio.ByteBuffer;
  61 import java.nio.charset.StandardCharsets;
  62 import java.util.ArrayList;
  63 import java.util.HashSet;
  64 import java.util.List;
  65 import java.util.Map;
  66 import java.util.Objects;
  67 import java.util.Optional;
  68 import java.util.Set;
  69 import java.util.stream.Collectors;
  70 import java.util.zip.ZipEntry;
  71 import java.util.zip.ZipInputStream;
  72 import java.util.zip.ZipOutputStream;
  73
  74 public class CandidateServiceImpl implements CandidateService {
  75     protected static final Logger logger = LoggerFactory.getLogger(CandidateServiceImpl.class);
  76     private static MdcDataDebugMessage mdcDataDebugMessage = new MdcDataDebugMessage();
  77
  78     private CandidateServiceValidator candidateServiceValidator = new CandidateServiceValidator();
  79     private ManifestCreator manifestCreator;
  80     private OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDataDao;
  81
  82     public CandidateServiceImpl(ManifestCreator manifestCreator,
  83                                 OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDataDao) {
  84         this.manifestCreator = manifestCreator;
  85         this.orchestrationTemplateCandidateDataDao = orchestrationTemplateCandidateDataDao;
  86
  87     }
  88
  89     public CandidateServiceImpl() {
  90     }
  91
  92     @Override
  93     public Optional<ErrorMessage> validateNonEmptyFileToUpload(InputStream fileToUpload) {
  94
  95
  96         mdcDataDebugMessage.debugEntryMessage(null);
  97
  98         if (Objects.isNull(fileToUpload)) {
  99             return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 100                 Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 101         } else {
 102             try {
 103                 int available = fileToUpload.available();
 104                 if (available == 0) {
 105                     mdcDataDebugMessage.debugExitMessage(null);
 106                     return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 107                         Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 108                 }
 109             } catch (IOException e) {
 110                 logger.debug(e.getMessage(), e);
 111                 mdcDataDebugMessage.debugExitMessage(null);
 112                 return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 113                     Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 114             }
 115         }
 116
 117         mdcDataDebugMessage.debugExitMessage(null);
 118         return Optional.empty();
 119     }
 120
 121     @Override
 122     public Optional<ErrorMessage> validateRawZipData(byte[] uploadedFileData) {
 123         if (Objects.isNull(uploadedFileData)) {
 124             return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
 125                 Messages.NO_ZIP_FILE_WAS_UPLOADED_OR_ZIP_NOT_EXIST.getErrorMessage()));
 126         }
 127         return Optional.empty();
 128     }
 129
 130     private String heatStructureTreeToFileDataStructure(HeatStructureTree tree,
 131                                                         FileContentHandler zipContentMap,
 132                                                         Map<String, List<ErrorMessage>> uploadErrors,
 133                                                         AnalyzedZipHeatFiles analyzedZipHeatFiles)
 134         throws Exception {
 135         FilesDataStructure structure = new FilesDataStructure();
 136         Set<String> usedEnvFiles = new HashSet<>();
 137         addHeatsToFileDataStructure(tree, usedEnvFiles, structure, uploadErrors,
 138             analyzedZipHeatFiles);
 139         handleOtherResources(tree, usedEnvFiles, structure);
 140         FilesDataStructure fileDataStructureFromManifest =
 141             createFileDataStructureFromManifest(zipContentMap.getFileContent
 142                 (SdcCommon.MANIFEST_NAME));
 143         List<String> structureArtifacts = structure.getArtifacts();
 144         structureArtifacts.addAll(fileDataStructureFromManifest.getArtifacts().stream().filter
 145             (artifact -> isNotStrctureArtifact(structureArtifacts, artifact)).collect((Collectors.toList())));
 146         handleArtifactsFromTree(tree, structure);
 147
 148         return JsonUtil.object2Json(structure);
 149     }
 150
 151     private boolean isNotStrctureArtifact(List<String> structureArtifacts, String artifact) {
 152         return !structureArtifacts.contains(artifact);
 153     }
 154
 155     @Override
 156     public OrchestrationTemplateCandidateData createCandidateDataEntity(
 157         CandidateDataEntityTo candidateDataEntityTo, InputStream zipFileManifest,
 158         AnalyzedZipHeatFiles analyzedZipHeatFiles) throws Exception {
 159
 160
 161         mdcDataDebugMessage.debugEntryMessage(null);
 162
 163         FileContentHandler zipContentMap = candidateDataEntityTo.getContentMap();
 164         FilesDataStructure filesDataStructure;
 165         String dataStructureJson;
 166
 167         if (zipFileManifest != null) {
 168             // create data structure from manifest
 169             filesDataStructure = createFileDataStructureFromManifest(zipFileManifest);
 170             Set<String> zipFileList = zipContentMap.getFileList();
 171             balanceManifestFilesWithZipFiles(filesDataStructure,
 172                 zipContentMap, analyzedZipHeatFiles);
 173             Set<String> filesDataStructureFiles = getFlatFileNames(filesDataStructure);
 174             filesDataStructure.getUnassigned().addAll(zipFileList.stream()
 175                 .filter(fileName -> (!filesDataStructureFiles.contains(fileName) &&
 176                     !filesDataStructure.getNested().contains(fileName) &&
 177                     !fileName.equals(SdcCommon.MANIFEST_NAME)))
 178                 .collect(Collectors.toList()));
 179             dataStructureJson = JsonUtil.object2Json(filesDataStructure);
 180         } else {
 181             // create data structure from based on naming convention
 182             dataStructureJson =
 183                 heatStructureTreeToFileDataStructure(candidateDataEntityTo.getTree(), zipContentMap,
 184                     candidateDataEntityTo.getErrors(), analyzedZipHeatFiles);
 185         }
 186
 187         mdcDataDebugMessage.debugExitMessage(null);
 188         return new OrchestrationTemplateCandidateData(
 189             ByteBuffer.wrap(candidateDataEntityTo.getUploadedFileData()), dataStructureJson);
 190     }
 191
 192     private void balanceManifestFilesWithZipFiles(
 193         FilesDataStructure filesDataStructure,
 194         FileContentHandler fileContentHandler, AnalyzedZipHeatFiles analyzedZipHeatFiles)
 195         throws Exception {
 196         Set<String> zipFileList = fileContentHandler.getFileList();
 197         filesDataStructure.getNested().addAll(analyzedZipHeatFiles.getNestedFiles());
 198         List<Module> modules = filesDataStructure.getModules();
 199         if (CollectionUtils.isEmpty(modules)) {
 200             return;
 201         }
 202
 203         for (int i = 0; i < modules.size(); i++) {
 204             Module module = modules.get(i);
 205             if (!isFileExistInZipContains(zipFileList, module.getYaml())) {
 206                 addFileToUnassigned(filesDataStructure, zipFileList, module.getEnv());
 207                 addFileToUnassigned(filesDataStructure, zipFileList, module.getVol());
 208                 addFileToUnassigned(filesDataStructure, zipFileList, module.getVolEnv());
 209                 modules.remove(i--);
 210             } else if (Objects.nonNull(module.getVol()) && !zipFileList.contains(module.getVol())) {
 211                 module.setVol(null);
 212                 CollectionUtils
 213                     .addIgnoreNull(filesDataStructure.getUnassigned(), module.getVolEnv());
 214             } else {
 215                 if (filesDataStructure.getNested().contains(module.getYaml())) {
 216                     moveModuleFileToNested(filesDataStructure, i--, module);
 217                 }
 218             }
 219         }
 220     }
 221
 222     private void addFileToUnassigned(FilesDataStructure filesDataStructure, Set<String> zipFileList,
 223                                      String fileName) {
 224         if (isFileExistInZipContains(zipFileList, fileName)) {
 225             filesDataStructure.getUnassigned().add(fileName);
 226         }
 227     }
 228
 229     private boolean isFileExistInZipContains(Set<String> zipFileList, String fileName) {
 230         return Objects.nonNull(fileName) && zipFileList.contains(fileName);
 231     }
 232
 233     private void moveModuleFileToNested(FilesDataStructure filesDataStructure, int i,
 234                                         Module module) {
 235         if (!filesDataStructure.getNested().contains(module.getYaml())) {
 236             filesDataStructure.getNested().add(module.getYaml());
 237         }
 238         if (Objects.nonNull(module.getEnv())) {
 239             filesDataStructure.getNested().add(module.getEnv());
 240         }
 241         if (Objects.nonNull(module.getVol())) {
 242             filesDataStructure.getNested().add(module.getVol());
 243         }
 244         if (Objects.nonNull(module.getVolEnv())) {
 245             filesDataStructure.getNested().add(module.getVolEnv());
 246         }
 247         filesDataStructure.getModules().remove(i);
 248     }
 249
 250     private Set<String> getFlatFileNames(FilesDataStructure filesDataStructure) {
 251         Set<String> fileNames = new HashSet<>();
 252         if (!CollectionUtils.isEmpty(filesDataStructure.getModules())) {
 253             for (Module module : filesDataStructure.getModules()) {
 254                 CollectionUtils.addIgnoreNull(fileNames, module.getEnv());
 255                 CollectionUtils.addIgnoreNull(fileNames, module.getVol());
 256                 CollectionUtils.addIgnoreNull(fileNames, module.getVolEnv());
 257                 CollectionUtils.addIgnoreNull(fileNames, module.getYaml());
 258             }
 259         }
 260         fileNames.addAll(filesDataStructure.getArtifacts().stream().collect(Collectors.toSet()));
 261         fileNames.addAll(filesDataStructure.getNested().stream().collect(Collectors.toSet()));
 262         fileNames.addAll(filesDataStructure.getUnassigned().stream().collect(Collectors.toSet()));
 263
 264         return fileNames;
 265     }
 266
 267     private FilesDataStructure createFileDataStructureFromManifest(InputStream isManifestContent) {
 268
 269
 270         mdcDataDebugMessage.debugEntryMessage(null);
 271
 272         ManifestContent manifestContent =
 273             JsonUtil.json2Object(isManifestContent, ManifestContent.class);
 274         FilesDataStructure structure = new FilesDataStructure();
 275         for (FileData fileData : manifestContent.getData()) {
 276             if (Objects.nonNull(fileData.getType()) &&
 277                 fileData.getType().equals(FileData.Type.HEAT)) {
 278                 Module module = new Module();
 279                 module.setYaml(fileData.getFile());
 280                 module.setIsBase(fileData.getBase());
 281                 addHeatDependenciesToModule(module, fileData.getData());
 282                 structure.getModules().add(module);
 283             } else if (HeatFileAnalyzer.isYamlOrEnvFile(fileData.getFile()) &&
 284                 !FileData.Type.isArtifact(fileData.getType())) {
 285                 structure.getUnassigned().add(fileData.getFile());
 286             } else {
 287                 structure.getArtifacts().add(fileData.getFile());
 288             }
 289         }
 290
 291         mdcDataDebugMessage.debugExitMessage(null);
 292         return structure;
 293     }
 294
 295     private void addHeatDependenciesToModule(Module module, List<FileData> data) {
 296         if (CollectionUtils.isEmpty(data)) {
 297             return;
 298         }
 299
 300         for (FileData fileData : data) {
 301             if (fileData.getType().equals(FileData.Type.HEAT_ENV)) {
 302                 module.setEnv(fileData.getFile());
 303             } else if (fileData.getType().equals(FileData.Type.HEAT_VOL))// must be volume
 304             {
 305                 module.setVol(fileData.getFile());
 306                 if (!CollectionUtils.isEmpty(fileData.getData())) {
 307                     FileData volEnv = fileData.getData().get(0);
 308                     if (volEnv.getType().equals(FileData.Type.HEAT_ENV)) {
 309                         module.setVolEnv(volEnv.getFile());
 310                     } else {
 311                         throw new CoreException((new ErrorCode.ErrorCodeBuilder())
 312                             .withMessage(Messages.ILLEGAL_MANIFEST.getErrorMessage())
 313                             .withId(Messages.ILLEGAL_MANIFEST.getErrorMessage())
 314                             .withCategory(ErrorCategory.APPLICATION).build());
 315                     }
 316                 }
 317             } else {
 318                 throw new CoreException((new ErrorCode.ErrorCodeBuilder())
 319                     .withMessage(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
 320                     .withId(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
 321                     .withCategory(ErrorCategory.APPLICATION).build());
 322             }
 323         }
 324     }
 325
 326     @Override
 327     public void updateCandidateUploadData(OrchestrationTemplateCandidateData uploadData,
 328                                           String itemId) {
 329         mdcDataDebugMessage.debugEntryMessage(null);
 330         orchestrationTemplateCandidateDataDao.update(itemId, uploadData);
 331         mdcDataDebugMessage.debugExitMessage(null);
 332     }
 333
 334     @Override
 335     public Optional<FilesDataStructure> getOrchestrationTemplateCandidateFileDataStructure(
 336         String vspId, Version version) {
 337
 338         mdcDataDebugMessage.debugEntryMessage("VSP Id", vspId);
 339
 340         Optional<String> jsonFileDataStructure =
 341             orchestrationTemplateCandidateDataDao.getStructure(vspId, version);
 342
 343         if (jsonFileDataStructure.isPresent()
 344             && JsonUtil.isValidJson(jsonFileDataStructure.get())) {
 345             mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 346             return Optional
 347                 .of(JsonUtil.json2Object(jsonFileDataStructure.get(), FilesDataStructure.class));
 348         } else {
 349             mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 350             return Optional.empty();
 351         }
 352     }
 353
 354     @Override
 355     public void updateOrchestrationTemplateCandidateFileDataStructure(String vspId, Version version,
 356                                                                       FilesDataStructure fileDataStructure) {
 357         OrchestrationTemplateCandidateDaoFactory.getInstance().createInterface()
 358             .updateStructure(vspId, version, fileDataStructure);
 359     }
 360
 361     @Override
 362     public OrchestrationTemplateCandidateData getOrchestrationTemplateCandidate(String vspId,
 363                                                                                 Version version) {
 364         mdcDataDebugMessage.debugEntryMessage("VSP Id", vspId);
 365         mdcDataDebugMessage.debugExitMessage("VSP Id", vspId);
 366
 367         return orchestrationTemplateCandidateDataDao.get(vspId, version);
 368     }
 369
 370   @Override
 371   public String createManifest(VspDetails vspDetails, FilesDataStructure structure) {
 372
 373         mdcDataDebugMessage.debugEntryMessage("VSP Id", vspDetails.getId());
 374
 375         Optional<ManifestContent> manifest = manifestCreator.createManifest(vspDetails, structure);
 376         if (!manifest.isPresent()) {
 377             throw new RuntimeException(Messages.CREATE_MANIFEST_FROM_ZIP.getErrorMessage());
 378         }
 379
 380         mdcDataDebugMessage.debugExitMessage("VSP Id", vspDetails.getId());
 381         return JsonUtil.object2Json(manifest.get());
 382     }
 383
 384     @Override
 385     public Optional<ManifestContent> createManifest(VspDetails vspDetails,
 386                                                     FileContentHandler fileContentHandler,
 387                                                     AnalyzedZipHeatFiles analyzedZipHeatFiles) {
 388
 389
 390         mdcDataDebugMessage.debugEntryMessage("VSP Id", vspDetails.getId());
 391
 392         mdcDataDebugMessage.debugExitMessage("VSP Id", vspDetails.getId());
 393         return manifestCreator.createManifest(vspDetails, fileContentHandler, analyzedZipHeatFiles);
 394     }
 395
 396     @Override
 397     public Optional<ByteArrayInputStream> fetchZipFileByteArrayInputStream(String vspId,
 398                                                                            OrchestrationTemplateCandidateData candidateDataEntity,
 399                                                                            String manifest,
 400                                                                            OnboardingTypesEnum type,
 401                                                                            Map<String, List<ErrorMessage>> uploadErrors) {
 402         byte[] file;
 403         ByteArrayInputStream byteArrayInputStream = null;
 404         try {
 405             file = replaceManifestInZip(candidateDataEntity.getContentData(), manifest, vspId, type);
 406             byteArrayInputStream = new ByteArrayInputStream(
 407                 Objects.isNull(file) ? candidateDataEntity.getContentData().array()
 408                     : file);
 409         } catch (IOException e) {
 410             ErrorMessage errorMessage =
 411                 new ErrorMessage(ErrorLevel.ERROR,
 412                     Messages.CANDIDATE_PROCESS_FAILED.getErrorMessage());
 413             logger.error(errorMessage.getMessage(), e);
 414             ErrorsUtil
 415                 .addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE, errorMessage, uploadErrors);
 416         }
 417         return Optional.ofNullable(byteArrayInputStream);
 418     }
 419
 420     @Override
 421     public byte[] replaceManifestInZip(ByteBuffer contentData, String manifest, String vspId,
 422                                        OnboardingTypesEnum type)
 423         throws IOException {
 424         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 425
 426         try (final ZipOutputStream zos = new ZipOutputStream(baos);
 427              ZipInputStream zipStream = new ZipInputStream(
 428                  new ByteArrayInputStream(contentData.array()))) {
 429             ZipEntry zipEntry;
 430             boolean manifestWritten = false;
 431             while ((zipEntry = zipStream.getNextEntry()) != null) {
 432                 if (!zipEntry.getName().equalsIgnoreCase(SdcCommon.MANIFEST_NAME)) {
 433                     ZipEntry loc_ze = new ZipEntry(zipEntry.getName());
 434                     zos.putNextEntry(loc_ze);
 435                     byte[] buf = new byte[1024];
 436                     int len;
 437                     while ((len = zipStream.read(buf)) > 0) {
 438                         zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 439                     }
 440                 } else {
 441                     manifestWritten = true;
 442                     writeManifest(manifest, type, zos);
 443                 }
 444                 zos.closeEntry();
 445             }
 446             if (!manifestWritten) {
 447                 writeManifest(manifest, type, zos);
 448                 zos.closeEntry();
 449             }
 450         }
 451         return baos.toByteArray();
 452     }
 453
 454     @Override
 455     public Optional<List<ErrorMessage>> validateFileDataStructure(
 456         FilesDataStructure filesDataStructure) {
 457         return candidateServiceValidator.validateFileDataStructure(filesDataStructure);
 458     }
 459
 460     private void writeManifest(String manifest,
 461                                OnboardingTypesEnum type,
 462                                ZipOutputStream zos) throws IOException {
 463
 464         if(isManifestNeedsToGetWritten(type)){
 465             return;
 466         }
 467
 468         zos.putNextEntry(new ZipEntry(SdcCommon.MANIFEST_NAME));
 469         try (InputStream manifestStream = new ByteArrayInputStream(
 470             manifest.getBytes(StandardCharsets.UTF_8))) {
 471             byte[] buf = new byte[1024];
 472             int len;
 473             while ((len = (manifestStream.read(buf))) > 0) {
 474                 zos.write(buf, 0, (len < buf.length) ? len : buf.length);
 475             }
 476         }
 477     }
 478
 479     private boolean isManifestNeedsToGetWritten(OnboardingTypesEnum type) {
 480         return type.equals(OnboardingTypesEnum.CSAR);
 481     }
 482
 483     private void handleArtifactsFromTree(HeatStructureTree tree, FilesDataStructure structure) {
 484
 485         if (Objects.isNull(tree) || Objects.isNull(tree.getArtifacts())) {
 486             return;
 487         }
 488
 489         if (CollectionUtils.isNotEmpty(tree.getArtifacts())) {
 490             structure.getArtifacts().addAll(
 491                 tree.getArtifacts()
 492                     .stream()
 493                     .map(Artifact::getFileName)
 494                     .filter(fileName -> !structure.getArtifacts().contains(fileName))
 495                     .collect(Collectors.toList()));
 496         }
 497     }
 498
 499     private void handleOtherResources(HeatStructureTree tree, Set<String> usedEnvFiles,
 500                                       FilesDataStructure structure) {
 501         Set<HeatStructureTree> others = tree.getOther();
 502         if (Objects.isNull(others)) {
 503             return;
 504         }
 505
 506         List<String> artifacts = new ArrayList<>();
 507         List<String> unassigned = new ArrayList<>();
 508         for (HeatStructureTree other : others) {
 509             if (HeatFileAnalyzer.isYamlOrEnvFile(other.getFileName())) {
 510                 if (isEnvFileUsedByHeatFile(usedEnvFiles, other)) {
 511                     continue;
 512                 }
 513                 unassigned.add(other.getFileName());
 514             } else {
 515                 artifacts.add(other.getFileName());
 516             }
 517             handleArtifactsFromTree(other, structure);
 518         }
 519         structure.getArtifacts().addAll(artifacts);
 520         structure.getUnassigned().addAll(unassigned);
 521     }
 522
 523     private boolean isEnvFileUsedByHeatFile(Set<String> usedEnvFiles, HeatStructureTree other) {
 524         if (HeatFileAnalyzer.isEnvFile(other.getFileName())) {
 525             if (usedEnvFiles.contains(other.getFileName())) {
 526                 return true;
 527             }
 528         }
 529         return false;
 530     }
 531
 532     private void addHeatsToFileDataStructure(HeatStructureTree tree, Set<String> usedEnvFiles,
 533                                              FilesDataStructure structure,
 534                                              Map<String, List<ErrorMessage>> uploadErrors,
 535                                              AnalyzedZipHeatFiles analyzedZipHeatFiles)
 536         throws Exception {
 537         List<Module> modules = new ArrayList<>();
 538         Set<HeatStructureTree> heatsSet = tree.getHeat();
 539         if (Objects.isNull(heatsSet)) {
 540             return;
 541         }
 542         for (HeatStructureTree heat : heatsSet) {
 543             if (isFileBaseFile(heat.getFileName())) {
 544                 handleSingleHeat(structure, modules, heat, uploadErrors);
 545             } else if (isFileModuleFile(heat.getFileName(),
 546                 analyzedZipHeatFiles.getModuleFiles())) {
 547                 handleSingleHeat(structure, modules, heat, uploadErrors);
 548             } else {
 549                 structure.getUnassigned().add(heat.getFileName());
 550                 addNestedToFileDataStructure(heat, structure);
 551             }
 552             if (!Objects.isNull(heat.getEnv())) {
 553                 usedEnvFiles.add(heat.getEnv() == null ? null : heat.getEnv().getFileName());
 554             }
 555         }
 556         structure.setModules(modules);
 557
 558     }
 559
 560     private boolean isFileModuleFile(String fileName, Set<String> modulesFileNames) {
 561         return modulesFileNames.contains(fileName);
 562     }
 563
 564     private boolean isFileBaseFile(String fileName) {
 565         return manifestCreator.isFileBaseFile(fileName);
 566     }
 567
 568     private void handleSingleHeat(FilesDataStructure structure, List<Module> modules,
 569                                   HeatStructureTree heat,
 570                                   Map<String, List<ErrorMessage>> uploadErrors) {
 571
 572
 573         mdcDataDebugMessage.debugEntryMessage(null);
 574
 575         Module module = new Module();
 576         module.setYaml(heat.getFileName());
 577         module.setIsBase(heat.getBase());
 578         addNestedToFileDataStructure(heat, structure);
 579         Set<HeatStructureTree> volumeSet = heat.getVolume();
 580         int inx = 0;
 581         if (Objects.nonNull(volumeSet)) {
 582             handleVolumes(module, volumeSet, structure, inx, uploadErrors);
 583         }
 584         handleEnv(module, heat, false, structure);
 585         modules.add(module);
 586
 587         mdcDataDebugMessage.debugExitMessage(null);
 588     }
 589
 590     private void handleVolumes(Module module, Set<HeatStructureTree> volumeSet,
 591                                FilesDataStructure structure, int inx,
 592                                Map<String, List<ErrorMessage>> uploadErrors) {
 593
 594
 595         mdcDataDebugMessage.debugEntryMessage(null);
 596
 597         for (HeatStructureTree volume : volumeSet) {
 598             if (inx++ > 0) {
 599                 ErrorsUtil.addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE,
 600                     new ErrorMessage(ErrorLevel.WARNING,
 601                         Messages.MORE_THEN_ONE_VOL_FOR_HEAT.getErrorMessage()), uploadErrors);
 602                 break;
 603             }
 604             handleArtifactsFromTree(volume, structure);
 605             module.setVol(volume.getFileName());
 606             handleEnv(module, volume, true, structure);
 607             addNestedToFileDataStructure(volume, structure);
 608         }
 609
 610         mdcDataDebugMessage.debugExitMessage(null);
 611     }
 612
 613     private void handleEnv(Module module, HeatStructureTree tree, boolean isVolEnv,
 614                            FilesDataStructure structure) {
 615
 616
 617         mdcDataDebugMessage.debugEntryMessage(null);
 618
 619         if (Objects.nonNull(tree.getEnv())) {
 620             if (isVolEnv) {
 621                 module.setVolEnv(tree.getEnv().getFileName());
 622             } else {
 623                 module.setEnv(tree.getEnv().getFileName());
 624             }
 625             handleArtifactsFromTree(tree.getEnv(), structure);
 626         }
 627
 628         mdcDataDebugMessage.debugExitMessage(null);
 629     }
 630
 631     private void addNestedToFileDataStructure(HeatStructureTree heat,
 632                                               FilesDataStructure structure) {
 633         Set<HeatStructureTree> nestedSet = heat.getNested();
 634         if (Objects.isNull(nestedSet)) {
 635             return;
 636         }
 637         for (HeatStructureTree nested : nestedSet) {
 638             if (structure.getNested().contains(nested.getFileName())) {
 639                 continue;
 640             }
 641             structure.getNested().add(nested.getFileName());
 642             if (CollectionUtils.isNotEmpty(nested.getArtifacts())) {
 643                 handleArtifactsFromTree(nested, structure);
 644             }
 645             addNestedToFileDataStructure(nested, structure);
 646         }
 647     }
 648 }