2 * Copyright © 2016-2018 European Support Limited
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package org.openecomp.sdc.vendorsoftwareproduct.services.impl.filedatastructuremodule;
19 import org.apache.commons.collections4.CollectionUtils;
20 import org.openecomp.core.utilities.file.FileContentHandler;
21 import org.openecomp.core.utilities.json.JsonUtil;
22 import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
23 import org.openecomp.sdc.common.errors.CoreException;
24 import org.openecomp.sdc.common.errors.ErrorCategory;
25 import org.openecomp.sdc.common.errors.ErrorCode;
26 import org.openecomp.sdc.common.errors.Messages;
27 import org.openecomp.sdc.common.utils.SdcCommon;
28 import org.openecomp.sdc.datatypes.error.ErrorLevel;
29 import org.openecomp.sdc.datatypes.error.ErrorMessage;
30 import org.openecomp.sdc.heat.datatypes.manifest.FileData;
31 import org.openecomp.sdc.heat.datatypes.manifest.ManifestContent;
32 import org.openecomp.sdc.heat.datatypes.structure.Artifact;
33 import org.openecomp.sdc.heat.datatypes.structure.HeatStructureTree;
34 import org.openecomp.sdc.heat.datatypes.structure.ValidationStructureList;
35 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDao;
36 import org.openecomp.sdc.vendorsoftwareproduct.dao.OrchestrationTemplateCandidateDaoFactory;
37 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.OrchestrationTemplateCandidateData;
38 import org.openecomp.sdc.vendorsoftwareproduct.dao.type.VspDetails;
39 import org.openecomp.sdc.vendorsoftwareproduct.errors.utils.ErrorsUtil;
40 import org.openecomp.sdc.vendorsoftwareproduct.services.HeatFileAnalyzer;
41 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.CandidateService;
42 import org.openecomp.sdc.vendorsoftwareproduct.services.filedatastructuremodule.ManifestCreator;
43 import org.openecomp.sdc.vendorsoftwareproduct.services.utils.CandidateServiceValidator;
44 import org.openecomp.sdc.vendorsoftwareproduct.types.CandidateDataEntityTo;
45 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.AnalyzedZipHeatFiles;
46 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.FilesDataStructure;
47 import org.openecomp.sdc.vendorsoftwareproduct.types.candidateheat.Module;
48 import org.openecomp.sdc.versioning.dao.types.Version;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
52 import java.io.ByteArrayInputStream;
53 import java.io.ByteArrayOutputStream;
54 import java.io.IOException;
55 import java.io.InputStream;
56 import java.nio.ByteBuffer;
57 import java.nio.charset.StandardCharsets;
58 import java.util.ArrayList;
59 import java.util.HashSet;
60 import java.util.List;
62 import java.util.Objects;
63 import java.util.Optional;
65 import java.util.stream.Collectors;
66 import java.util.zip.ZipEntry;
67 import java.util.zip.ZipInputStream;
68 import java.util.zip.ZipOutputStream;
70 import static org.openecomp.core.validation.errors.ErrorMessagesFormatBuilder.getErrorWithParameters;
72 public class CandidateServiceImpl implements CandidateService {
73 protected static final Logger logger = LoggerFactory.getLogger(CandidateServiceImpl.class);
74 private CandidateServiceValidator candidateServiceValidator = new CandidateServiceValidator();
75 private ManifestCreator manifestCreator;
76 private OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDao;
78 public CandidateServiceImpl(ManifestCreator manifestCreator,
79 OrchestrationTemplateCandidateDao orchestrationTemplateCandidateDao) {
80 this.manifestCreator = manifestCreator;
81 this.orchestrationTemplateCandidateDao = orchestrationTemplateCandidateDao;
85 public CandidateServiceImpl() {
89 public Optional<ErrorMessage> validateNonEmptyFileToUpload(InputStream fileToUpload,
92 getErrorWithParameters(Messages.NO_FILE_WAS_UPLOADED_OR_FILE_NOT_EXIST.getErrorMessage(),
95 if (Objects.isNull(fileToUpload)) {
96 return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
100 int available = fileToUpload.available();
101 if (available == 0) {
102 return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
105 } catch (IOException e) {
106 logger.debug(e.getMessage(), e);
107 return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
111 return Optional.empty();
115 public Optional<ErrorMessage> validateRawZipData(String fileSuffix,
116 byte[] uploadedFileData) {
117 if (Objects.isNull(uploadedFileData)) {
118 return Optional.of(new ErrorMessage(ErrorLevel.ERROR,
119 getErrorWithParameters(Messages.NO_FILE_WAS_UPLOADED_OR_FILE_NOT_EXIST.getErrorMessage(),
122 return Optional.empty();
125 private String heatStructureTreeToFileDataStructure(HeatStructureTree tree,
126 FileContentHandler zipContentMap,
127 Map<String, List<ErrorMessage>> uploadErrors,
128 AnalyzedZipHeatFiles analyzedZipHeatFiles)
130 FilesDataStructure structure = new FilesDataStructure();
131 Set<String> usedEnvFiles = new HashSet<>();
132 addHeatsToFileDataStructure(tree, usedEnvFiles, structure, uploadErrors,
133 analyzedZipHeatFiles);
134 handleOtherResources(tree, usedEnvFiles, structure);
135 FilesDataStructure fileDataStructureFromManifest =
136 createFileDataStructureFromManifest(zipContentMap.getFileContent(SdcCommon.MANIFEST_NAME));
137 List<String> structureArtifacts = structure.getArtifacts();
138 structureArtifacts.addAll(fileDataStructureFromManifest.getArtifacts().stream().filter
139 (artifact -> isNotStrctureArtifact(structureArtifacts, artifact))
140 .collect(Collectors.toList()));
141 handleArtifactsFromTree(tree, structure);
143 return JsonUtil.object2Json(structure);
146 private boolean isNotStrctureArtifact(List<String> structureArtifacts, String artifact) {
147 return !structureArtifacts.contains(artifact);
151 public OrchestrationTemplateCandidateData createCandidateDataEntity(
152 CandidateDataEntityTo candidateDataEntityTo, InputStream zipFileManifest,
153 AnalyzedZipHeatFiles analyzedZipHeatFiles) throws Exception {
154 FileContentHandler zipContentMap = candidateDataEntityTo.getContentMap();
155 FilesDataStructure filesDataStructure;
156 String dataStructureJson;
158 if (zipFileManifest != null) {
159 // create data structure from manifest
160 filesDataStructure = createFileDataStructureFromManifest(zipFileManifest);
161 Set<String> zipFileList = zipContentMap.getFileList();
162 balanceManifestFilesWithZipFiles(filesDataStructure,
163 zipContentMap, analyzedZipHeatFiles);
164 Set<String> filesDataStructureFiles = getFlatFileNames(filesDataStructure);
165 filesDataStructure.getUnassigned().addAll(zipFileList.stream()
166 .filter(fileName -> (!filesDataStructureFiles.contains(fileName)
167 && !filesDataStructure.getNested().contains(fileName)
168 && !fileName.equals(SdcCommon.MANIFEST_NAME)))
169 .collect(Collectors.toList()));
170 dataStructureJson = JsonUtil.object2Json(filesDataStructure);
172 // create data structure from based on naming convention
174 heatStructureTreeToFileDataStructure(candidateDataEntityTo.getTree(), zipContentMap,
175 candidateDataEntityTo.getErrors(), analyzedZipHeatFiles);
178 OrchestrationTemplateCandidateData candidateData = new OrchestrationTemplateCandidateData();
179 candidateData.setContentData(ByteBuffer.wrap(candidateDataEntityTo.getUploadedFileData()));
180 candidateData.setFilesDataStructure(dataStructureJson);
181 return candidateData;
184 private void balanceManifestFilesWithZipFiles(
185 FilesDataStructure filesDataStructure,
186 FileContentHandler fileContentHandler, AnalyzedZipHeatFiles analyzedZipHeatFiles)
188 Set<String> zipFileList = fileContentHandler.getFileList();
189 filesDataStructure.getNested().addAll(analyzedZipHeatFiles.getNestedFiles());
190 List<Module> modules = filesDataStructure.getModules();
191 if (CollectionUtils.isEmpty(modules)) {
195 for (int i = 0; i < modules.size(); i++) {
196 Module module = modules.get(i);
197 if (!isFileExistInZipContains(zipFileList, module.getYaml())) {
198 addFileToUnassigned(filesDataStructure, zipFileList, module.getEnv());
199 addFileToUnassigned(filesDataStructure, zipFileList, module.getVol());
200 addFileToUnassigned(filesDataStructure, zipFileList, module.getVolEnv());
202 } else if (Objects.nonNull(module.getVol()) && !zipFileList.contains(module.getVol())) {
205 .addIgnoreNull(filesDataStructure.getUnassigned(), module.getVolEnv());
207 if (filesDataStructure.getNested().contains(module.getYaml())) {
208 moveModuleFileToNested(filesDataStructure, i--, module);
214 private void addFileToUnassigned(FilesDataStructure filesDataStructure, Set<String> zipFileList,
216 if (isFileExistInZipContains(zipFileList, fileName)) {
217 filesDataStructure.getUnassigned().add(fileName);
221 private boolean isFileExistInZipContains(Set<String> zipFileList, String fileName) {
222 return Objects.nonNull(fileName) && zipFileList.contains(fileName);
225 private void moveModuleFileToNested(FilesDataStructure filesDataStructure, int i,
227 if (!filesDataStructure.getNested().contains(module.getYaml())) {
228 filesDataStructure.getNested().add(module.getYaml());
230 if (Objects.nonNull(module.getEnv())) {
231 filesDataStructure.getNested().add(module.getEnv());
233 if (Objects.nonNull(module.getVol())) {
234 filesDataStructure.getNested().add(module.getVol());
236 if (Objects.nonNull(module.getVolEnv())) {
237 filesDataStructure.getNested().add(module.getVolEnv());
239 filesDataStructure.getModules().remove(i);
242 private Set<String> getFlatFileNames(FilesDataStructure filesDataStructure) {
243 Set<String> fileNames = new HashSet<>();
244 if (!CollectionUtils.isEmpty(filesDataStructure.getModules())) {
245 for (Module module : filesDataStructure.getModules()) {
246 CollectionUtils.addIgnoreNull(fileNames, module.getEnv());
247 CollectionUtils.addIgnoreNull(fileNames, module.getVol());
248 CollectionUtils.addIgnoreNull(fileNames, module.getVolEnv());
249 CollectionUtils.addIgnoreNull(fileNames, module.getYaml());
252 fileNames.addAll(filesDataStructure.getArtifacts().stream().collect(Collectors.toSet()));
253 fileNames.addAll(filesDataStructure.getNested().stream().collect(Collectors.toSet()));
254 fileNames.addAll(filesDataStructure.getUnassigned().stream().collect(Collectors.toSet()));
259 private FilesDataStructure createFileDataStructureFromManifest(InputStream isManifestContent) {
260 ManifestContent manifestContent =
261 JsonUtil.json2Object(isManifestContent, ManifestContent.class);
262 FilesDataStructure structure = new FilesDataStructure();
263 for (FileData fileData : manifestContent.getData()) {
264 if (Objects.nonNull(fileData.getType()) &&
265 fileData.getType().equals(FileData.Type.HEAT)) {
266 Module module = new Module();
267 module.setYaml(fileData.getFile());
268 module.setIsBase(fileData.getBase());
269 addHeatDependenciesToModule(module, fileData.getData());
270 structure.getModules().add(module);
271 } else if (HeatFileAnalyzer.isYamlOrEnvFile(fileData.getFile()) &&
272 !FileData.Type.isArtifact(fileData.getType())) {
273 structure.getUnassigned().add(fileData.getFile());
275 structure.getArtifacts().add(fileData.getFile());
281 private void addHeatDependenciesToModule(Module module, List<FileData> data) {
282 if (CollectionUtils.isEmpty(data)) {
286 for (FileData fileData : data) {
287 if (fileData.getType().equals(FileData.Type.HEAT_ENV)) {
288 module.setEnv(fileData.getFile());
289 } else if (fileData.getType().equals(FileData.Type.HEAT_VOL)) { // must be volume
290 module.setVol(fileData.getFile());
291 if (!CollectionUtils.isEmpty(fileData.getData())) {
292 FileData volEnv = fileData.getData().get(0);
293 if (volEnv.getType().equals(FileData.Type.HEAT_ENV)) {
294 module.setVolEnv(volEnv.getFile());
296 throw new CoreException((new ErrorCode.ErrorCodeBuilder())
297 .withMessage(Messages.ILLEGAL_MANIFEST.getErrorMessage())
298 .withId(Messages.ILLEGAL_MANIFEST.getErrorMessage())
299 .withCategory(ErrorCategory.APPLICATION).build());
303 throw new CoreException((new ErrorCode.ErrorCodeBuilder())
304 .withMessage(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
305 .withId(Messages.FILE_TYPE_NOT_LEGAL.getErrorMessage())
306 .withCategory(ErrorCategory.APPLICATION).build());
312 public void updateCandidateUploadData(String vspId, Version version,
313 OrchestrationTemplateCandidateData uploadData) {
314 orchestrationTemplateCandidateDao.update(vspId, version, uploadData);
318 public Optional<FilesDataStructure> getOrchestrationTemplateCandidateFileDataStructure(
319 String vspId, Version version) {
320 Optional<String> jsonFileDataStructure =
321 orchestrationTemplateCandidateDao.getStructure(vspId, version);
323 if (jsonFileDataStructure.isPresent() && JsonUtil.isValidJson(jsonFileDataStructure.get())) {
325 .of(JsonUtil.json2Object(jsonFileDataStructure.get(), FilesDataStructure.class));
327 return Optional.empty();
332 public void updateOrchestrationTemplateCandidateFileDataStructure(String vspId, Version version,
333 FilesDataStructure fileDataStructure) {
334 OrchestrationTemplateCandidateDaoFactory.getInstance().createInterface()
335 .updateStructure(vspId, version, fileDataStructure);
339 public OrchestrationTemplateCandidateData getOrchestrationTemplateCandidate(String vspId,
341 return orchestrationTemplateCandidateDao.get(vspId, version);
345 public OrchestrationTemplateCandidateData getOrchestrationTemplateCandidateInfo(String vspId,
347 return orchestrationTemplateCandidateDao.getInfo(vspId, version);
351 public String createManifest(VspDetails vspDetails, FilesDataStructure structure) {
352 Optional<ManifestContent> manifest = manifestCreator.createManifest(vspDetails, structure);
353 if (!manifest.isPresent()) {
354 throw new RuntimeException(Messages.CREATE_MANIFEST_FROM_ZIP.getErrorMessage());
356 return JsonUtil.object2Json(manifest.get());
360 public Optional<ManifestContent> createManifest(VspDetails vspDetails,
361 FileContentHandler fileContentHandler,
362 AnalyzedZipHeatFiles analyzedZipHeatFiles) {
363 return manifestCreator.createManifest(vspDetails, fileContentHandler, analyzedZipHeatFiles);
367 public Optional<ByteArrayInputStream> fetchZipFileByteArrayInputStream(String vspId,
368 OrchestrationTemplateCandidateData candidateDataEntity,
370 OnboardingTypesEnum type,
371 Map<String, List<ErrorMessage>> uploadErrors) {
373 ByteArrayInputStream byteArrayInputStream = null;
375 file = replaceManifestInZip(candidateDataEntity.getContentData(), manifest, vspId, type);
376 byteArrayInputStream = new ByteArrayInputStream(
377 Objects.isNull(file) ? candidateDataEntity.getContentData().array()
379 } catch (IOException e) {
380 ErrorMessage errorMessage =
381 new ErrorMessage(ErrorLevel.ERROR,
382 Messages.CANDIDATE_PROCESS_FAILED.getErrorMessage());
383 logger.error(errorMessage.getMessage(), e);
385 .addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE, errorMessage, uploadErrors);
387 return Optional.ofNullable(byteArrayInputStream);
391 public byte[] replaceManifestInZip(ByteBuffer contentData, String manifest, String vspId,
392 OnboardingTypesEnum type)
394 ByteArrayOutputStream baos = new ByteArrayOutputStream();
396 try (final ZipOutputStream zos = new ZipOutputStream(baos);
397 ZipInputStream zipStream = new ZipInputStream(
398 new ByteArrayInputStream(contentData.array()))) {
400 boolean manifestWritten = false;
401 while ((zipEntry = zipStream.getNextEntry()) != null) {
402 if (!zipEntry.getName().equalsIgnoreCase(SdcCommon.MANIFEST_NAME)) {
403 ZipEntry loc_ze = new ZipEntry(zipEntry.getName());
404 zos.putNextEntry(loc_ze);
405 byte[] buf = new byte[1024];
407 while ((len = zipStream.read(buf)) > 0) {
408 zos.write(buf, 0, (len < buf.length) ? len : buf.length);
411 manifestWritten = true;
412 writeManifest(manifest, type, zos);
416 if (!manifestWritten) {
417 writeManifest(manifest, type, zos);
421 return baos.toByteArray();
425 public Optional<List<ErrorMessage>> validateFileDataStructure(
426 FilesDataStructure filesDataStructure) {
427 return candidateServiceValidator.validateFileDataStructure(filesDataStructure);
431 public void deleteOrchestrationTemplateCandidate(String vspId, Version versionId) {
432 orchestrationTemplateCandidateDao.delete(vspId, versionId);
436 public void updateValidationData(String vspId, Version version, ValidationStructureList
438 orchestrationTemplateCandidateDao.updateValidationData(vspId, version, validationData);
441 private void writeManifest(String manifest,
442 OnboardingTypesEnum type,
443 ZipOutputStream zos) throws IOException {
445 if (isManifestNeedsToGetWritten(type)) {
449 zos.putNextEntry(new ZipEntry(SdcCommon.MANIFEST_NAME));
450 try (InputStream manifestStream = new ByteArrayInputStream(
451 manifest.getBytes(StandardCharsets.UTF_8))) {
452 byte[] buf = new byte[1024];
454 while ((len = (manifestStream.read(buf))) > 0) {
455 zos.write(buf, 0, (len < buf.length) ? len : buf.length);
460 private boolean isManifestNeedsToGetWritten(OnboardingTypesEnum type) {
461 return type.equals(OnboardingTypesEnum.CSAR);
464 private void handleArtifactsFromTree(HeatStructureTree tree, FilesDataStructure structure) {
466 if (Objects.isNull(tree) || Objects.isNull(tree.getArtifacts())) {
470 if (CollectionUtils.isNotEmpty(tree.getArtifacts())) {
471 structure.getArtifacts().addAll(
474 .map(Artifact::getFileName)
475 .filter(fileName -> !structure.getArtifacts().contains(fileName))
476 .collect(Collectors.toList()));
480 private void handleOtherResources(HeatStructureTree tree, Set<String> usedEnvFiles,
481 FilesDataStructure structure) {
482 Set<HeatStructureTree> others = tree.getOther();
483 if (Objects.isNull(others)) {
487 List<String> artifacts = new ArrayList<>();
488 List<String> unassigned = new ArrayList<>();
489 for (HeatStructureTree other : others) {
490 if (HeatFileAnalyzer.isYamlOrEnvFile(other.getFileName())) {
491 if (isEnvFileUsedByHeatFile(usedEnvFiles, other)) {
494 unassigned.add(other.getFileName());
496 artifacts.add(other.getFileName());
498 handleArtifactsFromTree(other, structure);
500 structure.getArtifacts().addAll(artifacts);
501 structure.getUnassigned().addAll(unassigned);
504 private boolean isEnvFileUsedByHeatFile(Set<String> usedEnvFiles, HeatStructureTree other) {
505 if (HeatFileAnalyzer.isEnvFile(other.getFileName())) {
506 if (usedEnvFiles.contains(other.getFileName())) {
513 private void addHeatsToFileDataStructure(HeatStructureTree tree, Set<String> usedEnvFiles,
514 FilesDataStructure structure,
515 Map<String, List<ErrorMessage>> uploadErrors,
516 AnalyzedZipHeatFiles analyzedZipHeatFiles)
518 List<Module> modules = new ArrayList<>();
519 Set<HeatStructureTree> heatsSet = tree.getHeat();
520 if (Objects.isNull(heatsSet)) {
523 for (HeatStructureTree heat : heatsSet) {
524 if (isFileBaseFile(heat.getFileName())) {
525 handleSingleHeat(structure, modules, heat, uploadErrors);
526 } else if (isFileModuleFile(heat.getFileName(),
527 analyzedZipHeatFiles.getModuleFiles())) {
528 handleSingleHeat(structure, modules, heat, uploadErrors);
530 structure.getUnassigned().add(heat.getFileName());
531 addNestedToFileDataStructure(heat, structure);
533 if (!Objects.isNull(heat.getEnv())) {
534 usedEnvFiles.add(heat.getEnv() == null ? null : heat.getEnv().getFileName());
537 structure.setModules(modules);
541 private boolean isFileModuleFile(String fileName, Set<String> modulesFileNames) {
542 return modulesFileNames.contains(fileName);
545 private boolean isFileBaseFile(String fileName) {
546 return manifestCreator.isFileBaseFile(fileName);
549 private void handleSingleHeat(FilesDataStructure structure, List<Module> modules,
550 HeatStructureTree heat,
551 Map<String, List<ErrorMessage>> uploadErrors) {
552 Module module = new Module();
553 module.setYaml(heat.getFileName());
554 module.setIsBase(heat.getBase());
555 addNestedToFileDataStructure(heat, structure);
556 Set<HeatStructureTree> volumeSet = heat.getVolume();
558 if (Objects.nonNull(volumeSet)) {
559 handleVolumes(module, volumeSet, structure, inx, uploadErrors);
561 handleEnv(module, heat, false, structure);
565 private void handleVolumes(Module module, Set<HeatStructureTree> volumeSet,
566 FilesDataStructure structure, int inx,
567 Map<String, List<ErrorMessage>> uploadErrors) {
568 for (HeatStructureTree volume : volumeSet) {
570 ErrorsUtil.addStructureErrorToErrorMap(SdcCommon.UPLOAD_FILE,
571 new ErrorMessage(ErrorLevel.WARNING,
572 Messages.MORE_THEN_ONE_VOL_FOR_HEAT.getErrorMessage()), uploadErrors);
575 handleArtifactsFromTree(volume, structure);
576 module.setVol(volume.getFileName());
577 handleEnv(module, volume, true, structure);
578 addNestedToFileDataStructure(volume, structure);
582 private void handleEnv(Module module, HeatStructureTree tree, boolean isVolEnv,
583 FilesDataStructure structure) {
584 if (Objects.nonNull(tree.getEnv())) {
586 module.setVolEnv(tree.getEnv().getFileName());
588 module.setEnv(tree.getEnv().getFileName());
590 handleArtifactsFromTree(tree.getEnv(), structure);
594 private void addNestedToFileDataStructure(HeatStructureTree heat,
595 FilesDataStructure structure) {
596 Set<HeatStructureTree> nestedSet = heat.getNested();
597 if (Objects.isNull(nestedSet)) {
600 for (HeatStructureTree nested : nestedSet) {
601 if (structure.getNested().contains(nested.getFileName())) {
604 structure.getNested().add(nested.getFileName());
605 if (CollectionUtils.isNotEmpty(nested.getArtifacts())) {
606 handleArtifactsFromTree(nested, structure);
608 addNestedToFileDataStructure(nested, structure);