2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2023 Nordix Foundation. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 package org.openecomp.sdc.be.tosca;
22 import static org.openecomp.sdc.be.dao.api.ActionStatus.ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION;
23 import static org.openecomp.sdc.be.dao.api.ActionStatus.ERROR_DURING_CSAR_CREATION;
24 import static org.openecomp.sdc.be.tosca.ComponentCache.MergeStrategy.overwriteIfSameVersions;
25 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.fromEither;
26 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.javaListToVavrList;
27 import static org.openecomp.sdc.common.api.Constants.ADDITIONAL_TYPE_DEFINITIONS;
29 import com.google.common.primitives.Bytes;
31 import fj.data.Either;
32 import io.vavr.Tuple2;
33 import io.vavr.control.Try;
34 import java.io.BufferedOutputStream;
35 import java.io.ByteArrayInputStream;
37 import java.io.IOException;
38 import java.nio.charset.StandardCharsets;
39 import java.nio.file.Path;
40 import java.nio.file.Paths;
41 import java.text.SimpleDateFormat;
42 import java.util.ArrayList;
43 import java.util.Date;
44 import java.util.EnumMap;
45 import java.util.HashMap;
46 import java.util.HashSet;
47 import java.util.List;
49 import java.util.Map.Entry;
50 import java.util.Objects;
51 import java.util.Optional;
53 import java.util.TimeZone;
54 import java.util.function.Function;
55 import java.util.function.Predicate;
56 import java.util.function.Supplier;
57 import java.util.regex.Matcher;
58 import java.util.regex.Pattern;
59 import java.util.stream.Collectors;
60 import java.util.stream.Stream;
61 import java.util.zip.ZipEntry;
62 import java.util.zip.ZipInputStream;
63 import java.util.zip.ZipOutputStream;
64 import org.apache.commons.collections.CollectionUtils;
65 import org.apache.commons.collections.MapUtils;
66 import org.apache.commons.io.output.ByteArrayOutputStream;
67 import org.apache.commons.lang.StringUtils;
68 import org.apache.commons.lang3.tuple.ImmutableTriple;
69 import org.apache.commons.lang3.tuple.Triple;
70 import org.apache.commons.text.WordUtils;
71 import org.onap.sdc.tosca.services.YamlUtil;
72 import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException;
73 import org.openecomp.sdc.be.config.CategoryBaseTypeConfig;
74 import org.openecomp.sdc.be.config.ConfigurationManager;
75 import org.openecomp.sdc.be.dao.api.ActionStatus;
76 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
77 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
78 import org.openecomp.sdc.be.dao.cassandra.SdcSchemaFilesCassandraDao;
79 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
80 import org.openecomp.sdc.be.datatypes.components.ResourceMetadataDataDefinition;
81 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
82 import org.openecomp.sdc.be.datatypes.elements.OperationDataDefinition;
83 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
84 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
85 import org.openecomp.sdc.be.impl.ComponentsUtils;
86 import org.openecomp.sdc.be.model.ArtifactDefinition;
87 import org.openecomp.sdc.be.model.Component;
88 import org.openecomp.sdc.be.model.ComponentInstance;
89 import org.openecomp.sdc.be.model.InterfaceDefinition;
90 import org.openecomp.sdc.be.model.LifecycleStateEnum;
91 import org.openecomp.sdc.be.model.Resource;
92 import org.openecomp.sdc.be.model.Service;
93 import org.openecomp.sdc.be.model.category.CategoryDefinition;
94 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
95 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
96 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
97 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
98 import org.openecomp.sdc.be.model.operations.impl.ModelOperation;
99 import org.openecomp.sdc.be.plugins.CsarEntryGenerator;
100 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
101 import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil;
102 import org.openecomp.sdc.be.utils.TypeUtils;
103 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
104 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
105 import org.openecomp.sdc.common.impl.ExternalConfiguration;
106 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
107 import org.openecomp.sdc.common.util.GeneralUtility;
108 import org.openecomp.sdc.common.zip.ZipUtils;
109 import org.openecomp.sdc.exception.ResponseFormat;
110 import org.slf4j.Logger;
111 import org.slf4j.LoggerFactory;
112 import org.springframework.beans.factory.annotation.Autowired;
113 import org.yaml.snakeyaml.DumperOptions;
114 import org.yaml.snakeyaml.DumperOptions.FlowStyle;
115 import org.yaml.snakeyaml.Yaml;
118 * Generates a Network Service CSAR based on a SERVICE component and wraps it in a SDC CSAR entry.
120 @org.springframework.stereotype.Component("commonCsarGenerator")
121 public class CommonCsarGenerator {
123 private static final Logger LOGGER = LoggerFactory.getLogger(CommonCsarGenerator.class);
124 public static final String ARTIFACTS_PATH = "Artifacts/";
125 private static final String RESOURCES_PATH = "Resources/";
126 private static final String PATH_DELIMITER = "/";
127 private static final String SERVICE_MANIFEST = "NS.mf";
128 private static final String ARTIFACT_NAME_UNIQUE_ID = "ArtifactName {}, unique ID {}";
129 private static final String TOSCA_META_PATH_FILE_NAME = "TOSCA-Metadata/TOSCA.meta";
130 private static final String TOSCA_META_VERSION = "1.0";
131 private static final String CSAR_VERSION = "1.1";
132 private static final String BLOCK_0_TEMPLATE = "SDC-TOSCA-Meta-File-Version: %s\nSDC-TOSCA-Definitions-Version: %s\n";
133 private static final String CSAR_META_PATH_FILE_NAME = "csar.meta";
134 private static final String SDC_VERSION = ExternalConfiguration.getAppVersion();
135 public static final String NODES_YML = "nodes.yml";
136 private static final String CONFORMANCE_LEVEL = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
137 private final ToscaOperationFacade toscaOperationFacade;
138 private final ComponentsUtils componentsUtils;
139 private final ToscaExportHandler toscaExportUtils;
140 private final List<CsarEntryGenerator> generators;
141 private final ArtifactCassandraDao artifactCassandraDao;
142 private final String versionFirstThreeOctets;
143 private final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
144 private final ModelOperation modelOperation;
147 public CommonCsarGenerator(
148 final ToscaOperationFacade toscaOperationFacade,
149 final ComponentsUtils componentsUtils,
150 final ToscaExportHandler toscaExportUtils,
151 final List<CsarEntryGenerator> generators,
152 final ArtifactCassandraDao artifactCassandraDao,
153 final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao,
154 final ModelOperation modelOperation) {
155 this.toscaOperationFacade = toscaOperationFacade;
156 this.componentsUtils = componentsUtils;
157 this.toscaExportUtils = toscaExportUtils;
158 this.generators = generators;
159 this.artifactCassandraDao = artifactCassandraDao;
160 this.versionFirstThreeOctets = readVersionFirstThreeOctets();
161 this.sdcSchemaFilesCassandraDao = sdcSchemaFilesCassandraDao;
162 this.modelOperation = modelOperation;
165 private String readVersionFirstThreeOctets() {
166 if (StringUtils.isEmpty(SDC_VERSION)) {
169 // change regex to avoid DoS sonar issue
170 Matcher matcher = Pattern.compile("(?!\\.)(\\d{1,9}(\\.\\d{1,9}){1,9})(?![\\d\\.])").matcher(SDC_VERSION);
172 return matcher.group(0);
176 * Generates a Network Service CSAR based on a SERVICE component that has category configured in
177 * CategoriesToGenerateNsd enum and wraps it in a SDC CSAR entry.
179 * @param component the component to create the NS CSAR from
180 * @return an entry to be added in the Component CSAR by SDC
182 public Either<ZipOutputStream, ResponseFormat> generateCsarZip(Component component,
185 boolean isInCertificationRequest,
186 boolean isAsdPackage,
187 String definitionsPath,
188 boolean addDependencies,
189 boolean isSkipImports) throws IOException {
190 ArtifactDefinition artifactDef = component.getToscaArtifacts().get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
191 Either<ToscaRepresentation, ResponseFormat> toscaRepresentation = fetchToscaRepresentation(component, getFromCS, artifactDef, isSkipImports);
193 // This should not be done but in order to keep the refactoring small enough we stop here.
194 // TODO: Refactor the rest of this function
196 List<Triple<String, String, Component>> dependencies;
197 if (toscaRepresentation.isLeft()) {
198 mainYaml = toscaRepresentation.left().value().getMainYaml();
199 dependencies = toscaRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
201 return Either.right(toscaRepresentation.right().value());
204 if (!isSkipImports) {
205 final String toscaConformanceLevel = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
206 zip.putNextEntry(new ZipEntry(CSAR_META_PATH_FILE_NAME));
207 zip.write(createCsarBlock0(TOSCA_META_VERSION, toscaConformanceLevel).getBytes());
210 final String fileName = artifactDef.getArtifactName();
211 final byte[] toscaBlock0Byte = createToscaBlock0(
212 TOSCA_META_VERSION, CSAR_VERSION, component.getCreatorFullName(), fileName, isAsdPackage, definitionsPath, isSkipImports).getBytes();
213 zip.putNextEntry(new ZipEntry(TOSCA_META_PATH_FILE_NAME));
214 zip.write(toscaBlock0Byte);
215 zip.putNextEntry(new ZipEntry(definitionsPath + fileName));
217 LifecycleStateEnum lifecycleState = component.getLifecycleState();
218 if (addDependencies) {
219 addServiceMf(component, zip, lifecycleState, isInCertificationRequest, fileName, mainYaml, definitionsPath);
220 //US798487 - Abstraction of complex types
221 if (hasToWriteComponentSubstitutionType(component)) {
222 LOGGER.debug("Component {} is complex - generating abstract type for it..", component.getName());
223 dependencies.addAll(writeComponentInterface(component, zip, fileName, definitionsPath));
225 //UID <cassandraId,filename,component>
226 Either<ZipOutputStream, ResponseFormat> zipOutputStreamOrResponseFormat =
227 getZipOutputStreamResponseFormatEither(zip, dependencies, definitionsPath);
228 if (zipOutputStreamOrResponseFormat != null && zipOutputStreamOrResponseFormat.isRight()) {
229 return zipOutputStreamOrResponseFormat;
232 if (component.getModel() == null) {
233 //retrieve SDC.zip from Cassandra
234 Either<byte[], ResponseFormat> latestSchemaFiles = getLatestSchemaFilesFromCassandra();
235 if (latestSchemaFiles.isRight()) {
236 LOGGER.error("Error retrieving SDC Schema files from cassandra");
237 return Either.right(latestSchemaFiles.right().value());
239 final byte[] schemaFileZip = latestSchemaFiles.left().value();
240 final List<String> nodesFromPackage = findNonRootNodesFromPackage(dependencies);
241 //add files from retrieved SDC.zip to Definitions folder in CSAR
242 addSchemaFilesFromCassandra(zip, schemaFileZip, nodesFromPackage, definitionsPath);
244 //retrieve schema files by model from Cassandra
245 addSchemaFilesByModel(zip, component.getModel(), definitionsPath, addDependencies,
246 dependencies.stream().map(d -> d.getRight()).collect(Collectors.toList()));
248 Either<CsarDefinition, ResponseFormat> collectedComponentCsarDefinition = collectComponentCsarDefinition(component);
249 if (collectedComponentCsarDefinition.isRight()) {
250 return Either.right(collectedComponentCsarDefinition.right().value());
252 if (generators != null) {
253 for (CsarEntryGenerator generator : generators) {
254 LOGGER.debug("Invoking CsarEntryGenerator: {}", generator.getClass().getName());
255 for (Map.Entry<String, byte[]> pluginGeneratedFile : generator.generateCsarEntries(component).entrySet()) {
256 zip.putNextEntry(new ZipEntry(pluginGeneratedFile.getKey()));
257 zip.write(pluginGeneratedFile.getValue());
261 return writeAllFilesToCsar(component, collectedComponentCsarDefinition.left().value(), zip, isInCertificationRequest);
264 private String createCsarBlock0(String metaFileVersion, String toscaConformanceLevel) {
265 return String.format(BLOCK_0_TEMPLATE, metaFileVersion, toscaConformanceLevel);
268 private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(Component component, boolean getFromCS,
269 ArtifactDefinition artifactDef, boolean isSkipImports) {
270 LifecycleStateEnum lifecycleState = component.getLifecycleState();
271 boolean shouldBeFetchedFromCassandra =
272 getFromCS || !(lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN || lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT);
273 Either<ToscaRepresentation, ResponseFormat> toscaRepresentation =
274 shouldBeFetchedFromCassandra ? fetchToscaRepresentation(artifactDef) : generateToscaRepresentation(component, isSkipImports);
275 return toscaRepresentation.left()
276 .bind(iff(myd -> !myd.getDependencies().isDefined(), myd -> fetchToscaTemplateDependencies(myd.getMainYaml(), component)));
279 private Either<ToscaRepresentation, ResponseFormat> fetchToscaTemplateDependencies(byte[] mainYml, Component component) {
280 return toscaExportUtils.getDependencies(component).right().map(toscaError -> {
281 LOGGER.debug("Failed to retrieve dependencies for component {}, error {}", component.getUniqueId(), toscaError);
282 return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
283 }).left().map(tt -> ToscaRepresentation.make(mainYml, tt));
286 private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(ArtifactDefinition artifactDef) {
287 return getFromCassandra(artifactDef.getEsId()).right().map(as -> {
288 LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, artifactDef.getArtifactName(), artifactDef.getUniqueId());
289 return componentsUtils.getResponseFormat(as);
290 }).left().map(ToscaRepresentation::make);
293 private Either<byte[], ActionStatus> getFromCassandra(String cassandraId) {
294 return artifactCassandraDao.getArtifact(cassandraId).right().map(operationstatus -> {
295 LOGGER.info("Failed to fetch artifact from Cassandra by id {} error {}.", cassandraId, operationstatus);
296 StorageOperationStatus storageStatus = DaoStatusConverter.convertCassandraStatusToStorageStatus(operationstatus);
297 return componentsUtils.convertFromStorageResponse(storageStatus);
298 }).left().map(DAOArtifactData::getDataAsArray);
301 private static <L, R> F<L, Either<L, R>> iff(Predicate<L> p, Function<L, Either<L, R>> ifTrue) {
302 return l -> p.test(l) ? ifTrue.apply(l) : Either.left(l);
305 private static <A, B> F<A, B> iff(Predicate<A> p, Supplier<B> s, Function<A, B> orElse) {
306 return a -> p.test(a) ? s.get() : orElse.apply(a);
309 private void addServiceMf(Component component, ZipOutputStream zip, LifecycleStateEnum lifecycleState, boolean isInCertificationRequest,
310 String fileName, byte[] mainYaml, String definitionsPath) throws IOException {
312 if ((component.getComponentType() == ComponentTypeEnum.SERVICE) && (lifecycleState != LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT)) {
313 String serviceName = component.getName();
314 String createdBy = component.getCreatorUserId();
315 String serviceVersion;
316 if (isInCertificationRequest) {
317 int tmp = Integer.valueOf(component.getVersion().split("\\.")[0]) + 1;
318 serviceVersion = String.valueOf(tmp) + ".0";
320 serviceVersion = component.getVersion();
322 SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
323 format.setTimeZone(TimeZone.getTimeZone("UTC"));
324 Date date = new Date();
325 String releaseTime = format.format(date);
326 if (component.getCategories() == null || component.getCategories().get(0) == null) {
329 String serviceType = component.getCategories().get(0).getName();
330 String description = component.getDescription();
331 String serviceTemplate = definitionsPath + fileName;
332 String hash = GeneralUtility.calculateMD5Base64EncodedByByteArray(mainYaml);
333 String nsMfBlock0 = createNsMfBlock0(serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate,
335 byte[] nsMfBlock0Byte = nsMfBlock0.getBytes();
336 zip.putNextEntry(new ZipEntry(SERVICE_MANIFEST));
337 zip.write(nsMfBlock0Byte);
341 private String createNsMfBlock0(String serviceName, String createdBy, String serviceVersion, String releaseTime, String serviceType,
342 String description, String serviceTemplate, String hash) {
343 final String block0template = "metadata??\n" + "ns_product_name: %s\n" + "ns_provider_id: %s\n" + "ns_package_version: %s\n" +
344 "ns_release_data_time: %s\n" + "ns_type: %s\n" + "ns_package_description: %s\n\n" + "Source: %s\n" + "Algorithm: MD5\n" + "Hash: %s\n\n";
345 return String.format(block0template, serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate, hash);
348 private boolean hasToWriteComponentSubstitutionType(final Component component) {
349 final Map<String, CategoryBaseTypeConfig> serviceNodeTypesConfig =
350 ConfigurationManager.getConfigurationManager().getConfiguration().getServiceBaseNodeTypes();
351 List<CategoryDefinition> categories = component.getCategories();
352 if (CollectionUtils.isNotEmpty(categories) && MapUtils.isNotEmpty(serviceNodeTypesConfig)
353 && serviceNodeTypesConfig.get(categories.get(0).getName()) != null) {
354 boolean doNotExtendBaseType = serviceNodeTypesConfig.get(categories.get(0).getName()).isDoNotExtendBaseType();
355 if (doNotExtendBaseType) {
359 if (component instanceof Service) {
360 return !ModelConverter.isAtomicComponent(component) && ((Service) component).isSubstituteCandidate();
362 return !ModelConverter.isAtomicComponent(component);
365 private Either<ZipOutputStream, ResponseFormat> writeComponentInterface(Either<ToscaRepresentation, ToscaError> interfaceRepresentation,
366 ZipOutputStream zip, String fileName, String definitionsPath) {
367 // TODO: This should not be done but we need this to keep the refactoring small enough to be easily reviewable
368 return writeComponentInterface(interfaceRepresentation, fileName, ZipWriter.live(zip), definitionsPath)
369 .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)).recover(th -> {
370 LOGGER.error("#writeComponentInterface - zip writing failed with error: ", th);
371 return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
375 private Try<Void> writeComponentInterface(
376 Either<ToscaRepresentation, ToscaError> interfaceRepresentation, String fileName, ZipWriter zw, String definitionsPath) {
377 Either<byte[], ToscaError> yml = interfaceRepresentation.left()
378 .map(ToscaRepresentation::getMainYaml);
379 return fromEither(yml, ToscaErrorException::new).flatMap(zw.write(definitionsPath + ToscaExportHandler.getInterfaceFilename(fileName)));
382 private List<Triple<String, String, Component>> writeComponentInterface(final Component component, final ZipOutputStream zip,
383 final String fileName, final String definitionsPath) {
384 final Either<ToscaRepresentation, ToscaError> interfaceRepresentation = toscaExportUtils.exportComponentInterface(component, false);
385 writeComponentInterface(interfaceRepresentation, zip, fileName, definitionsPath);
386 return interfaceRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
389 private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither(ZipOutputStream zip,
390 List<Triple<String, String, Component>> dependencies,
391 String definitionsPath)
394 innerComponentsCache = ComponentCache.overwritable(overwriteIfSameVersions()).onMerge((oldValue, newValue) ->
395 LOGGER.warn("Overwriting component invariantID {} of version {} with a newer version {}", oldValue.getId(),
396 oldValue.getComponentVersion(),
397 newValue.getComponentVersion()));
398 if (dependencies != null && !dependencies.isEmpty()) {
399 for (Triple<String, String, Component> d : dependencies) {
400 String cassandraId = d.getMiddle();
401 Component childComponent = d.getRight();
402 Either<byte[], ResponseFormat> entryData = getEntryData(cassandraId, childComponent).right().map(componentsUtils::getResponseFormat);
403 if (entryData.isRight()) {
404 return Either.right(entryData.right().value());
406 //fill innerComponentsCache
407 String fileName = d.getLeft();
408 innerComponentsCache.put(cassandraId, fileName, childComponent);
409 addInnerComponentsToCache(innerComponentsCache, childComponent);
411 //add inner components to CSAR
412 return addInnerComponentsToCSAR(zip, innerComponentsCache, definitionsPath);
417 private Either<ZipOutputStream, ResponseFormat> addInnerComponentsToCSAR(ZipOutputStream zip, ComponentCache innerComponentsCache,
418 String definitionsPath)
420 for (ImmutableTriple<String, String, Component> ict : innerComponentsCache.iterable()) {
421 Component innerComponent = ict.getRight();
422 String icFileName = ict.getMiddle();
423 // add component to zip
424 Either<Tuple2<byte[], ZipEntry>, ResponseFormat> zipEntry = toZipEntry(ict, definitionsPath);
425 // TODO: this should not be done, we should instead compose this either further,
427 // but in order to keep this refactoring small, we'll stop here.
428 if (zipEntry.isRight()) {
429 return Either.right(zipEntry.right().value());
431 Tuple2<byte[], ZipEntry> value = zipEntry.left().value();
432 zip.putNextEntry(value._2);
434 // add component interface to zip
435 if (hasToWriteComponentSubstitutionType(innerComponent)) {
436 writeComponentInterface(innerComponent, zip, icFileName, definitionsPath);
442 private Either<Tuple2<byte[], ZipEntry>, ResponseFormat> toZipEntry(ImmutableTriple<String, String, Component> cachedEntry,
443 String definitionsPath) {
444 String cassandraId = cachedEntry.getLeft();
445 String fileName = cachedEntry.getMiddle();
446 Component innerComponent = cachedEntry.getRight();
447 return getEntryData(cassandraId, innerComponent).right().map(status -> {
448 LOGGER.debug("Failed adding to zip component {}, error {}", cassandraId, status);
449 return componentsUtils.getResponseFormat(status);
450 }).left().map(content -> new Tuple2<>(content, new ZipEntry(definitionsPath + fileName)));
453 private void addInnerComponentsToCache(ComponentCache componentCache, Component childComponent) {
454 javaListToVavrList(childComponent.getComponentInstances()).filter(ci -> componentCache.notCached(ci.getComponentUid())).forEach(ci -> {
455 // all resource must be only once!
456 Either<Resource, StorageOperationStatus> resource = toscaOperationFacade.getToscaElement(ci.getComponentUid());
457 Component componentRI = checkAndAddComponent(componentCache, ci, resource);
458 //if not atomic - insert inner components as well
460 // TODO: This could potentially create a StackOverflowException if the call stack
462 // happens to be too large. Tail-recursive optimization should be used here.
463 if (!ModelConverter.isAtomicComponent(componentRI)) {
464 addInnerComponentsToCache(componentCache, componentRI);
469 private Component checkAndAddComponent(ComponentCache componentCache, ComponentInstance ci, Either<Resource, StorageOperationStatus> resource) {
470 if (resource.isRight()) {
471 LOGGER.debug("Failed to fetch resource with id {} for instance {}", ci.getComponentUid(), ci.getName());
473 Component componentRI = resource.left().value();
474 Map<String, ArtifactDefinition> childToscaArtifacts = componentRI.getToscaArtifacts();
475 ArtifactDefinition childArtifactDefinition = childToscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
476 if (childArtifactDefinition != null) {
478 componentCache.put(childArtifactDefinition.getEsId(), childArtifactDefinition.getArtifactName(), componentRI);
483 private Either<byte[], ActionStatus> getEntryData(String cassandraId, Component childComponent) {
484 if (cassandraId == null || cassandraId.isEmpty()) {
485 return toscaExportUtils.exportComponent(childComponent).right().map(toscaErrorToActionStatus(childComponent)).left()
486 .map(ToscaRepresentation::getMainYaml);
488 return getFromCassandra(cassandraId);
492 private F<ToscaError, ActionStatus> toscaErrorToActionStatus(Component childComponent) {
493 return toscaError -> {
494 LOGGER.debug("Failed to export tosca template for child component {} error {}", childComponent.getUniqueId(), toscaError);
495 return componentsUtils.convertFromToscaError(toscaError);
499 private Either<byte[], ResponseFormat> getLatestSchemaFilesFromCassandra() {
500 String fto = versionFirstThreeOctets;
501 return sdcSchemaFilesCassandraDao.getSpecificSchemaFiles(fto, CONFORMANCE_LEVEL).right().map(schemaFilesFetchDBError(fto)).left()
502 .bind(iff(List::isEmpty, () -> schemaFileFetchError(fto), s -> Either.left(s.iterator().next().getPayloadAsArray())));
505 private F<CassandraOperationStatus, ResponseFormat> schemaFilesFetchDBError(String firstThreeOctets) {
507 LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}. Please fix DB table accordingly.", firstThreeOctets,
509 StorageOperationStatus sos = DaoStatusConverter.convertCassandraStatusToStorageStatus(cos);
510 return componentsUtils.getResponseFormat(componentsUtils.convertFromStorageResponse(sos));
514 private Either<byte[], ResponseFormat> schemaFileFetchError(String firstThreeOctets) {
515 LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}", firstThreeOctets, CONFORMANCE_LEVEL);
516 return Either.right(componentsUtils.getResponseFormat(ActionStatus.TOSCA_SCHEMA_FILES_NOT_FOUND, firstThreeOctets, CONFORMANCE_LEVEL));
520 * Create a list of all derived nodes found on the package
522 * @param dependencies all node dependencies
523 * @return a list of nodes
525 private List<String> findNonRootNodesFromPackage(final List<Triple<String, String, Component>> dependencies) {
526 final List<String> nodes = new ArrayList<>();
527 if (CollectionUtils.isNotEmpty(dependencies)) {
528 final String NATIVE_ROOT = "tosca.nodes.Root";
529 dependencies.forEach(dependency -> {
530 if (dependency.getRight() instanceof Resource) {
531 final Resource resource = (Resource) dependency.getRight();
532 if (CollectionUtils.isNotEmpty(resource.getDerivedList())) {
533 resource.getDerivedList().stream().filter(node -> !nodes.contains(node) && !NATIVE_ROOT.equalsIgnoreCase(node))
534 .forEach(node -> nodes.add(node));
543 * Writes to a CSAR zip from casandra schema data
545 * @param zipOutputStream stores the input stream content
546 * @param schemaFileZip zip data from Cassandra
547 * @param nodesFromPackage list of all nodes found on the onboarded package
549 private void addSchemaFilesFromCassandra(final ZipOutputStream zipOutputStream, final byte[] schemaFileZip, final List<String> nodesFromPackage,
550 final String definitionsPath) {
551 final int initSize = 2048;
552 LOGGER.debug("Starting copy from Schema file zip to CSAR zip");
553 try (final ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(schemaFileZip));
554 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
555 final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(byteArrayOutputStream, initSize)) {
557 while ((entry = zipInputStream.getNextEntry()) != null) {
558 ZipUtils.checkForZipSlipInRead(entry);
559 final String entryName = entry.getName();
560 int readSize = initSize;
561 final byte[] entryData = new byte[initSize];
562 if (shouldZipEntryBeHandled(entryName)) {
563 if (NODES_YML.equalsIgnoreCase(entryName)) {
564 handleNode(zipInputStream, byteArrayOutputStream, nodesFromPackage);
566 while ((readSize = zipInputStream.read(entryData, 0, readSize)) != -1) {
567 bufferedOutputStream.write(entryData, 0, readSize);
569 bufferedOutputStream.flush();
571 byteArrayOutputStream.flush();
572 zipOutputStream.putNextEntry(new ZipEntry(definitionsPath + entryName));
573 zipOutputStream.write(byteArrayOutputStream.toByteArray());
574 zipOutputStream.flush();
575 byteArrayOutputStream.reset();
578 } catch (final Exception e) {
579 LOGGER.error("Error while writing the SDC schema file to the CSAR", e);
580 throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
582 LOGGER.debug("Finished copy from Schema file zip to CSAR zip");
586 * Handles the nodes.yml zip entry, updating the nodes.yml to avoid duplicated nodes on it.
588 * @param zipInputStream the zip entry to be read
589 * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
590 * @param nodesFromPackage list of all nodes found on the onboarded package
592 private void handleNode(final ZipInputStream zipInputStream, final ByteArrayOutputStream byteArrayOutputStream,
593 final List<String> nodesFromPackage) throws IOException {
594 final Map<String, Object> nodesFromArtifactFile = readYamlZipEntry(zipInputStream);
595 final Map<String, Object> nodesYaml = updateNodeYml(nodesFromPackage, nodesFromArtifactFile);
596 updateZipEntry(byteArrayOutputStream, nodesYaml);
600 * Updates the zip entry from the given parameters
602 * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
603 * @param nodesYaml a Map of nodes to be written
605 private void updateZipEntry(final ByteArrayOutputStream byteArrayOutputStream, final Map<String, Object> nodesYaml) throws IOException {
606 if (MapUtils.isNotEmpty(nodesYaml)) {
607 byteArrayOutputStream.write(new YamlUtil().objectToYaml(nodesYaml).getBytes());
612 * Filters and removes all duplicated nodes found
614 * @param nodesFromPackage a List of all derived nodes found on the given package
615 * @param nodesFromArtifactFile represents the nodes.yml file stored in Cassandra
616 * @return a nodes Map updated
618 private Map<String, Object> updateNodeYml(final List<String> nodesFromPackage, final Map<String, Object> nodesFromArtifactFile) {
619 if (MapUtils.isNotEmpty(nodesFromArtifactFile)) {
620 final String nodeTypeBlock = TypeUtils.ToscaTagNamesEnum.NODE_TYPES.getElementName();
621 final Map<String, Object> nodeTypes = (Map<String, Object>) nodesFromArtifactFile.get(nodeTypeBlock);
622 nodesFromPackage.stream().filter(nodeTypes::containsKey).forEach(nodeTypes::remove);
623 nodesFromArtifactFile.replace(nodeTypeBlock, nodeTypes);
625 return nodesFromArtifactFile;
629 * Writes a new zip entry
631 * @param zipInputStream the zip entry to be read
632 * @return a map of the given zip entry
634 private Map<String, Object> readYamlZipEntry(final ZipInputStream zipInputStream) throws IOException {
635 final int initSize = 2048;
636 final StringBuilder zipEntry = new StringBuilder();
637 final byte[] buffer = new byte[initSize];
639 while ((read = zipInputStream.read(buffer, 0, initSize)) >= 0) {
640 zipEntry.append(new String(buffer, 0, read));
642 return (Map<String, Object>) new Yaml().load(zipEntry.toString());
646 * Checks if the zip entry should or should not be added to the CSAR based on the given global type list
648 * @param entryName the zip entry name
649 * @return true if the zip entry should be handled
651 private boolean shouldZipEntryBeHandled(final String entryName) {
652 return ConfigurationManager.getConfigurationManager().getConfiguration().getGlobalCsarImports().stream()
653 .anyMatch(entry -> entry.contains(entryName));
656 private void addSchemaFilesByModel(final ZipOutputStream zipOutputStream, final String modelName,
657 final String definitionsPath, final boolean isSingleImportsFile,
658 final List<Component> dependencies) {
660 final Set<Path> writtenEntryPathList = new HashSet<>();
661 final Path defsPath = Path.of(definitionsPath);
662 final Map<Path, byte[]> contentToMerge = new HashMap<>();
663 final List<ToscaImportByModel> modelDefaultImportList = modelOperation.findAllModelImports(modelName, true);
664 for (final ToscaImportByModel toscaImportByModel : modelDefaultImportList) {
665 var importPath = Path.of(toscaImportByModel.getFullPath());
666 if (!isSingleImportsFile) {
667 if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(importPath)).normalize().toString())) {
668 final Path entryPath = defsPath.resolve(importPath);
669 contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
671 if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
672 importPath = ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
674 final Path entryPath = defsPath.resolve(importPath);
675 writtenEntryPathList.add(entryPath);
676 contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
679 if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
680 importPath = ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
682 final Path entryPath = defsPath.resolve(importPath);
683 zipOutputStream.putNextEntry(new ZipEntry(entryPath.toString()));
684 writtenEntryPathList.add(entryPath);
685 final byte[] content = toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8);
686 zipOutputStream.write(content, 0, content.length);
687 zipOutputStream.closeEntry();
690 if (!isSingleImportsFile) {
691 byte[] mergingContent = new byte[0];
692 for (Map.Entry<Path, byte[]> entry : contentToMerge.entrySet()) {
693 if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(entry.getKey())).normalize().toString())) {
694 mergingContent = mergeContent(mergingContent, entry.getValue());
696 final var zipEntry = new ZipEntry(entry.getKey().toString());
697 zipOutputStream.putNextEntry(zipEntry);
698 writtenEntryPathList.add(entry.getKey());
699 mergingContent = mergeContent(mergingContent, entry.getValue());
700 mergingContent = updateMergingContentFromDependencies(mergingContent, dependencies);
701 zipOutputStream.write(mergingContent, 0, mergingContent.length);
702 zipOutputStream.closeEntry();
706 } catch (final IOException e) {
707 LOGGER.error(String.valueOf(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR), CsarUtils.class.getName(),
708 "Error while writing the schema files by model to the CSAR", e);
709 throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.CSAR_TOSCA_IMPORTS_ERROR));
713 private byte[] updateMergingContentFromDependencies(final byte[] mergingContent, final List<Component> dependencies) {
714 final DumperOptions options = new DumperOptions();
715 options.setDefaultFlowStyle(FlowStyle.BLOCK);
716 final Yaml yaml = new Yaml(options);
717 final Map<String, Object> stringObjectMap = (Map<String, Object>) yaml.load(new String(mergingContent));
718 final Map<String, Object> nodeTypes = (Map<String, Object>) stringObjectMap.get("node_types");
719 for (final Component dependency : dependencies) {
720 final Map<String, Object> dependencyAsMap = yaml.load(yaml.dumpAsMap(dependency));
721 final String toscaResourceName = ((ResourceMetadataDataDefinition) dependency.getComponentMetadataDefinition()
722 .getMetadataDataDefinition()).getToscaResourceName();
723 final Map<String, Object> nodeType = (Map<String, Object>) nodeTypes.get(toscaResourceName);
724 final Map<String, Object> propertiesFromDependency = (Map<String, Object>) ((List) dependencyAsMap.get("properties"))
725 .stream().collect(Collectors.toMap(s -> ((Map<String, Object>) s).get("name"), s -> s));
726 if (MapUtils.isNotEmpty(nodeType) && MapUtils.isNotEmpty(propertiesFromDependency)) {
727 final Map<String, Object> propertiesFromMergingContent = (Map<String, Object>) nodeType.get("properties");
728 final Map<String, Object> updatedMap = updatePropertiesFromDependency(propertiesFromMergingContent, propertiesFromDependency);
729 nodeType.replace("properties", updatedMap);
730 nodeTypes.replace(toscaResourceName, nodeType);
733 stringObjectMap.replace("node_types", nodeTypes);
734 return yaml.dumpAsMap(stringObjectMap).getBytes();
737 private Map<String, Object> updatePropertiesFromDependency(final Map<String, Object> propertiesFromMergingContent,
738 final Map<String, Object> propertiesFromDependency) {
739 final Map<String, Object> result = new HashMap<>();
740 for (final Entry<String, Object> entry : propertiesFromDependency.entrySet()) {
741 final Map<String, Object> propertiesMap = new HashMap<>();
742 final String key = entry.getKey();
743 final Object value = entry.getValue();
744 if (propertiesFromMergingContent instanceof Map) {
745 final Object object = propertiesFromMergingContent.get(key);
746 if (object instanceof Map) {
747 ((Map<String, Object>) object).keySet().forEach(s ->
748 propertiesMap.put(s, ((Map<String, Object>) value).get(s))
751 propertiesMap.putAll(createProperties(value));
754 propertiesMap.putAll(createProperties(value));
756 result.put(key, propertiesMap);
761 private Map<String, Object> createProperties(final Object value) {
762 final Map<String, Object> propertiesMap = new HashMap<>();
763 propertiesMap.put("type", ((Map<String, Object>) value).get("type"));
764 propertiesMap.put("required", ((Map<String, Object>) value).get("required"));
765 final Object entrySchema = ((Map<String, Object>) value).get("entry_schema");
766 if (entrySchema != null) {
767 propertiesMap.put("entry_schema", entrySchema);
769 return propertiesMap;
772 private byte[] mergeContent(final byte[] first, final byte[] second) {
773 byte[] merged = new byte[0];
774 final Map<String, Object> firstMap = new Yaml().load(new String(first));
775 final Map<String, Object> secondMap = new Yaml().load(new String(second));
776 if (MapUtils.isNotEmpty(secondMap)) {
777 final DumperOptions options = new DumperOptions();
778 options.setDefaultFlowStyle(FlowStyle.BLOCK);
779 final Yaml yaml = new Yaml(options);
780 for (final Entry<String, Object> secondMapEntry : secondMap.entrySet()) {
781 final Map<String, Object> newMap = new HashMap<>();
782 if (secondMapEntry.getKey().endsWith("_types")) {
783 if (MapUtils.isNotEmpty(firstMap) && firstMap.containsKey(secondMapEntry.getKey())) {
784 final Map<String, Object> secondMapEntryValue = (Map<String, Object>) secondMapEntry.getValue();
785 final Map<String, Object> firstMapValue = (Map<String, Object>) firstMap.get(secondMapEntry.getKey());
786 secondMapEntryValue.putAll(firstMapValue);
787 newMap.put(secondMapEntry.getKey(), secondMapEntryValue);
789 newMap.put(secondMapEntry.getKey(), secondMapEntry.getValue());
792 newMap.put(secondMapEntry.getKey(), secondMapEntry.getValue());
794 merged = Bytes.concat(merged, yaml.dumpAsMap(newMap).getBytes());
800 private Either<CsarDefinition, ResponseFormat> collectComponentCsarDefinition(Component component) {
801 ComponentArtifacts componentArtifacts = new ComponentArtifacts();
802 Component updatedComponent = component;
804 //get service to receive the AII artifacts uploaded to the service
805 if (updatedComponent.getComponentType() == ComponentTypeEnum.SERVICE) {
806 Either<Service, StorageOperationStatus> getServiceResponse = toscaOperationFacade.getToscaElement(updatedComponent.getUniqueId());
808 if (getServiceResponse.isRight()) {
809 ActionStatus actionStatus = componentsUtils.convertFromStorageResponse(getServiceResponse.right().value());
810 return Either.right(componentsUtils.getResponseFormat(actionStatus));
813 updatedComponent = getServiceResponse.left().value();
816 //find the artifacts of the main component, it would have its composed instances artifacts in a separate folder
817 ComponentTypeArtifacts componentInstanceArtifacts = new ComponentTypeArtifacts();
818 ArtifactsInfo artifactsInfo = collectComponentArtifacts(updatedComponent);
819 componentInstanceArtifacts.setComponentArtifacts(artifactsInfo);
820 componentArtifacts.setMainTypeAndCIArtifacts(componentInstanceArtifacts);
822 Map<String, ComponentTypeArtifacts> resourceTypeArtifacts = componentArtifacts
823 .getComponentTypeArtifacts(); //artifacts mapped by the component type(tosca name+version)
824 //get the component instances
825 List<ComponentInstance> componentInstances = updatedComponent.getComponentInstances();
826 if (componentInstances != null) {
827 for (ComponentInstance componentInstance : componentInstances) {
828 //call recursive to find artifacts for all the path
829 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
830 updatedComponent, componentInstance, resourceTypeArtifacts, componentInstanceArtifacts);
831 if (collectComponentInstanceArtifacts.isRight()) {
832 return Either.right(collectComponentInstanceArtifacts.right().value());
837 if (LOGGER.isDebugEnabled()) {
838 printResult(componentArtifacts, updatedComponent.getName());
841 return Either.left(new CsarDefinition(componentArtifacts));
844 private void printResult(ComponentArtifacts componentArtifacts, String name) {
845 StringBuilder result = new StringBuilder();
846 result.append("Artifacts of main component " + name + "\n");
847 ComponentTypeArtifacts componentInstanceArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
848 printArtifacts(componentInstanceArtifacts);
849 result.append("Type Artifacts\n");
850 for (Map.Entry<String, ComponentTypeArtifacts> typeArtifacts : componentArtifacts.getComponentTypeArtifacts().entrySet()) {
851 result.append("Folder " + typeArtifacts.getKey() + "\n");
852 result.append(printArtifacts(typeArtifacts.getValue()));
855 if (LOGGER.isDebugEnabled()) {
856 LOGGER.debug(result.toString());
860 private String printArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
861 StringBuilder result = new StringBuilder();
862 ArtifactsInfo artifactsInfo = componentInstanceArtifacts.getComponentArtifacts();
863 Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentArtifacts = artifactsInfo.getArtifactsInfo();
864 printArtifacts(componentArtifacts);
865 result = result.append("Resources\n");
866 for (Map.Entry<String, ArtifactsInfo> resourceInstance : componentInstanceArtifacts.getComponentInstancesArtifacts().entrySet()) {
867 result.append("Folder" + resourceInstance.getKey() + "\n");
868 result.append(printArtifacts(resourceInstance.getValue().getArtifactsInfo()));
871 return result.toString();
874 private String printArtifacts(Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componetArtifacts) {
875 StringBuilder result = new StringBuilder();
876 for (Map.Entry<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactGroup : componetArtifacts.entrySet()) {
877 result.append(" " + artifactGroup.getKey().getType());
878 for (Map.Entry<String, List<ArtifactDefinition>> groupArtifacts : artifactGroup.getValue().entrySet()) {
879 result.append(" " + groupArtifacts.getKey());
880 for (ArtifactDefinition artifact : groupArtifacts.getValue()) {
881 result.append(" " + artifact.getArtifactDisplayName());
886 return result.toString();
889 private Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts(Component parentComponent, ComponentInstance componentInstance,
890 Map<String, ComponentTypeArtifacts> resourcesTypeArtifacts,
891 ComponentTypeArtifacts instanceArtifactsLocation) {
892 //1. get the component instance component
894 if (componentInstance.getOriginType() == OriginTypeEnum.ServiceProxy) {
895 componentUid = componentInstance.getSourceModelUid();
897 componentUid = componentInstance.getComponentUid();
899 Either<Component, StorageOperationStatus> component = toscaOperationFacade.getToscaElement(componentUid);
900 if (component.isRight()) {
901 LOGGER.error("Failed to fetch resource with id {} for instance {}", componentUid, parentComponent.getUUID());
902 return Either.right(componentsUtils.getResponseFormat(ActionStatus.ASSET_NOT_FOUND_DURING_CSAR_CREATION,
903 parentComponent.getComponentType().getValue(), parentComponent.getUUID(),
904 componentInstance.getOriginType().getComponentType().getValue(), componentUid));
906 Component fetchedComponent = component.left().value();
908 //2. fill the artifacts for the current component parent type
909 String toscaComponentName =
910 componentInstance.getToscaComponentName() + "_v" + componentInstance.getComponentVersion();
912 // if there are no artifacts for this component type we need to fetch and build them
913 ComponentTypeArtifacts componentParentArtifacts = Optional
914 .ofNullable(resourcesTypeArtifacts.get(toscaComponentName))
915 .orElseGet(() -> collectComponentTypeArtifacts(fetchedComponent));
917 if (componentParentArtifacts.getComponentArtifacts().isNotEmpty()) {
918 resourcesTypeArtifacts.put(toscaComponentName, componentParentArtifacts);
921 //3. find the artifacts specific to the instance
922 Map<String, List<ArtifactDefinition>> componentInstanceSpecificInformationalArtifacts =
923 getComponentInstanceSpecificArtifacts(componentInstance.getArtifacts(),
924 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.INFORMATIONAL);
925 Map<String, List<ArtifactDefinition>> componentInstanceSpecificDeploymentArtifacts =
926 getComponentInstanceSpecificArtifacts(componentInstance.getDeploymentArtifacts(),
927 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.DEPLOYMENT);
929 //4. add the instances artifacts to the component type
930 ArtifactsInfo artifactsInfo = new ArtifactsInfo();
931 if (!componentInstanceSpecificInformationalArtifacts.isEmpty()) {
932 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, componentInstanceSpecificInformationalArtifacts);
934 if (!componentInstanceSpecificDeploymentArtifacts.isEmpty()) {
935 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, componentInstanceSpecificDeploymentArtifacts);
937 if (!artifactsInfo.isEmpty()) {
938 instanceArtifactsLocation.addComponentInstancesArtifacts(componentInstance.getNormalizedName(), artifactsInfo);
941 //5. do the same for all the component instances
942 List<ComponentInstance> componentInstances = fetchedComponent.getComponentInstances();
943 if (componentInstances != null) {
944 for (ComponentInstance childComponentInstance : componentInstances) {
945 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
946 fetchedComponent, childComponentInstance, resourcesTypeArtifacts, componentParentArtifacts);
947 if (collectComponentInstanceArtifacts.isRight()) {
948 return collectComponentInstanceArtifacts;
953 return Either.left(true);
956 private Map<String, List<ArtifactDefinition>> getComponentInstanceSpecificArtifacts(Map<String, ArtifactDefinition> componentArtifacts,
957 Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentTypeArtifacts,
958 ArtifactGroupTypeEnum artifactGroupTypeEnum) {
959 Map<String, List<ArtifactDefinition>> parentArtifacts = componentTypeArtifacts
960 .get(artifactGroupTypeEnum); //the artfiacts of the component itself and not the instance
962 Map<String, List<ArtifactDefinition>> artifactsByTypeOfComponentInstance = new HashMap<>();
963 if (componentArtifacts != null) {
964 for (ArtifactDefinition artifact : componentArtifacts.values()) {
965 List<ArtifactDefinition> parentArtifactsByType = null;
966 if (parentArtifacts != null) {
967 parentArtifactsByType = parentArtifacts.get(artifact.getArtifactType());
969 //the artifact is of instance
970 if (parentArtifactsByType == null || !parentArtifactsByType.contains(artifact)) {
971 List<ArtifactDefinition> typeArtifacts = artifactsByTypeOfComponentInstance.get(artifact.getArtifactType());
972 if (typeArtifacts == null) {
973 typeArtifacts = new ArrayList<>();
974 artifactsByTypeOfComponentInstance.put(artifact.getArtifactType(), typeArtifacts);
976 typeArtifacts.add(artifact);
981 return artifactsByTypeOfComponentInstance;
984 private ComponentTypeArtifacts collectComponentTypeArtifacts(Component fetchedComponent) {
985 ArtifactsInfo componentArtifacts = collectComponentArtifacts(fetchedComponent);
986 ComponentTypeArtifacts componentArtifactsInfo = new ComponentTypeArtifacts();
987 if (componentArtifacts.isNotEmpty()) {
988 componentArtifactsInfo.setComponentArtifacts(componentArtifacts);
990 return componentArtifactsInfo;
993 private ArtifactsInfo collectComponentArtifacts(Component component) {
994 Map<String, ArtifactDefinition> informationalArtifacts = component.getArtifacts();
995 Map<String, List<ArtifactDefinition>> informationalArtifactsByType = collectGroupArtifacts(informationalArtifacts);
996 Map<String, ArtifactDefinition> deploymentArtifacts = component.getDeploymentArtifacts();
997 Map<String, List<ArtifactDefinition>> deploymentArtifactsByType = collectGroupArtifacts(deploymentArtifacts);
998 ArtifactsInfo artifactsInfo = new ArtifactsInfo();
999 if (!informationalArtifactsByType.isEmpty()) {
1000 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, informationalArtifactsByType);
1002 if (!deploymentArtifactsByType.isEmpty()) {
1003 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, deploymentArtifactsByType);
1006 return artifactsInfo;
1009 private Map<String, List<ArtifactDefinition>> collectGroupArtifacts(
1010 final Map<String, ArtifactDefinition> componentArtifacts) {
1011 final Map<String, List<ArtifactDefinition>> artifactsByType = new HashMap<>();
1012 for (final ArtifactDefinition artifact : componentArtifacts.values()) {
1013 if (artifact.getArtifactUUID() != null) {
1014 artifactsByType.putIfAbsent(artifact.getArtifactType(), new ArrayList<>());
1015 final List<ArtifactDefinition> typeArtifacts = artifactsByType.get(artifact.getArtifactType());
1016 typeArtifacts.add(artifact);
1019 return artifactsByType;
1022 private Either<ZipOutputStream, ResponseFormat> writeAllFilesToCsar(Component mainComponent, CsarDefinition csarDefinition,
1023 ZipOutputStream zipstream, boolean isInCertificationRequest)
1024 throws IOException {
1025 ComponentArtifacts componentArtifacts = csarDefinition.getComponentArtifacts();
1026 Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath = writeComponentArtifactsToSpecifiedPath(mainComponent,
1027 componentArtifacts, zipstream, ARTIFACTS_PATH, isInCertificationRequest);
1028 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1029 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1031 ComponentTypeArtifacts mainTypeAndCIArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
1032 writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, mainTypeAndCIArtifacts.getComponentArtifacts(),
1033 zipstream, ARTIFACTS_PATH, isInCertificationRequest);
1034 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1035 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1037 Map<String, ArtifactsInfo> componentInstancesArtifacts = mainTypeAndCIArtifacts.getComponentInstancesArtifacts();
1038 String currentPath = ARTIFACTS_PATH + RESOURCES_PATH;
1039 for (String keyAssetName : componentInstancesArtifacts.keySet()) {
1040 ArtifactsInfo artifactsInfo = componentInstancesArtifacts.get(keyAssetName);
1041 String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
1042 writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, artifactsInfo, zipstream, pathWithAssetName,
1043 isInCertificationRequest);
1044 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1045 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1048 writeComponentArtifactsToSpecifiedPath = writeOperationsArtifactsToCsar(mainComponent, zipstream);
1049 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1050 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1052 return Either.left(zipstream);
1055 private Either<ZipOutputStream, ResponseFormat> writeOperationsArtifactsToCsar(Component component, ZipOutputStream zipstream) {
1056 if (checkComponentBeforeOperation(component)) {
1057 return Either.left(zipstream);
1059 for (Map.Entry<String, InterfaceDefinition> interfaceEntry : ((Resource) component).getInterfaces().entrySet()) {
1060 for (OperationDataDefinition operation : interfaceEntry.getValue().getOperations().values()) {
1062 if (checkComponentBeforeWrite(component, interfaceEntry, operation)) {
1065 final String artifactUUID = operation.getImplementation().getArtifactUUID();
1066 if (artifactUUID == null) {
1069 final Either<byte[], ActionStatus> artifactFromCassandra = getFromCassandra(artifactUUID);
1070 final String artifactName = operation.getImplementation().getArtifactName();
1071 if (artifactFromCassandra.isRight()) {
1072 LOGGER.error(ARTIFACT_NAME_UNIQUE_ID, artifactName, artifactUUID);
1073 LOGGER.error("Failed to get {} payload from DB reason: {}", artifactName, artifactFromCassandra.right().value());
1074 return Either.right(componentsUtils.getResponseFormat(
1075 ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION, "Resource", component.getUniqueId(), artifactName, artifactUUID));
1077 zipstream.putNextEntry(new ZipEntry(OperationArtifactUtil.createOperationArtifactPath(component, null, operation, true)));
1078 zipstream.write(artifactFromCassandra.left().value());
1079 } catch (IOException e) {
1080 LOGGER.error("Component Name {}, Interface Name {}, Operation Name {}", component.getNormalizedName(), interfaceEntry.getKey(),
1081 operation.getName());
1082 LOGGER.error("Error while writing the operation's artifacts to the CSAR", e);
1083 return Either.right(componentsUtils.getResponseFormat(ERROR_DURING_CSAR_CREATION, "Resource", component.getUniqueId()));
1087 return Either.left(zipstream);
1090 private boolean checkComponentBeforeWrite(Component component, Map.Entry<String, InterfaceDefinition> interfaceEntry,
1091 OperationDataDefinition operation) {
1092 final ArtifactDataDefinition implementation = operation.getImplementation();
1093 if (implementation == null) {
1094 LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no Operation Implementation found", component.getNormalizedName(),
1095 interfaceEntry.getValue().getUniqueId(), operation.getName());
1098 final String artifactName = implementation.getArtifactName();
1099 if (artifactName == null) {
1100 LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no artifact found", component.getNormalizedName(),
1101 interfaceEntry.getValue().getUniqueId(), operation.getName());
1104 if (OperationArtifactUtil.artifactNameIsALiteralValue(artifactName)) {
1105 LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - artifact name is a literal value rather than an SDC artifact",
1106 component.getNormalizedName(), interfaceEntry.getValue().getUniqueId(), operation.getName());
1112 private boolean checkComponentBeforeOperation(Component component) {
1113 if (component instanceof Service) {
1116 if (Objects.isNull(((Resource) component).getInterfaces())) {
1117 LOGGER.debug("Component Name {}- no interfaces found", component.getNormalizedName());
1123 private Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath(final Component mainComponent,
1124 final ArtifactsInfo currArtifactsInfo,
1125 final ZipOutputStream zip, final String path,
1126 final boolean isInCertificationRequest) throws IOException {
1127 final Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfo = currArtifactsInfo.getArtifactsInfo();
1128 for (final ArtifactGroupTypeEnum artifactGroupTypeEnum : artifactsInfo.keySet()) {
1129 final String groupTypeFolder = path + WordUtils.capitalizeFully(artifactGroupTypeEnum.getType()) + PATH_DELIMITER;
1130 final Map<String, List<ArtifactDefinition>> artifactTypesMap = artifactsInfo.get(artifactGroupTypeEnum);
1131 for (final String artifactType : artifactTypesMap.keySet()) {
1132 final List<ArtifactDefinition> artifactDefinitionList = artifactTypesMap.get(artifactType);
1133 String artifactTypeFolder = groupTypeFolder + artifactType + PATH_DELIMITER;
1134 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType) && path.contains(ARTIFACTS_PATH + RESOURCES_PATH)) {
1135 // Ignore this packaging as BPMN artifacts needs to be packaged in different manner
1138 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType)) {
1139 artifactTypeFolder += OperationArtifactUtil.BPMN_ARTIFACT_PATH + File.separator;
1140 } else if (ArtifactTypeEnum.ONBOARDED_PACKAGE.getType().equals(artifactType)) {
1141 // renaming legacy folder ONBOARDED_PACKAGE to the new folder ETSI_PACKAGE
1142 artifactTypeFolder = artifactTypeFolder
1143 .replace(ArtifactTypeEnum.ONBOARDED_PACKAGE.getType(), ArtifactTypeEnum.ETSI_PACKAGE.getType());
1145 // TODO: We should not do this but in order to keep this refactoring small enough,
1147 // we'll leave this as is for now
1148 List<ArtifactDefinition> collect = filterArtifactDefinitionToZip(mainComponent, artifactDefinitionList, isInCertificationRequest)
1149 .collect(Collectors.toList());
1150 for (ArtifactDefinition ad : collect) {
1151 zip.putNextEntry(new ZipEntry(artifactTypeFolder + ad.getArtifactName()));
1152 zip.write(ad.getPayloadData());
1156 return Either.left(zip);
1159 private Stream<ArtifactDefinition> filterArtifactDefinitionToZip(Component mainComponent, List<ArtifactDefinition> artifactDefinitionList,
1160 boolean isInCertificationRequest) {
1161 return artifactDefinitionList.stream().filter(shouldBeInZip(isInCertificationRequest, mainComponent)).map(this::fetchPayLoadData)
1162 .filter(Either::isLeft).map(e -> e.left().value());
1165 private Predicate<ArtifactDefinition> shouldBeInZip(boolean isInCertificationRequest, Component component) {
1166 return artifactDefinition -> !(!isInCertificationRequest && component.isService() && artifactDefinition.isHeatEnvType() || artifactDefinition
1167 .hasNoMandatoryEsId());
1170 private Either<ArtifactDefinition, ActionStatus> fetchPayLoadData(ArtifactDefinition ad) {
1171 byte[] payloadData = ad.getPayloadData();
1172 if (payloadData == null) {
1173 return getFromCassandra(ad.getEsId()).left().map(pd -> {
1176 }).right().map(as -> {
1177 LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, ad.getArtifactName(), ad.getUniqueId());
1178 LOGGER.debug("Failed to get {} payload from DB reason: {}", ad.getArtifactName(), as);
1182 return Either.left(ad);
1186 private Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath(Component mainComponent,
1187 ComponentArtifacts componentArtifacts,
1188 ZipOutputStream zipstream, String currentPath,
1189 boolean isInCertificationRequest) throws IOException {
1190 Map<String, ComponentTypeArtifacts> componentTypeArtifacts = componentArtifacts.getComponentTypeArtifacts();
1193 //<Inner Asset TOSCA name (e.g. VFC name)> folder name: <Inner Asset TOSCA name (e.g. VFC name)>_v<version>.
1195 //E.g. "org.openecomp.resource.vf.vipr_atm_v1.0"
1196 Set<String> componentTypeArtifactsKeys = componentTypeArtifacts.keySet();
1197 for (String keyAssetName : componentTypeArtifactsKeys) {
1198 ComponentTypeArtifacts componentInstanceArtifacts = componentTypeArtifacts.get(keyAssetName);
1199 ArtifactsInfo componentArtifacts2 = componentInstanceArtifacts.getComponentArtifacts();
1200 String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
1201 Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent,
1202 componentArtifacts2, zipstream, pathWithAssetName, isInCertificationRequest);
1203 if (writeArtifactsInfoToSpecifiedPath.isRight()) {
1204 return writeArtifactsInfoToSpecifiedPath;
1207 return Either.left(zipstream);
1210 private Either<ToscaRepresentation, ResponseFormat> generateToscaRepresentation(Component component, boolean isSkipImports) {
1211 return toscaExportUtils.exportComponent(component, isSkipImports).right().map(toscaError -> {
1212 LOGGER.debug("exportComponent failed {}", toscaError);
1213 return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
1217 private String createToscaBlock0(String metaFileVersion, String csarVersion, String createdBy, String entryDef, boolean isAsdPackage,
1218 String definitionsPath, boolean isSkipImports) {
1219 final String block0template = "TOSCA-Meta-File-Version: %s\nCSAR-Version: %s\nCreated-By: %s\nEntry-Definitions: "
1220 + definitionsPath + "%s\n%s\n" + (!isSkipImports ? "Name: csar.meta\nContent-Type: text/plain\n" : "");
1221 return String.format(block0template, metaFileVersion, csarVersion, createdBy, entryDef, isAsdPackage ? "entry_definition_type: asd" : "");
1224 private class CsarDefinition {
1226 private ComponentArtifacts componentArtifacts;
1228 // add list of tosca artifacts and meta describes CSAR zip root
1229 public CsarDefinition(ComponentArtifacts componentArtifacts) {
1230 this.componentArtifacts = componentArtifacts;
1233 public ComponentArtifacts getComponentArtifacts() {
1234 return componentArtifacts;
1238 private class ComponentArtifacts {
1240 //artifacts of the component and CI's artifacts contained in it's composition (represents Informational, Deployment & Resource folders of main component)
1241 private ComponentTypeArtifacts mainTypeAndCIArtifacts;
1242 //artifacts of all component types mapped by their tosca name
1243 private Map<String, ComponentTypeArtifacts> componentTypeArtifacts;
1245 public ComponentArtifacts() {
1246 mainTypeAndCIArtifacts = new ComponentTypeArtifacts();
1247 componentTypeArtifacts = new HashMap<>();
1250 public ComponentTypeArtifacts getMainTypeAndCIArtifacts() {
1251 return mainTypeAndCIArtifacts;
1254 public void setMainTypeAndCIArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
1255 this.mainTypeAndCIArtifacts = componentInstanceArtifacts;
1258 public Map<String, ComponentTypeArtifacts> getComponentTypeArtifacts() {
1259 return componentTypeArtifacts;
1264 * The artifacts of the component and of all its composed instances
1266 private class ComponentTypeArtifacts {
1268 private ArtifactsInfo componentArtifacts; //component artifacts (describes the Informational Deployment folders)
1270 private Map<String, ArtifactsInfo> componentInstancesArtifacts; //artifacts of the composed instances mapped by the resourceInstance normalized name (describes the Resources folder)
1272 public ComponentTypeArtifacts() {
1273 componentArtifacts = new ArtifactsInfo();
1274 componentInstancesArtifacts = new HashMap<>();
1277 public ArtifactsInfo getComponentArtifacts() {
1278 return componentArtifacts;
1281 public void setComponentArtifacts(ArtifactsInfo artifactsInfo) {
1282 this.componentArtifacts = artifactsInfo;
1285 public Map<String, ArtifactsInfo> getComponentInstancesArtifacts() {
1286 return componentInstancesArtifacts;
1289 public void addComponentInstancesArtifacts(String normalizedName, ArtifactsInfo artifactsInfo) {
1290 componentInstancesArtifacts.put(normalizedName, artifactsInfo);
1295 * The artifacts Definition saved by their structure
1297 private class ArtifactsInfo {
1298 //Key is the type of artifacts(Informational/Deployment)
1300 //Value is a map between an artifact type and a list of all artifacts of this type
1301 private Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfoField;
1303 public ArtifactsInfo() {
1304 this.artifactsInfoField = new EnumMap<>(ArtifactGroupTypeEnum.class);
1307 public Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> getArtifactsInfo() {
1308 return artifactsInfoField;
1311 public void addArtifactsToGroup(ArtifactGroupTypeEnum artifactGroup, Map<String, List<ArtifactDefinition>> artifactsDefinition) {
1312 if (artifactsInfoField.get(artifactGroup) == null) {
1313 artifactsInfoField.put(artifactGroup, artifactsDefinition);
1315 Map<String, List<ArtifactDefinition>> artifactTypeEnumListMap = artifactsInfoField.get(artifactGroup);
1316 artifactTypeEnumListMap.putAll(artifactsDefinition);
1317 artifactsInfoField.put(artifactGroup, artifactTypeEnumListMap);
1321 public boolean isEmpty() {
1322 return artifactsInfoField.isEmpty();
1325 public boolean isNotEmpty() {
1330 public static class ToscaErrorException extends Exception {
1332 ToscaErrorException(ToscaError error) {
1333 super("Error while exporting component's interface (toscaError:" + error + ")");