2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2023 Nordix Foundation. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
20 package org.openecomp.sdc.be.tosca;
22 import static org.openecomp.sdc.be.dao.api.ActionStatus.ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION;
23 import static org.openecomp.sdc.be.dao.api.ActionStatus.ERROR_DURING_CSAR_CREATION;
24 import static org.openecomp.sdc.be.tosca.ComponentCache.MergeStrategy.overwriteIfSameVersions;
25 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.fromEither;
26 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.javaListToVavrList;
27 import static org.openecomp.sdc.common.api.Constants.ADDITIONAL_TYPE_DEFINITIONS;
29 import com.google.common.primitives.Bytes;
31 import fj.data.Either;
32 import io.vavr.Tuple2;
33 import io.vavr.control.Try;
34 import java.io.BufferedOutputStream;
35 import java.io.ByteArrayInputStream;
37 import java.io.IOException;
38 import java.nio.charset.StandardCharsets;
39 import java.nio.file.Path;
40 import java.nio.file.Paths;
41 import java.text.SimpleDateFormat;
42 import java.util.ArrayList;
43 import java.util.Collections;
44 import java.util.Date;
45 import java.util.EnumMap;
46 import java.util.HashMap;
47 import java.util.HashSet;
48 import java.util.List;
50 import java.util.Map.Entry;
51 import java.util.Objects;
52 import java.util.Optional;
54 import java.util.TimeZone;
55 import java.util.function.Function;
56 import java.util.function.Predicate;
57 import java.util.function.Supplier;
58 import java.util.regex.Matcher;
59 import java.util.regex.Pattern;
60 import java.util.stream.Collectors;
61 import java.util.stream.Stream;
62 import java.util.zip.ZipEntry;
63 import java.util.zip.ZipInputStream;
64 import java.util.zip.ZipOutputStream;
65 import org.apache.commons.collections.CollectionUtils;
66 import org.apache.commons.collections.MapUtils;
67 import org.apache.commons.io.output.ByteArrayOutputStream;
68 import org.apache.commons.lang.StringUtils;
69 import org.apache.commons.lang3.tuple.ImmutableTriple;
70 import org.apache.commons.lang3.tuple.Triple;
71 import org.apache.commons.text.WordUtils;
72 import org.onap.sdc.tosca.services.YamlUtil;
73 import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException;
74 import org.openecomp.sdc.be.config.CategoryBaseTypeConfig;
75 import org.openecomp.sdc.be.config.ConfigurationManager;
76 import org.openecomp.sdc.be.dao.api.ActionStatus;
77 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
78 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
79 import org.openecomp.sdc.be.dao.cassandra.SdcSchemaFilesCassandraDao;
80 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
81 import org.openecomp.sdc.be.datatypes.components.ResourceMetadataDataDefinition;
82 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
83 import org.openecomp.sdc.be.datatypes.elements.OperationDataDefinition;
84 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
85 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
86 import org.openecomp.sdc.be.impl.ComponentsUtils;
87 import org.openecomp.sdc.be.model.ArtifactDefinition;
88 import org.openecomp.sdc.be.model.Component;
89 import org.openecomp.sdc.be.model.ComponentInstance;
90 import org.openecomp.sdc.be.model.InterfaceDefinition;
91 import org.openecomp.sdc.be.model.LifecycleStateEnum;
92 import org.openecomp.sdc.be.model.Resource;
93 import org.openecomp.sdc.be.model.Service;
94 import org.openecomp.sdc.be.model.category.CategoryDefinition;
95 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
96 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
97 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
98 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
99 import org.openecomp.sdc.be.model.operations.impl.ModelOperation;
100 import org.openecomp.sdc.be.plugins.CsarEntryGenerator;
101 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
102 import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil;
103 import org.openecomp.sdc.be.utils.TypeUtils;
104 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
105 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
106 import org.openecomp.sdc.common.impl.ExternalConfiguration;
107 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
108 import org.openecomp.sdc.common.util.GeneralUtility;
109 import org.openecomp.sdc.common.zip.ZipUtils;
110 import org.openecomp.sdc.exception.ResponseFormat;
111 import org.slf4j.Logger;
112 import org.slf4j.LoggerFactory;
113 import org.springframework.beans.factory.annotation.Autowired;
114 import org.yaml.snakeyaml.DumperOptions;
115 import org.yaml.snakeyaml.DumperOptions.FlowStyle;
116 import org.yaml.snakeyaml.Yaml;
119 * Generates a Network Service CSAR based on a SERVICE component and wraps it in a SDC CSAR entry.
121 @org.springframework.stereotype.Component("commonCsarGenerator")
122 public class CommonCsarGenerator {
124 private static final Logger LOGGER = LoggerFactory.getLogger(CommonCsarGenerator.class);
125 public static final String ARTIFACTS_PATH = "Artifacts/";
126 private static final String RESOURCES_PATH = "Resources/";
127 private static final String PATH_DELIMITER = "/";
128 private static final String SERVICE_MANIFEST = "NS.mf";
129 private static final String ARTIFACT_NAME_UNIQUE_ID = "ArtifactName {}, unique ID {}";
130 private static final String TOSCA_META_PATH_FILE_NAME = "TOSCA-Metadata/TOSCA.meta";
131 private static final String TOSCA_META_VERSION = "1.0";
132 private static final String CSAR_VERSION = "1.1";
133 private static final String BLOCK_0_TEMPLATE = "SDC-TOSCA-Meta-File-Version: %s\nSDC-TOSCA-Definitions-Version: %s\n";
134 private static final String CSAR_META_PATH_FILE_NAME = "csar.meta";
135 private static final String SDC_VERSION = ExternalConfiguration.getAppVersion();
136 public static final String NODES_YML = "nodes.yml";
137 private static final String CONFORMANCE_LEVEL = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
138 private final ToscaOperationFacade toscaOperationFacade;
139 private final ComponentsUtils componentsUtils;
140 private final ToscaExportHandler toscaExportUtils;
141 private final List<CsarEntryGenerator> generators;
142 private final ArtifactCassandraDao artifactCassandraDao;
143 private final String versionFirstThreeOctets;
144 private final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
145 private final ModelOperation modelOperation;
148 public CommonCsarGenerator(
149 final ToscaOperationFacade toscaOperationFacade,
150 final ComponentsUtils componentsUtils,
151 final ToscaExportHandler toscaExportUtils,
152 final List<CsarEntryGenerator> generators,
153 final ArtifactCassandraDao artifactCassandraDao,
154 final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao,
155 final ModelOperation modelOperation) {
156 this.toscaOperationFacade = toscaOperationFacade;
157 this.componentsUtils = componentsUtils;
158 this.toscaExportUtils = toscaExportUtils;
159 this.generators = generators;
160 this.artifactCassandraDao = artifactCassandraDao;
161 this.versionFirstThreeOctets = readVersionFirstThreeOctets();
162 this.sdcSchemaFilesCassandraDao = sdcSchemaFilesCassandraDao;
163 this.modelOperation = modelOperation;
166 private String readVersionFirstThreeOctets() {
167 if (StringUtils.isEmpty(SDC_VERSION)) {
170 // change regex to avoid DoS sonar issue
171 Matcher matcher = Pattern.compile("(?!\\.)(\\d{1,9}(\\.\\d{1,9}){1,9})(?![\\d\\.])").matcher(SDC_VERSION);
173 return matcher.group(0);
177 * Generates a Network Service CSAR based on a SERVICE component that has category configured in
178 * CategoriesToGenerateNsd enum and wraps it in a SDC CSAR entry.
180 * @param component the component to create the NS CSAR from
181 * @return an entry to be added in the Component CSAR by SDC
183 public Either<ZipOutputStream, ResponseFormat> generateCsarZip(Component component,
186 boolean isInCertificationRequest,
187 boolean isAsdPackage,
188 String definitionsPath,
189 boolean addDependencies,
190 boolean isSkipImports) throws IOException {
191 ArtifactDefinition artifactDef = component.getToscaArtifacts().get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
192 Either<ToscaRepresentation, ResponseFormat> toscaRepresentation = fetchToscaRepresentation(component, getFromCS, artifactDef, isSkipImports);
194 // This should not be done but in order to keep the refactoring small enough we stop here.
195 // TODO: Refactor the rest of this function
197 List<Triple<String, String, Component>> dependencies;
198 if (toscaRepresentation.isLeft()) {
199 mainYaml = toscaRepresentation.left().value().getMainYaml();
200 dependencies = toscaRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
202 return Either.right(toscaRepresentation.right().value());
205 if (!isSkipImports) {
206 final String toscaConformanceLevel = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
207 zip.putNextEntry(new ZipEntry(CSAR_META_PATH_FILE_NAME));
208 zip.write(createCsarBlock0(TOSCA_META_VERSION, toscaConformanceLevel).getBytes());
211 final String fileName = artifactDef.getArtifactName();
212 final byte[] toscaBlock0Byte = createToscaBlock0(
213 TOSCA_META_VERSION, CSAR_VERSION, component.getCreatorFullName(), fileName, isAsdPackage, definitionsPath, isSkipImports).getBytes();
214 zip.putNextEntry(new ZipEntry(TOSCA_META_PATH_FILE_NAME));
215 zip.write(toscaBlock0Byte);
216 zip.putNextEntry(new ZipEntry(definitionsPath + fileName));
218 LifecycleStateEnum lifecycleState = component.getLifecycleState();
219 if (addDependencies) {
220 addServiceMf(component, zip, lifecycleState, isInCertificationRequest, fileName, mainYaml, definitionsPath);
221 //US798487 - Abstraction of complex types
222 if (hasToWriteComponentSubstitutionType(component)) {
223 LOGGER.debug("Component {} is complex - generating abstract type for it..", component.getName());
224 dependencies.addAll(writeComponentInterface(component, zip, fileName, definitionsPath));
226 //UID <cassandraId,filename,component>
227 Either<ZipOutputStream, ResponseFormat> zipOutputStreamOrResponseFormat =
228 getZipOutputStreamResponseFormatEither(zip, dependencies, definitionsPath);
229 if (zipOutputStreamOrResponseFormat != null && zipOutputStreamOrResponseFormat.isRight()) {
230 return zipOutputStreamOrResponseFormat;
233 if (component.getModel() == null) {
234 //retrieve SDC.zip from Cassandra
235 Either<byte[], ResponseFormat> latestSchemaFiles = getLatestSchemaFilesFromCassandra();
236 if (latestSchemaFiles.isRight()) {
237 LOGGER.error("Error retrieving SDC Schema files from cassandra");
238 return Either.right(latestSchemaFiles.right().value());
240 final byte[] schemaFileZip = latestSchemaFiles.left().value();
241 final List<String> nodesFromPackage = findNonRootNodesFromPackage(dependencies);
242 //add files from retrieved SDC.zip to Definitions folder in CSAR
243 addSchemaFilesFromCassandra(zip, schemaFileZip, nodesFromPackage, definitionsPath);
245 //retrieve schema files by model from Cassandra
246 addSchemaFilesByModel(zip, component.getModel(), definitionsPath, addDependencies,
247 dependencies.stream().map(d -> d.getRight()).collect(Collectors.toList()));
249 Either<CsarDefinition, ResponseFormat> collectedComponentCsarDefinition = collectComponentCsarDefinition(component);
250 if (collectedComponentCsarDefinition.isRight()) {
251 return Either.right(collectedComponentCsarDefinition.right().value());
253 if (generators != null) {
254 for (CsarEntryGenerator generator : generators) {
255 LOGGER.debug("Invoking CsarEntryGenerator: {}", generator.getClass().getName());
256 for (Map.Entry<String, byte[]> pluginGeneratedFile : generator.generateCsarEntries(component).entrySet()) {
257 zip.putNextEntry(new ZipEntry(pluginGeneratedFile.getKey()));
258 zip.write(pluginGeneratedFile.getValue());
262 return writeAllFilesToCsar(component, collectedComponentCsarDefinition.left().value(), zip, isInCertificationRequest);
265 private String createCsarBlock0(String metaFileVersion, String toscaConformanceLevel) {
266 return String.format(BLOCK_0_TEMPLATE, metaFileVersion, toscaConformanceLevel);
269 private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(Component component, boolean getFromCS,
270 ArtifactDefinition artifactDef, boolean isSkipImports) {
271 LifecycleStateEnum lifecycleState = component.getLifecycleState();
272 boolean shouldBeFetchedFromCassandra =
273 getFromCS || !(lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN || lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT);
274 Either<ToscaRepresentation, ResponseFormat> toscaRepresentation =
275 shouldBeFetchedFromCassandra ? fetchToscaRepresentation(artifactDef) : generateToscaRepresentation(component, isSkipImports);
276 return toscaRepresentation.left()
277 .bind(iff(myd -> !myd.getDependencies().isDefined(), myd -> fetchToscaTemplateDependencies(myd.getMainYaml(), component)));
280 private Either<ToscaRepresentation, ResponseFormat> fetchToscaTemplateDependencies(byte[] mainYml, Component component) {
281 return toscaExportUtils.getDependencies(component).right().map(toscaError -> {
282 LOGGER.debug("Failed to retrieve dependencies for component {}, error {}", component.getUniqueId(), toscaError);
283 return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
284 }).left().map(tt -> ToscaRepresentation.make(mainYml, tt));
287 private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(ArtifactDefinition artifactDef) {
288 return getFromCassandra(artifactDef.getEsId()).right().map(as -> {
289 LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, artifactDef.getArtifactName(), artifactDef.getUniqueId());
290 return componentsUtils.getResponseFormat(as);
291 }).left().map(ToscaRepresentation::make);
294 private Either<byte[], ActionStatus> getFromCassandra(String cassandraId) {
295 return artifactCassandraDao.getArtifact(cassandraId).right().map(operationstatus -> {
296 LOGGER.info("Failed to fetch artifact from Cassandra by id {} error {}.", cassandraId, operationstatus);
297 StorageOperationStatus storageStatus = DaoStatusConverter.convertCassandraStatusToStorageStatus(operationstatus);
298 return componentsUtils.convertFromStorageResponse(storageStatus);
299 }).left().map(DAOArtifactData::getDataAsArray);
302 private static <L, R> F<L, Either<L, R>> iff(Predicate<L> p, Function<L, Either<L, R>> ifTrue) {
303 return l -> p.test(l) ? ifTrue.apply(l) : Either.left(l);
306 private static <A, B> F<A, B> iff(Predicate<A> p, Supplier<B> s, Function<A, B> orElse) {
307 return a -> p.test(a) ? s.get() : orElse.apply(a);
310 private void addServiceMf(Component component, ZipOutputStream zip, LifecycleStateEnum lifecycleState, boolean isInCertificationRequest,
311 String fileName, byte[] mainYaml, String definitionsPath) throws IOException {
313 if ((component.getComponentType() == ComponentTypeEnum.SERVICE) && (lifecycleState != LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT)) {
314 String serviceName = component.getName();
315 String createdBy = component.getCreatorUserId();
316 String serviceVersion;
317 if (isInCertificationRequest) {
318 int tmp = Integer.valueOf(component.getVersion().split("\\.")[0]) + 1;
319 serviceVersion = String.valueOf(tmp) + ".0";
321 serviceVersion = component.getVersion();
323 SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
324 format.setTimeZone(TimeZone.getTimeZone("UTC"));
325 Date date = new Date();
326 String releaseTime = format.format(date);
327 if (component.getCategories() == null || component.getCategories().get(0) == null) {
330 String serviceType = component.getCategories().get(0).getName();
331 String description = component.getDescription();
332 String serviceTemplate = definitionsPath + fileName;
333 String hash = GeneralUtility.calculateMD5Base64EncodedByByteArray(mainYaml);
334 String nsMfBlock0 = createNsMfBlock0(serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate,
336 byte[] nsMfBlock0Byte = nsMfBlock0.getBytes();
337 zip.putNextEntry(new ZipEntry(SERVICE_MANIFEST));
338 zip.write(nsMfBlock0Byte);
342 private String createNsMfBlock0(String serviceName, String createdBy, String serviceVersion, String releaseTime, String serviceType,
343 String description, String serviceTemplate, String hash) {
344 final String block0template = "metadata??\n" + "ns_product_name: %s\n" + "ns_provider_id: %s\n" + "ns_package_version: %s\n" +
345 "ns_release_data_time: %s\n" + "ns_type: %s\n" + "ns_package_description: %s\n\n" + "Source: %s\n" + "Algorithm: MD5\n" + "Hash: %s\n\n";
346 return String.format(block0template, serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate, hash);
349 private boolean hasToWriteComponentSubstitutionType(final Component component) {
350 final Map<String, CategoryBaseTypeConfig> serviceNodeTypesConfig =
351 ConfigurationManager.getConfigurationManager().getConfiguration().getServiceBaseNodeTypes();
352 List<CategoryDefinition> categories = component.getCategories();
353 if (CollectionUtils.isNotEmpty(categories) && MapUtils.isNotEmpty(serviceNodeTypesConfig)
354 && serviceNodeTypesConfig.get(categories.get(0).getName()) != null) {
355 boolean doNotExtendBaseType = serviceNodeTypesConfig.get(categories.get(0).getName()).isDoNotExtendBaseType();
356 if (doNotExtendBaseType) {
360 if (component instanceof Service) {
361 return !ModelConverter.isAtomicComponent(component) && ((Service) component).isSubstituteCandidate();
363 return !ModelConverter.isAtomicComponent(component);
366 private Either<ZipOutputStream, ResponseFormat> writeComponentInterface(Either<ToscaRepresentation, ToscaError> interfaceRepresentation,
367 ZipOutputStream zip, String fileName, String definitionsPath) {
368 // TODO: This should not be done but we need this to keep the refactoring small enough to be easily reviewable
369 return writeComponentInterface(interfaceRepresentation, fileName, ZipWriter.live(zip), definitionsPath)
370 .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)).recover(th -> {
371 LOGGER.error("#writeComponentInterface - zip writing failed with error: ", th);
372 return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
376 private Try<Void> writeComponentInterface(
377 Either<ToscaRepresentation, ToscaError> interfaceRepresentation, String fileName, ZipWriter zw, String definitionsPath) {
378 Either<byte[], ToscaError> yml = interfaceRepresentation.left()
379 .map(ToscaRepresentation::getMainYaml);
380 return fromEither(yml, ToscaErrorException::new).flatMap(zw.write(definitionsPath + ToscaExportHandler.getInterfaceFilename(fileName)));
383 private List<Triple<String, String, Component>> writeComponentInterface(final Component component, final ZipOutputStream zip,
384 final String fileName, final String definitionsPath) {
385 final Either<ToscaRepresentation, ToscaError> interfaceRepresentation = toscaExportUtils.exportComponentInterface(component, false);
386 writeComponentInterface(interfaceRepresentation, zip, fileName, definitionsPath);
387 return interfaceRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
390 private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither(ZipOutputStream zip,
391 List<Triple<String, String, Component>> dependencies,
392 String definitionsPath)
395 innerComponentsCache = ComponentCache.overwritable(overwriteIfSameVersions()).onMerge((oldValue, newValue) ->
396 LOGGER.warn("Overwriting component invariantID {} of version {} with a newer version {}", oldValue.getId(),
397 oldValue.getComponentVersion(),
398 newValue.getComponentVersion()));
399 if (dependencies != null && !dependencies.isEmpty()) {
400 for (Triple<String, String, Component> d : dependencies) {
401 String cassandraId = d.getMiddle();
402 Component childComponent = d.getRight();
403 Either<byte[], ResponseFormat> entryData = getEntryData(cassandraId, childComponent).right().map(componentsUtils::getResponseFormat);
404 if (entryData.isRight()) {
405 return Either.right(entryData.right().value());
407 //fill innerComponentsCache
408 String fileName = d.getLeft();
409 innerComponentsCache.put(cassandraId, fileName, childComponent);
410 addInnerComponentsToCache(innerComponentsCache, childComponent);
412 //add inner components to CSAR
413 return addInnerComponentsToCSAR(zip, innerComponentsCache, definitionsPath);
418 private Either<ZipOutputStream, ResponseFormat> addInnerComponentsToCSAR(ZipOutputStream zip, ComponentCache innerComponentsCache,
419 String definitionsPath)
421 for (ImmutableTriple<String, String, Component> ict : innerComponentsCache.iterable()) {
422 Component innerComponent = ict.getRight();
423 String icFileName = ict.getMiddle();
424 // add component to zip
425 Either<Tuple2<byte[], ZipEntry>, ResponseFormat> zipEntry = toZipEntry(ict, definitionsPath);
426 // TODO: this should not be done, we should instead compose this either further,
428 // but in order to keep this refactoring small, we'll stop here.
429 if (zipEntry.isRight()) {
430 return Either.right(zipEntry.right().value());
432 Tuple2<byte[], ZipEntry> value = zipEntry.left().value();
433 zip.putNextEntry(value._2);
435 // add component interface to zip
436 if (hasToWriteComponentSubstitutionType(innerComponent)) {
437 writeComponentInterface(innerComponent, zip, icFileName, definitionsPath);
443 private Either<Tuple2<byte[], ZipEntry>, ResponseFormat> toZipEntry(ImmutableTriple<String, String, Component> cachedEntry,
444 String definitionsPath) {
445 String cassandraId = cachedEntry.getLeft();
446 String fileName = cachedEntry.getMiddle();
447 Component innerComponent = cachedEntry.getRight();
448 return getEntryData(cassandraId, innerComponent).right().map(status -> {
449 LOGGER.debug("Failed adding to zip component {}, error {}", cassandraId, status);
450 return componentsUtils.getResponseFormat(status);
451 }).left().map(content -> new Tuple2<>(content, new ZipEntry(definitionsPath + fileName)));
454 private void addInnerComponentsToCache(ComponentCache componentCache, Component childComponent) {
455 javaListToVavrList(childComponent.getComponentInstances()).filter(ci -> componentCache.notCached(ci.getComponentUid())).forEach(ci -> {
456 // all resource must be only once!
457 Either<Resource, StorageOperationStatus> resource = toscaOperationFacade.getToscaElement(ci.getComponentUid());
458 Component componentRI = checkAndAddComponent(componentCache, ci, resource);
459 //if not atomic - insert inner components as well
461 // TODO: This could potentially create a StackOverflowException if the call stack
463 // happens to be too large. Tail-recursive optimization should be used here.
464 if (!ModelConverter.isAtomicComponent(componentRI)) {
465 addInnerComponentsToCache(componentCache, componentRI);
470 private Component checkAndAddComponent(ComponentCache componentCache, ComponentInstance ci, Either<Resource, StorageOperationStatus> resource) {
471 if (resource.isRight()) {
472 LOGGER.debug("Failed to fetch resource with id {} for instance {}", ci.getComponentUid(), ci.getName());
474 Component componentRI = resource.left().value();
475 Map<String, ArtifactDefinition> childToscaArtifacts = componentRI.getToscaArtifacts();
476 ArtifactDefinition childArtifactDefinition = childToscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
477 if (childArtifactDefinition != null) {
479 componentCache.put(childArtifactDefinition.getEsId(), childArtifactDefinition.getArtifactName(), componentRI);
484 private Either<byte[], ActionStatus> getEntryData(String cassandraId, Component childComponent) {
485 if (cassandraId == null || cassandraId.isEmpty()) {
486 return toscaExportUtils.exportComponent(childComponent).right().map(toscaErrorToActionStatus(childComponent)).left()
487 .map(ToscaRepresentation::getMainYaml);
489 return getFromCassandra(cassandraId);
493 private F<ToscaError, ActionStatus> toscaErrorToActionStatus(Component childComponent) {
494 return toscaError -> {
495 LOGGER.debug("Failed to export tosca template for child component {} error {}", childComponent.getUniqueId(), toscaError);
496 return componentsUtils.convertFromToscaError(toscaError);
500 private Either<byte[], ResponseFormat> getLatestSchemaFilesFromCassandra() {
501 String fto = versionFirstThreeOctets;
502 return sdcSchemaFilesCassandraDao.getSpecificSchemaFiles(fto, CONFORMANCE_LEVEL).right().map(schemaFilesFetchDBError(fto)).left()
503 .bind(iff(List::isEmpty, () -> schemaFileFetchError(fto), s -> Either.left(s.iterator().next().getPayloadAsArray())));
506 private F<CassandraOperationStatus, ResponseFormat> schemaFilesFetchDBError(String firstThreeOctets) {
508 LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}. Please fix DB table accordingly.", firstThreeOctets,
510 StorageOperationStatus sos = DaoStatusConverter.convertCassandraStatusToStorageStatus(cos);
511 return componentsUtils.getResponseFormat(componentsUtils.convertFromStorageResponse(sos));
515 private Either<byte[], ResponseFormat> schemaFileFetchError(String firstThreeOctets) {
516 LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}", firstThreeOctets, CONFORMANCE_LEVEL);
517 return Either.right(componentsUtils.getResponseFormat(ActionStatus.TOSCA_SCHEMA_FILES_NOT_FOUND, firstThreeOctets, CONFORMANCE_LEVEL));
521 * Create a list of all derived nodes found on the package
523 * @param dependencies all node dependencies
524 * @return a list of nodes
526 private List<String> findNonRootNodesFromPackage(final List<Triple<String, String, Component>> dependencies) {
527 final List<String> nodes = new ArrayList<>();
528 if (CollectionUtils.isNotEmpty(dependencies)) {
529 final String NATIVE_ROOT = "tosca.nodes.Root";
530 dependencies.forEach(dependency -> {
531 if (dependency.getRight() instanceof Resource) {
532 final Resource resource = (Resource) dependency.getRight();
533 if (CollectionUtils.isNotEmpty(resource.getDerivedList())) {
534 resource.getDerivedList().stream().filter(node -> !nodes.contains(node) && !NATIVE_ROOT.equalsIgnoreCase(node))
535 .forEach(node -> nodes.add(node));
544 * Writes to a CSAR zip from casandra schema data
546 * @param zipOutputStream stores the input stream content
547 * @param schemaFileZip zip data from Cassandra
548 * @param nodesFromPackage list of all nodes found on the onboarded package
550 private void addSchemaFilesFromCassandra(final ZipOutputStream zipOutputStream, final byte[] schemaFileZip, final List<String> nodesFromPackage,
551 final String definitionsPath) {
552 final int initSize = 2048;
553 LOGGER.debug("Starting copy from Schema file zip to CSAR zip");
554 try (final ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(schemaFileZip));
555 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
556 final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(byteArrayOutputStream, initSize)) {
558 while ((entry = zipInputStream.getNextEntry()) != null) {
559 ZipUtils.checkForZipSlipInRead(entry);
560 final String entryName = entry.getName();
561 int readSize = initSize;
562 final byte[] entryData = new byte[initSize];
563 if (shouldZipEntryBeHandled(entryName)) {
564 if (NODES_YML.equalsIgnoreCase(entryName)) {
565 handleNode(zipInputStream, byteArrayOutputStream, nodesFromPackage);
567 while ((readSize = zipInputStream.read(entryData, 0, readSize)) != -1) {
568 bufferedOutputStream.write(entryData, 0, readSize);
570 bufferedOutputStream.flush();
572 byteArrayOutputStream.flush();
573 zipOutputStream.putNextEntry(new ZipEntry(definitionsPath + entryName));
574 zipOutputStream.write(byteArrayOutputStream.toByteArray());
575 zipOutputStream.flush();
576 byteArrayOutputStream.reset();
579 } catch (final Exception e) {
580 LOGGER.error("Error while writing the SDC schema file to the CSAR", e);
581 throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
583 LOGGER.debug("Finished copy from Schema file zip to CSAR zip");
587 * Handles the nodes.yml zip entry, updating the nodes.yml to avoid duplicated nodes on it.
589 * @param zipInputStream the zip entry to be read
590 * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
591 * @param nodesFromPackage list of all nodes found on the onboarded package
593 private void handleNode(final ZipInputStream zipInputStream, final ByteArrayOutputStream byteArrayOutputStream,
594 final List<String> nodesFromPackage) throws IOException {
595 final Map<String, Object> nodesFromArtifactFile = readYamlZipEntry(zipInputStream);
596 final Map<String, Object> nodesYaml = updateNodeYml(nodesFromPackage, nodesFromArtifactFile);
597 updateZipEntry(byteArrayOutputStream, nodesYaml);
601 * Updates the zip entry from the given parameters
603 * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
604 * @param nodesYaml a Map of nodes to be written
606 private void updateZipEntry(final ByteArrayOutputStream byteArrayOutputStream, final Map<String, Object> nodesYaml) throws IOException {
607 if (MapUtils.isNotEmpty(nodesYaml)) {
608 byteArrayOutputStream.write(new YamlUtil().objectToYaml(nodesYaml).getBytes());
613 * Filters and removes all duplicated nodes found
615 * @param nodesFromPackage a List of all derived nodes found on the given package
616 * @param nodesFromArtifactFile represents the nodes.yml file stored in Cassandra
617 * @return a nodes Map updated
619 private Map<String, Object> updateNodeYml(final List<String> nodesFromPackage, final Map<String, Object> nodesFromArtifactFile) {
620 if (MapUtils.isNotEmpty(nodesFromArtifactFile)) {
621 final String nodeTypeBlock = TypeUtils.ToscaTagNamesEnum.NODE_TYPES.getElementName();
622 final Map<String, Object> nodeTypes = (Map<String, Object>) nodesFromArtifactFile.get(nodeTypeBlock);
623 nodesFromPackage.stream().filter(nodeTypes::containsKey).forEach(nodeTypes::remove);
624 nodesFromArtifactFile.replace(nodeTypeBlock, nodeTypes);
626 return nodesFromArtifactFile;
630 * Writes a new zip entry
632 * @param zipInputStream the zip entry to be read
633 * @return a map of the given zip entry
635 private Map<String, Object> readYamlZipEntry(final ZipInputStream zipInputStream) throws IOException {
636 final int initSize = 2048;
637 final StringBuilder zipEntry = new StringBuilder();
638 final byte[] buffer = new byte[initSize];
640 while ((read = zipInputStream.read(buffer, 0, initSize)) >= 0) {
641 zipEntry.append(new String(buffer, 0, read));
643 return (Map<String, Object>) new Yaml().load(zipEntry.toString());
647 * Checks if the zip entry should or should not be added to the CSAR based on the given global type list
649 * @param entryName the zip entry name
650 * @return true if the zip entry should be handled
652 private boolean shouldZipEntryBeHandled(final String entryName) {
653 return ConfigurationManager.getConfigurationManager().getConfiguration().getGlobalCsarImports().stream()
654 .anyMatch(entry -> entry.contains(entryName));
657 private void addSchemaFilesByModel(final ZipOutputStream zipOutputStream, final String modelName,
658 final String definitionsPath, final boolean isSingleImportsFile,
659 final List<Component> dependencies) {
661 final Set<Path> writtenEntryPathList = new HashSet<>();
662 final Path defsPath = Path.of(definitionsPath);
663 final Map<Path, byte[]> contentToMerge = new HashMap<>();
664 final List<ToscaImportByModel> modelDefaultImportList = modelOperation.findAllModelImports(modelName, true);
665 for (final ToscaImportByModel toscaImportByModel : modelDefaultImportList) {
666 var importPath = Path.of(toscaImportByModel.getFullPath());
667 if (!isSingleImportsFile) {
668 if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(importPath)).normalize().toString())) {
669 final Path entryPath = defsPath.resolve(importPath);
670 contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
672 if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
673 importPath = ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
675 final Path entryPath = defsPath.resolve(importPath);
676 writtenEntryPathList.add(entryPath);
677 contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
680 if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
681 importPath = ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
683 final Path entryPath = defsPath.resolve(importPath);
684 zipOutputStream.putNextEntry(new ZipEntry(entryPath.toString()));
685 writtenEntryPathList.add(entryPath);
686 final byte[] content = toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8);
687 zipOutputStream.write(content, 0, content.length);
688 zipOutputStream.closeEntry();
691 if (!isSingleImportsFile) {
692 byte[] mergingContent = new byte[0];
693 for (Map.Entry<Path, byte[]> entry : contentToMerge.entrySet()) {
694 if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(entry.getKey())).normalize().toString())) {
695 mergingContent = mergeContent(mergingContent, entry.getValue());
697 final var zipEntry = new ZipEntry(entry.getKey().toString());
698 zipOutputStream.putNextEntry(zipEntry);
699 writtenEntryPathList.add(entry.getKey());
700 mergingContent = mergeContent(mergingContent, entry.getValue());
701 mergingContent = updateMergingContentFromDependencies(mergingContent, dependencies);
702 zipOutputStream.write(mergingContent, 0, mergingContent.length);
703 zipOutputStream.closeEntry();
707 } catch (final IOException e) {
708 LOGGER.error(String.valueOf(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR), CsarUtils.class.getName(),
709 "Error while writing the schema files by model to the CSAR", e);
710 throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.CSAR_TOSCA_IMPORTS_ERROR));
714 private byte[] updateMergingContentFromDependencies(final byte[] mergingContent, final List<Component> dependencies) {
715 final DumperOptions options = new DumperOptions();
716 options.setDefaultFlowStyle(FlowStyle.BLOCK);
717 final Yaml yaml = new Yaml(options);
718 final Map<String, Object> stringObjectMap = (Map<String, Object>) yaml.load(new String(mergingContent));
719 final Map<String, Object> nodeTypes = (Map<String, Object>) stringObjectMap.get("node_types");
720 for (final Component dependency : dependencies) {
721 final Map<String, Object> dependencyAsMap = yaml.load(yaml.dumpAsMap(dependency));
722 final String toscaResourceName = ((ResourceMetadataDataDefinition) dependency.getComponentMetadataDefinition()
723 .getMetadataDataDefinition()).getToscaResourceName();
724 final Map<String, Object> nodeType = (Map<String, Object>) nodeTypes.get(toscaResourceName);
725 final Map<String, Object> propertiesFromDependency = (Map<String, Object>) ((List) dependencyAsMap.get("properties"))
726 .stream().collect(Collectors.toMap(s -> ((Map<String, Object>) s).get("name"), s -> s));
727 if (MapUtils.isNotEmpty(nodeType) && MapUtils.isNotEmpty(propertiesFromDependency)) {
728 final Map<String, Object> propertiesFromMergingContent = (Map<String, Object>) nodeType.get("properties");
729 final Map<String, Object> updatedMap = updatePropertiesFromDependency(propertiesFromMergingContent, propertiesFromDependency);
730 nodeType.replace("properties", updatedMap);
731 nodeTypes.replace(toscaResourceName, nodeType);
734 stringObjectMap.replace("node_types", nodeTypes);
735 return yaml.dumpAsMap(stringObjectMap).getBytes();
738 private Map<String, Object> updatePropertiesFromDependency(final Map<String, Object> propertiesFromMergingContent,
739 final Map<String, Object> propertiesFromDependency) {
740 final Map<String, Object> result = new HashMap<>();
741 for (final Entry<String, Object> entry : propertiesFromDependency.entrySet()) {
742 final Map<String, Object> propertiesMap = new HashMap<>();
743 final String key = entry.getKey();
744 final Object value = entry.getValue();
745 if (propertiesFromMergingContent instanceof Map) {
746 final Object object = propertiesFromMergingContent.get(key);
747 if (object instanceof Map) {
748 ((Map<String, Object>) object).keySet().forEach(s ->
749 propertiesMap.put(s, getValue(s, (Map<String, Object>) value))
752 propertiesMap.putAll(createProperties(value));
755 propertiesMap.putAll(createProperties(value));
757 result.put(key, propertiesMap);
762 private Object getValue(final String key, Map<String, Object> value) {
763 final String mappedKey = mapKey(key);
764 if (mappedKey.equals("schemaType")) {
765 return Collections.singletonMap("type", value.get(mappedKey));
767 return value.get(mappedKey);
770 private String mapKey(final String key) {
771 if (key.equals("entry_schema")) {
774 if (key.equals("default")) {
775 return "defaultValue";
780 private Map<String, Object> createProperties(final Object value) {
781 final Map<String, Object> propertiesMap = new HashMap<>();
782 propertiesMap.put("type", ((Map<String, Object>) value).get("type"));
783 propertiesMap.put("required", ((Map<String, Object>) value).get("required"));
784 final Object entrySchema = ((Map<String, Object>) value).get("entry_schema");
785 if (entrySchema != null) {
786 propertiesMap.put("entry_schema", entrySchema);
788 return propertiesMap;
791 private byte[] mergeContent(final byte[] first, final byte[] second) {
792 byte[] merged = new byte[0];
793 final Map<String, Object> firstMap = new Yaml().load(new String(first));
794 final Map<String, Object> secondMap = new Yaml().load(new String(second));
795 if (MapUtils.isNotEmpty(secondMap)) {
796 final DumperOptions options = new DumperOptions();
797 options.setDefaultFlowStyle(FlowStyle.BLOCK);
798 final Yaml yaml = new Yaml(options);
799 for (final Entry<String, Object> secondMapEntry : secondMap.entrySet()) {
800 final Map<String, Object> newMap = new HashMap<>();
801 if (secondMapEntry.getKey().endsWith("_types")) {
802 if (MapUtils.isNotEmpty(firstMap) && firstMap.containsKey(secondMapEntry.getKey())) {
803 final Map<String, Object> secondMapEntryValue = (Map<String, Object>) secondMapEntry.getValue();
804 final Map<String, Object> firstMapValue = (Map<String, Object>) firstMap.get(secondMapEntry.getKey());
805 secondMapEntryValue.putAll(firstMapValue);
806 newMap.put(secondMapEntry.getKey(), secondMapEntryValue);
808 newMap.put(secondMapEntry.getKey(), secondMapEntry.getValue());
811 newMap.put(secondMapEntry.getKey(), secondMapEntry.getValue());
813 merged = Bytes.concat(merged, yaml.dumpAsMap(newMap).getBytes());
819 private Either<CsarDefinition, ResponseFormat> collectComponentCsarDefinition(Component component) {
820 ComponentArtifacts componentArtifacts = new ComponentArtifacts();
821 Component updatedComponent = component;
823 //get service to receive the AII artifacts uploaded to the service
824 if (updatedComponent.getComponentType() == ComponentTypeEnum.SERVICE) {
825 Either<Service, StorageOperationStatus> getServiceResponse = toscaOperationFacade.getToscaElement(updatedComponent.getUniqueId());
827 if (getServiceResponse.isRight()) {
828 ActionStatus actionStatus = componentsUtils.convertFromStorageResponse(getServiceResponse.right().value());
829 return Either.right(componentsUtils.getResponseFormat(actionStatus));
832 updatedComponent = getServiceResponse.left().value();
835 //find the artifacts of the main component, it would have its composed instances artifacts in a separate folder
836 ComponentTypeArtifacts componentInstanceArtifacts = new ComponentTypeArtifacts();
837 ArtifactsInfo artifactsInfo = collectComponentArtifacts(updatedComponent);
838 componentInstanceArtifacts.setComponentArtifacts(artifactsInfo);
839 componentArtifacts.setMainTypeAndCIArtifacts(componentInstanceArtifacts);
841 Map<String, ComponentTypeArtifacts> resourceTypeArtifacts = componentArtifacts
842 .getComponentTypeArtifacts(); //artifacts mapped by the component type(tosca name+version)
843 //get the component instances
844 List<ComponentInstance> componentInstances = updatedComponent.getComponentInstances();
845 if (componentInstances != null) {
846 for (ComponentInstance componentInstance : componentInstances) {
847 //call recursive to find artifacts for all the path
848 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
849 updatedComponent, componentInstance, resourceTypeArtifacts, componentInstanceArtifacts);
850 if (collectComponentInstanceArtifacts.isRight()) {
851 return Either.right(collectComponentInstanceArtifacts.right().value());
856 if (LOGGER.isDebugEnabled()) {
857 printResult(componentArtifacts, updatedComponent.getName());
860 return Either.left(new CsarDefinition(componentArtifacts));
863 private void printResult(ComponentArtifacts componentArtifacts, String name) {
864 StringBuilder result = new StringBuilder();
865 result.append("Artifacts of main component " + name + "\n");
866 ComponentTypeArtifacts componentInstanceArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
867 printArtifacts(componentInstanceArtifacts);
868 result.append("Type Artifacts\n");
869 for (Map.Entry<String, ComponentTypeArtifacts> typeArtifacts : componentArtifacts.getComponentTypeArtifacts().entrySet()) {
870 result.append("Folder " + typeArtifacts.getKey() + "\n");
871 result.append(printArtifacts(typeArtifacts.getValue()));
874 if (LOGGER.isDebugEnabled()) {
875 LOGGER.debug(result.toString());
879 private String printArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
880 StringBuilder result = new StringBuilder();
881 ArtifactsInfo artifactsInfo = componentInstanceArtifacts.getComponentArtifacts();
882 Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentArtifacts = artifactsInfo.getArtifactsInfo();
883 printArtifacts(componentArtifacts);
884 result = result.append("Resources\n");
885 for (Map.Entry<String, ArtifactsInfo> resourceInstance : componentInstanceArtifacts.getComponentInstancesArtifacts().entrySet()) {
886 result.append("Folder" + resourceInstance.getKey() + "\n");
887 result.append(printArtifacts(resourceInstance.getValue().getArtifactsInfo()));
890 return result.toString();
893 private String printArtifacts(Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componetArtifacts) {
894 StringBuilder result = new StringBuilder();
895 for (Map.Entry<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactGroup : componetArtifacts.entrySet()) {
896 result.append(" " + artifactGroup.getKey().getType());
897 for (Map.Entry<String, List<ArtifactDefinition>> groupArtifacts : artifactGroup.getValue().entrySet()) {
898 result.append(" " + groupArtifacts.getKey());
899 for (ArtifactDefinition artifact : groupArtifacts.getValue()) {
900 result.append(" " + artifact.getArtifactDisplayName());
905 return result.toString();
908 private Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts(Component parentComponent, ComponentInstance componentInstance,
909 Map<String, ComponentTypeArtifacts> resourcesTypeArtifacts,
910 ComponentTypeArtifacts instanceArtifactsLocation) {
911 //1. get the component instance component
913 if (componentInstance.getOriginType() == OriginTypeEnum.ServiceProxy) {
914 componentUid = componentInstance.getSourceModelUid();
916 componentUid = componentInstance.getComponentUid();
918 Either<Component, StorageOperationStatus> component = toscaOperationFacade.getToscaElement(componentUid);
919 if (component.isRight()) {
920 LOGGER.error("Failed to fetch resource with id {} for instance {}", componentUid, parentComponent.getUUID());
921 return Either.right(componentsUtils.getResponseFormat(ActionStatus.ASSET_NOT_FOUND_DURING_CSAR_CREATION,
922 parentComponent.getComponentType().getValue(), parentComponent.getUUID(),
923 componentInstance.getOriginType().getComponentType().getValue(), componentUid));
925 Component fetchedComponent = component.left().value();
927 //2. fill the artifacts for the current component parent type
928 String toscaComponentName =
929 componentInstance.getToscaComponentName() + "_v" + componentInstance.getComponentVersion();
931 // if there are no artifacts for this component type we need to fetch and build them
932 ComponentTypeArtifacts componentParentArtifacts = Optional
933 .ofNullable(resourcesTypeArtifacts.get(toscaComponentName))
934 .orElseGet(() -> collectComponentTypeArtifacts(fetchedComponent));
936 if (componentParentArtifacts.getComponentArtifacts().isNotEmpty()) {
937 resourcesTypeArtifacts.put(toscaComponentName, componentParentArtifacts);
940 //3. find the artifacts specific to the instance
941 Map<String, List<ArtifactDefinition>> componentInstanceSpecificInformationalArtifacts =
942 getComponentInstanceSpecificArtifacts(componentInstance.getArtifacts(),
943 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.INFORMATIONAL);
944 Map<String, List<ArtifactDefinition>> componentInstanceSpecificDeploymentArtifacts =
945 getComponentInstanceSpecificArtifacts(componentInstance.getDeploymentArtifacts(),
946 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.DEPLOYMENT);
948 //4. add the instances artifacts to the component type
949 ArtifactsInfo artifactsInfo = new ArtifactsInfo();
950 if (!componentInstanceSpecificInformationalArtifacts.isEmpty()) {
951 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, componentInstanceSpecificInformationalArtifacts);
953 if (!componentInstanceSpecificDeploymentArtifacts.isEmpty()) {
954 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, componentInstanceSpecificDeploymentArtifacts);
956 if (!artifactsInfo.isEmpty()) {
957 instanceArtifactsLocation.addComponentInstancesArtifacts(componentInstance.getNormalizedName(), artifactsInfo);
960 //5. do the same for all the component instances
961 List<ComponentInstance> componentInstances = fetchedComponent.getComponentInstances();
962 if (componentInstances != null) {
963 for (ComponentInstance childComponentInstance : componentInstances) {
964 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
965 fetchedComponent, childComponentInstance, resourcesTypeArtifacts, componentParentArtifacts);
966 if (collectComponentInstanceArtifacts.isRight()) {
967 return collectComponentInstanceArtifacts;
972 return Either.left(true);
975 private Map<String, List<ArtifactDefinition>> getComponentInstanceSpecificArtifacts(Map<String, ArtifactDefinition> componentArtifacts,
976 Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentTypeArtifacts,
977 ArtifactGroupTypeEnum artifactGroupTypeEnum) {
978 Map<String, List<ArtifactDefinition>> parentArtifacts = componentTypeArtifacts
979 .get(artifactGroupTypeEnum); //the artfiacts of the component itself and not the instance
981 Map<String, List<ArtifactDefinition>> artifactsByTypeOfComponentInstance = new HashMap<>();
982 if (componentArtifacts != null) {
983 for (ArtifactDefinition artifact : componentArtifacts.values()) {
984 List<ArtifactDefinition> parentArtifactsByType = null;
985 if (parentArtifacts != null) {
986 parentArtifactsByType = parentArtifacts.get(artifact.getArtifactType());
988 //the artifact is of instance
989 if (parentArtifactsByType == null || !parentArtifactsByType.contains(artifact)) {
990 List<ArtifactDefinition> typeArtifacts = artifactsByTypeOfComponentInstance.get(artifact.getArtifactType());
991 if (typeArtifacts == null) {
992 typeArtifacts = new ArrayList<>();
993 artifactsByTypeOfComponentInstance.put(artifact.getArtifactType(), typeArtifacts);
995 typeArtifacts.add(artifact);
1000 return artifactsByTypeOfComponentInstance;
1003 private ComponentTypeArtifacts collectComponentTypeArtifacts(Component fetchedComponent) {
1004 ArtifactsInfo componentArtifacts = collectComponentArtifacts(fetchedComponent);
1005 ComponentTypeArtifacts componentArtifactsInfo = new ComponentTypeArtifacts();
1006 if (componentArtifacts.isNotEmpty()) {
1007 componentArtifactsInfo.setComponentArtifacts(componentArtifacts);
1009 return componentArtifactsInfo;
1012 private ArtifactsInfo collectComponentArtifacts(Component component) {
1013 Map<String, ArtifactDefinition> informationalArtifacts = component.getArtifacts();
1014 Map<String, List<ArtifactDefinition>> informationalArtifactsByType = collectGroupArtifacts(informationalArtifacts);
1015 Map<String, ArtifactDefinition> deploymentArtifacts = component.getDeploymentArtifacts();
1016 Map<String, List<ArtifactDefinition>> deploymentArtifactsByType = collectGroupArtifacts(deploymentArtifacts);
1017 ArtifactsInfo artifactsInfo = new ArtifactsInfo();
1018 if (!informationalArtifactsByType.isEmpty()) {
1019 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, informationalArtifactsByType);
1021 if (!deploymentArtifactsByType.isEmpty()) {
1022 artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, deploymentArtifactsByType);
1025 return artifactsInfo;
1028 private Map<String, List<ArtifactDefinition>> collectGroupArtifacts(
1029 final Map<String, ArtifactDefinition> componentArtifacts) {
1030 final Map<String, List<ArtifactDefinition>> artifactsByType = new HashMap<>();
1031 for (final ArtifactDefinition artifact : componentArtifacts.values()) {
1032 if (artifact.getArtifactUUID() != null) {
1033 artifactsByType.putIfAbsent(artifact.getArtifactType(), new ArrayList<>());
1034 final List<ArtifactDefinition> typeArtifacts = artifactsByType.get(artifact.getArtifactType());
1035 typeArtifacts.add(artifact);
1038 return artifactsByType;
1041 private Either<ZipOutputStream, ResponseFormat> writeAllFilesToCsar(Component mainComponent, CsarDefinition csarDefinition,
1042 ZipOutputStream zipstream, boolean isInCertificationRequest)
1043 throws IOException {
1044 ComponentArtifacts componentArtifacts = csarDefinition.getComponentArtifacts();
1045 Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath = writeComponentArtifactsToSpecifiedPath(mainComponent,
1046 componentArtifacts, zipstream, ARTIFACTS_PATH, isInCertificationRequest);
1047 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1048 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1050 ComponentTypeArtifacts mainTypeAndCIArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
1051 writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, mainTypeAndCIArtifacts.getComponentArtifacts(),
1052 zipstream, ARTIFACTS_PATH, isInCertificationRequest);
1053 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1054 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1056 Map<String, ArtifactsInfo> componentInstancesArtifacts = mainTypeAndCIArtifacts.getComponentInstancesArtifacts();
1057 String currentPath = ARTIFACTS_PATH + RESOURCES_PATH;
1058 for (String keyAssetName : componentInstancesArtifacts.keySet()) {
1059 ArtifactsInfo artifactsInfo = componentInstancesArtifacts.get(keyAssetName);
1060 String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
1061 writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, artifactsInfo, zipstream, pathWithAssetName,
1062 isInCertificationRequest);
1063 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1064 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1067 writeComponentArtifactsToSpecifiedPath = writeOperationsArtifactsToCsar(mainComponent, zipstream);
1068 if (writeComponentArtifactsToSpecifiedPath.isRight()) {
1069 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
1071 return Either.left(zipstream);
1074 private Either<ZipOutputStream, ResponseFormat> writeOperationsArtifactsToCsar(Component component, ZipOutputStream zipstream) {
1075 if (checkComponentBeforeOperation(component)) {
1076 return Either.left(zipstream);
1078 for (Map.Entry<String, InterfaceDefinition> interfaceEntry : ((Resource) component).getInterfaces().entrySet()) {
1079 for (OperationDataDefinition operation : interfaceEntry.getValue().getOperations().values()) {
1081 if (checkComponentBeforeWrite(component, interfaceEntry, operation)) {
1084 final String artifactUUID = operation.getImplementation().getArtifactUUID();
1085 if (artifactUUID == null) {
1088 final Either<byte[], ActionStatus> artifactFromCassandra = getFromCassandra(artifactUUID);
1089 final String artifactName = operation.getImplementation().getArtifactName();
1090 if (artifactFromCassandra.isRight()) {
1091 LOGGER.error(ARTIFACT_NAME_UNIQUE_ID, artifactName, artifactUUID);
1092 LOGGER.error("Failed to get {} payload from DB reason: {}", artifactName, artifactFromCassandra.right().value());
1093 return Either.right(componentsUtils.getResponseFormat(
1094 ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION, "Resource", component.getUniqueId(), artifactName, artifactUUID));
1096 zipstream.putNextEntry(new ZipEntry(OperationArtifactUtil.createOperationArtifactPath(component, null, operation, true)));
1097 zipstream.write(artifactFromCassandra.left().value());
1098 } catch (IOException e) {
1099 LOGGER.error("Component Name {}, Interface Name {}, Operation Name {}", component.getNormalizedName(), interfaceEntry.getKey(),
1100 operation.getName());
1101 LOGGER.error("Error while writing the operation's artifacts to the CSAR", e);
1102 return Either.right(componentsUtils.getResponseFormat(ERROR_DURING_CSAR_CREATION, "Resource", component.getUniqueId()));
1106 return Either.left(zipstream);
1109 private boolean checkComponentBeforeWrite(Component component, Map.Entry<String, InterfaceDefinition> interfaceEntry,
1110 OperationDataDefinition operation) {
1111 final ArtifactDataDefinition implementation = operation.getImplementation();
1112 if (implementation == null) {
1113 LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no Operation Implementation found", component.getNormalizedName(),
1114 interfaceEntry.getValue().getUniqueId(), operation.getName());
1117 final String artifactName = implementation.getArtifactName();
1118 if (artifactName == null) {
1119 LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no artifact found", component.getNormalizedName(),
1120 interfaceEntry.getValue().getUniqueId(), operation.getName());
1123 if (OperationArtifactUtil.artifactNameIsALiteralValue(artifactName)) {
1124 LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - artifact name is a literal value rather than an SDC artifact",
1125 component.getNormalizedName(), interfaceEntry.getValue().getUniqueId(), operation.getName());
1131 private boolean checkComponentBeforeOperation(Component component) {
1132 if (component instanceof Service) {
1135 if (Objects.isNull(((Resource) component).getInterfaces())) {
1136 LOGGER.debug("Component Name {}- no interfaces found", component.getNormalizedName());
1142 private Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath(final Component mainComponent,
1143 final ArtifactsInfo currArtifactsInfo,
1144 final ZipOutputStream zip, final String path,
1145 final boolean isInCertificationRequest) throws IOException {
1146 final Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfo = currArtifactsInfo.getArtifactsInfo();
1147 for (final ArtifactGroupTypeEnum artifactGroupTypeEnum : artifactsInfo.keySet()) {
1148 final String groupTypeFolder = path + WordUtils.capitalizeFully(artifactGroupTypeEnum.getType()) + PATH_DELIMITER;
1149 final Map<String, List<ArtifactDefinition>> artifactTypesMap = artifactsInfo.get(artifactGroupTypeEnum);
1150 for (final String artifactType : artifactTypesMap.keySet()) {
1151 final List<ArtifactDefinition> artifactDefinitionList = artifactTypesMap.get(artifactType);
1152 String artifactTypeFolder = groupTypeFolder + artifactType + PATH_DELIMITER;
1153 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType) && path.contains(ARTIFACTS_PATH + RESOURCES_PATH)) {
1154 // Ignore this packaging as BPMN artifacts needs to be packaged in different manner
1157 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType)) {
1158 artifactTypeFolder += OperationArtifactUtil.BPMN_ARTIFACT_PATH + File.separator;
1159 } else if (ArtifactTypeEnum.ONBOARDED_PACKAGE.getType().equals(artifactType)) {
1160 // renaming legacy folder ONBOARDED_PACKAGE to the new folder ETSI_PACKAGE
1161 artifactTypeFolder = artifactTypeFolder
1162 .replace(ArtifactTypeEnum.ONBOARDED_PACKAGE.getType(), ArtifactTypeEnum.ETSI_PACKAGE.getType());
1164 // TODO: We should not do this but in order to keep this refactoring small enough,
1166 // we'll leave this as is for now
1167 List<ArtifactDefinition> collect = filterArtifactDefinitionToZip(mainComponent, artifactDefinitionList, isInCertificationRequest)
1168 .collect(Collectors.toList());
1169 for (ArtifactDefinition ad : collect) {
1170 zip.putNextEntry(new ZipEntry(artifactTypeFolder + ad.getArtifactName()));
1171 zip.write(ad.getPayloadData());
1175 return Either.left(zip);
1178 private Stream<ArtifactDefinition> filterArtifactDefinitionToZip(Component mainComponent, List<ArtifactDefinition> artifactDefinitionList,
1179 boolean isInCertificationRequest) {
1180 return artifactDefinitionList.stream().filter(shouldBeInZip(isInCertificationRequest, mainComponent)).map(this::fetchPayLoadData)
1181 .filter(Either::isLeft).map(e -> e.left().value());
1184 private Predicate<ArtifactDefinition> shouldBeInZip(boolean isInCertificationRequest, Component component) {
1185 return artifactDefinition -> !(!isInCertificationRequest && component.isService() && artifactDefinition.isHeatEnvType() || artifactDefinition
1186 .hasNoMandatoryEsId());
1189 private Either<ArtifactDefinition, ActionStatus> fetchPayLoadData(ArtifactDefinition ad) {
1190 byte[] payloadData = ad.getPayloadData();
1191 if (payloadData == null) {
1192 return getFromCassandra(ad.getEsId()).left().map(pd -> {
1195 }).right().map(as -> {
1196 LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, ad.getArtifactName(), ad.getUniqueId());
1197 LOGGER.debug("Failed to get {} payload from DB reason: {}", ad.getArtifactName(), as);
1201 return Either.left(ad);
1205 private Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath(Component mainComponent,
1206 ComponentArtifacts componentArtifacts,
1207 ZipOutputStream zipstream, String currentPath,
1208 boolean isInCertificationRequest) throws IOException {
1209 Map<String, ComponentTypeArtifacts> componentTypeArtifacts = componentArtifacts.getComponentTypeArtifacts();
1212 //<Inner Asset TOSCA name (e.g. VFC name)> folder name: <Inner Asset TOSCA name (e.g. VFC name)>_v<version>.
1214 //E.g. "org.openecomp.resource.vf.vipr_atm_v1.0"
1215 Set<String> componentTypeArtifactsKeys = componentTypeArtifacts.keySet();
1216 for (String keyAssetName : componentTypeArtifactsKeys) {
1217 ComponentTypeArtifacts componentInstanceArtifacts = componentTypeArtifacts.get(keyAssetName);
1218 ArtifactsInfo componentArtifacts2 = componentInstanceArtifacts.getComponentArtifacts();
1219 String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
1220 Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent,
1221 componentArtifacts2, zipstream, pathWithAssetName, isInCertificationRequest);
1222 if (writeArtifactsInfoToSpecifiedPath.isRight()) {
1223 return writeArtifactsInfoToSpecifiedPath;
1226 return Either.left(zipstream);
1229 private Either<ToscaRepresentation, ResponseFormat> generateToscaRepresentation(Component component, boolean isSkipImports) {
1230 return toscaExportUtils.exportComponent(component, isSkipImports).right().map(toscaError -> {
1231 LOGGER.debug("exportComponent failed {}", toscaError);
1232 return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
1236 private String createToscaBlock0(String metaFileVersion, String csarVersion, String createdBy, String entryDef, boolean isAsdPackage,
1237 String definitionsPath, boolean isSkipImports) {
1238 final String block0template = "TOSCA-Meta-File-Version: %s\nCSAR-Version: %s\nCreated-By: %s\nEntry-Definitions: "
1239 + definitionsPath + "%s\n%s\n" + (!isSkipImports ? "Name: csar.meta\nContent-Type: text/plain\n" : "");
1240 return String.format(block0template, metaFileVersion, csarVersion, createdBy, entryDef, isAsdPackage ? "entry_definition_type: asd" : "");
1243 private class CsarDefinition {
1245 private ComponentArtifacts componentArtifacts;
1247 // add list of tosca artifacts and meta describes CSAR zip root
1248 public CsarDefinition(ComponentArtifacts componentArtifacts) {
1249 this.componentArtifacts = componentArtifacts;
1252 public ComponentArtifacts getComponentArtifacts() {
1253 return componentArtifacts;
1257 private class ComponentArtifacts {
1259 //artifacts of the component and CI's artifacts contained in it's composition (represents Informational, Deployment & Resource folders of main component)
1260 private ComponentTypeArtifacts mainTypeAndCIArtifacts;
1261 //artifacts of all component types mapped by their tosca name
1262 private Map<String, ComponentTypeArtifacts> componentTypeArtifacts;
1264 public ComponentArtifacts() {
1265 mainTypeAndCIArtifacts = new ComponentTypeArtifacts();
1266 componentTypeArtifacts = new HashMap<>();
1269 public ComponentTypeArtifacts getMainTypeAndCIArtifacts() {
1270 return mainTypeAndCIArtifacts;
1273 public void setMainTypeAndCIArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
1274 this.mainTypeAndCIArtifacts = componentInstanceArtifacts;
1277 public Map<String, ComponentTypeArtifacts> getComponentTypeArtifacts() {
1278 return componentTypeArtifacts;
1283 * The artifacts of the component and of all its composed instances
1285 private class ComponentTypeArtifacts {
1287 private ArtifactsInfo componentArtifacts; //component artifacts (describes the Informational Deployment folders)
1289 private Map<String, ArtifactsInfo> componentInstancesArtifacts; //artifacts of the composed instances mapped by the resourceInstance normalized name (describes the Resources folder)
1291 public ComponentTypeArtifacts() {
1292 componentArtifacts = new ArtifactsInfo();
1293 componentInstancesArtifacts = new HashMap<>();
1296 public ArtifactsInfo getComponentArtifacts() {
1297 return componentArtifacts;
1300 public void setComponentArtifacts(ArtifactsInfo artifactsInfo) {
1301 this.componentArtifacts = artifactsInfo;
1304 public Map<String, ArtifactsInfo> getComponentInstancesArtifacts() {
1305 return componentInstancesArtifacts;
1308 public void addComponentInstancesArtifacts(String normalizedName, ArtifactsInfo artifactsInfo) {
1309 componentInstancesArtifacts.put(normalizedName, artifactsInfo);
1314 * The artifacts Definition saved by their structure
1316 private class ArtifactsInfo {
1317 //Key is the type of artifacts(Informational/Deployment)
1319 //Value is a map between an artifact type and a list of all artifacts of this type
1320 private Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfoField;
1322 public ArtifactsInfo() {
1323 this.artifactsInfoField = new EnumMap<>(ArtifactGroupTypeEnum.class);
1326 public Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> getArtifactsInfo() {
1327 return artifactsInfoField;
1330 public void addArtifactsToGroup(ArtifactGroupTypeEnum artifactGroup, Map<String, List<ArtifactDefinition>> artifactsDefinition) {
1331 if (artifactsInfoField.get(artifactGroup) == null) {
1332 artifactsInfoField.put(artifactGroup, artifactsDefinition);
1334 Map<String, List<ArtifactDefinition>> artifactTypeEnumListMap = artifactsInfoField.get(artifactGroup);
1335 artifactTypeEnumListMap.putAll(artifactsDefinition);
1336 artifactsInfoField.put(artifactGroup, artifactTypeEnumListMap);
1340 public boolean isEmpty() {
1341 return artifactsInfoField.isEmpty();
1344 public boolean isNotEmpty() {
1349 public static class ToscaErrorException extends Exception {
1351 ToscaErrorException(ToscaError error) {
1352 super("Error while exporting component's interface (toscaError:" + error + ")");