Remove generation of csar.meta
[sdc.git] / catalog-be / src / main / java / org / openecomp / sdc / be / tosca / CommonCsarGenerator.java
1 /*
2  * ============LICENSE_START=======================================================
3  * SDC
4  * ================================================================================
5  * Copyright (C) 2023 Nordix Foundation. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20 package org.openecomp.sdc.be.tosca;
21
22 import static org.openecomp.sdc.be.dao.api.ActionStatus.ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION;
23 import static org.openecomp.sdc.be.dao.api.ActionStatus.ERROR_DURING_CSAR_CREATION;
24 import static org.openecomp.sdc.be.tosca.ComponentCache.MergeStrategy.overwriteIfSameVersions;
25 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.fromEither;
26 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.javaListToVavrList;
27 import static org.openecomp.sdc.common.api.Constants.ADDITIONAL_TYPE_DEFINITIONS;
28
29 import com.google.common.primitives.Bytes;
30 import fj.F;
31 import fj.data.Either;
32 import io.vavr.Tuple2;
33 import io.vavr.control.Try;
34 import java.io.BufferedOutputStream;
35 import java.io.ByteArrayInputStream;
36 import java.io.File;
37 import java.io.IOException;
38 import java.nio.charset.StandardCharsets;
39 import java.nio.file.Path;
40 import java.nio.file.Paths;
41 import java.text.SimpleDateFormat;
42 import java.util.ArrayList;
43 import java.util.Date;
44 import java.util.EnumMap;
45 import java.util.HashMap;
46 import java.util.HashSet;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Map.Entry;
50 import java.util.Objects;
51 import java.util.Optional;
52 import java.util.Set;
53 import java.util.TimeZone;
54 import java.util.function.Function;
55 import java.util.function.Predicate;
56 import java.util.function.Supplier;
57 import java.util.regex.Matcher;
58 import java.util.regex.Pattern;
59 import java.util.stream.Collectors;
60 import java.util.stream.Stream;
61 import java.util.zip.ZipEntry;
62 import java.util.zip.ZipInputStream;
63 import java.util.zip.ZipOutputStream;
64 import org.apache.commons.collections.CollectionUtils;
65 import org.apache.commons.collections.MapUtils;
66 import org.apache.commons.io.output.ByteArrayOutputStream;
67 import org.apache.commons.lang.StringUtils;
68 import org.apache.commons.lang3.tuple.ImmutableTriple;
69 import org.apache.commons.lang3.tuple.Triple;
70 import org.apache.commons.text.WordUtils;
71 import org.onap.sdc.tosca.services.YamlUtil;
72 import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException;
73 import org.openecomp.sdc.be.config.CategoryBaseTypeConfig;
74 import org.openecomp.sdc.be.config.ConfigurationManager;
75 import org.openecomp.sdc.be.dao.api.ActionStatus;
76 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
77 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
78 import org.openecomp.sdc.be.dao.cassandra.SdcSchemaFilesCassandraDao;
79 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
80 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
81 import org.openecomp.sdc.be.datatypes.elements.OperationDataDefinition;
82 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
83 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
84 import org.openecomp.sdc.be.impl.ComponentsUtils;
85 import org.openecomp.sdc.be.model.ArtifactDefinition;
86 import org.openecomp.sdc.be.model.Component;
87 import org.openecomp.sdc.be.model.ComponentInstance;
88 import org.openecomp.sdc.be.model.InterfaceDefinition;
89 import org.openecomp.sdc.be.model.LifecycleStateEnum;
90 import org.openecomp.sdc.be.model.Resource;
91 import org.openecomp.sdc.be.model.Service;
92 import org.openecomp.sdc.be.model.category.CategoryDefinition;
93 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
94 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
95 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
96 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
97 import org.openecomp.sdc.be.model.operations.impl.ModelOperation;
98 import org.openecomp.sdc.be.plugins.CsarEntryGenerator;
99 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
100 import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil;
101 import org.openecomp.sdc.be.utils.TypeUtils;
102 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
103 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
104 import org.openecomp.sdc.common.impl.ExternalConfiguration;
105 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
106 import org.openecomp.sdc.common.util.GeneralUtility;
107 import org.openecomp.sdc.common.zip.ZipUtils;
108 import org.openecomp.sdc.exception.ResponseFormat;
109 import org.slf4j.Logger;
110 import org.slf4j.LoggerFactory;
111 import org.springframework.beans.factory.annotation.Autowired;
112 import org.yaml.snakeyaml.DumperOptions;
113 import org.yaml.snakeyaml.Yaml;
114
115 /**
116  * Generates a Network Service CSAR based on a SERVICE component and wraps it in a SDC CSAR entry.
117  */
118 @org.springframework.stereotype.Component("commonCsarGenerator")
119 public class CommonCsarGenerator {
120
121     private static final Logger LOGGER = LoggerFactory.getLogger(CommonCsarGenerator.class);
122     public static final String ARTIFACTS_PATH = "Artifacts/";
123     private static final String RESOURCES_PATH = "Resources/";
124     private static final String PATH_DELIMITER = "/";
125     private static final String SERVICE_MANIFEST = "NS.mf";
126     private static final String ARTIFACT_NAME_UNIQUE_ID = "ArtifactName {}, unique ID {}";
127     private static final String TOSCA_META_PATH_FILE_NAME = "TOSCA-Metadata/TOSCA.meta";
128     private static final String TOSCA_META_VERSION = "1.0";
129     private static final String CSAR_VERSION = "1.1";
130     private static final String BLOCK_0_TEMPLATE = "SDC-TOSCA-Meta-File-Version: %s\nSDC-TOSCA-Definitions-Version: %s\n";
131     private static final String CSAR_META_PATH_FILE_NAME = "csar.meta";
132     private static final String SDC_VERSION = ExternalConfiguration.getAppVersion();
133     public static final String NODES_YML = "nodes.yml";
134     private static final String CONFORMANCE_LEVEL = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
135     private final ToscaOperationFacade toscaOperationFacade;
136     private final ComponentsUtils componentsUtils;
137     private final ToscaExportHandler toscaExportUtils;
138     private final List<CsarEntryGenerator> generators;
139     private final ArtifactCassandraDao artifactCassandraDao;
140     private final String versionFirstThreeOctets;
141     private final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
142     private final ModelOperation modelOperation;
143
144     @Autowired
145     public CommonCsarGenerator(
146         final ToscaOperationFacade toscaOperationFacade,
147         final ComponentsUtils componentsUtils,
148         final ToscaExportHandler toscaExportUtils,
149         final List<CsarEntryGenerator> generators,
150         final ArtifactCassandraDao artifactCassandraDao,
151         final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao,
152         final ModelOperation modelOperation) {
153         this.toscaOperationFacade = toscaOperationFacade;
154         this.componentsUtils = componentsUtils;
155         this.toscaExportUtils = toscaExportUtils;
156         this.generators = generators;
157         this.artifactCassandraDao = artifactCassandraDao;
158         this.versionFirstThreeOctets = readVersionFirstThreeOctets();
159         this.sdcSchemaFilesCassandraDao = sdcSchemaFilesCassandraDao;
160         this.modelOperation = modelOperation;
161     }
162
163     private String readVersionFirstThreeOctets() {
164         if (StringUtils.isEmpty(SDC_VERSION)) {
165             return "";
166         }
167         // change regex to avoid DoS sonar issue
168         Matcher matcher = Pattern.compile("(?!\\.)(\\d{1,9}(\\.\\d{1,9}){1,9})(?![\\d\\.])").matcher(SDC_VERSION);
169         matcher.find();
170         return matcher.group(0);
171     }
172
173     /**
174      * Generates a Network Service CSAR based on a SERVICE component that has category configured in
175      * CategoriesToGenerateNsd enum and wraps it in a SDC CSAR entry.
176      *
177      * @param component the component to create the NS CSAR from
178      * @return an entry to be added in the Component CSAR by SDC
179      */
180     public Either<ZipOutputStream, ResponseFormat> generateCsarZip(Component component,
181                                                                    boolean getFromCS,
182                                                                    ZipOutputStream zip,
183                                                                    boolean isInCertificationRequest,
184                                                                    boolean isAsdPackage,
185                                                                    String definitionsPath,
186                                                                    boolean addDependencies,
187                                                                    boolean isSkipImports) throws IOException {
188         ArtifactDefinition artifactDef = component.getToscaArtifacts().get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
189         Either<ToscaRepresentation, ResponseFormat> toscaRepresentation = fetchToscaRepresentation(component, getFromCS, artifactDef, isSkipImports);
190
191         // This should not be done but in order to keep the refactoring small enough we stop here.
192         // TODO: Refactor the rest of this function
193         byte[] mainYaml;
194         List<Triple<String, String, Component>> dependencies;
195         if (toscaRepresentation.isLeft()) {
196             mainYaml = toscaRepresentation.left().value().getMainYaml();
197             dependencies = toscaRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
198         } else {
199             return Either.right(toscaRepresentation.right().value());
200         }
201
202         if (!isSkipImports) {
203             final String toscaConformanceLevel = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
204             zip.putNextEntry(new ZipEntry(CSAR_META_PATH_FILE_NAME));
205             zip.write(createCsarBlock0(TOSCA_META_VERSION, toscaConformanceLevel).getBytes());
206         }
207
208         final String fileName = artifactDef.getArtifactName();
209         final byte[] toscaBlock0Byte = createToscaBlock0(
210             TOSCA_META_VERSION, CSAR_VERSION, component.getCreatorFullName(), fileName, isAsdPackage, definitionsPath, isSkipImports).getBytes();
211         zip.putNextEntry(new ZipEntry(TOSCA_META_PATH_FILE_NAME));
212         zip.write(toscaBlock0Byte);
213         zip.putNextEntry(new ZipEntry(definitionsPath + fileName));
214         zip.write(mainYaml);
215         LifecycleStateEnum lifecycleState = component.getLifecycleState();
216         addServiceMf(component, zip, lifecycleState, isInCertificationRequest, fileName, mainYaml, definitionsPath);
217         if (addDependencies) {
218             //US798487 - Abstraction of complex types
219             if (hasToWriteComponentSubstitutionType(component)) {
220                 LOGGER.debug("Component {} is complex - generating abstract type for it..", component.getName());
221                 dependencies.addAll(writeComponentInterface(component, zip, fileName, definitionsPath));
222             }
223             //UID <cassandraId,filename,component>
224             Either<ZipOutputStream, ResponseFormat> zipOutputStreamOrResponseFormat =
225                 getZipOutputStreamResponseFormatEither(zip, dependencies, definitionsPath);
226             if (zipOutputStreamOrResponseFormat != null && zipOutputStreamOrResponseFormat.isRight()) {
227                 return zipOutputStreamOrResponseFormat;
228             }
229         }
230         if (component.getModel() == null) {
231             //retrieve SDC.zip from Cassandra
232             Either<byte[], ResponseFormat> latestSchemaFiles = getLatestSchemaFilesFromCassandra();
233             if (latestSchemaFiles.isRight()) {
234                 LOGGER.error("Error retrieving SDC Schema files from cassandra");
235                 return Either.right(latestSchemaFiles.right().value());
236             }
237             final byte[] schemaFileZip = latestSchemaFiles.left().value();
238             final List<String> nodesFromPackage = findNonRootNodesFromPackage(dependencies);
239             //add files from retrieved SDC.zip to Definitions folder in CSAR
240             addSchemaFilesFromCassandra(zip, schemaFileZip, nodesFromPackage, definitionsPath);
241         } else {
242             //retrieve schema files by model from Cassandra
243             addSchemaFilesByModel(zip, component.getModel(), definitionsPath, addDependencies);
244         }
245         Either<CsarDefinition, ResponseFormat> collectedComponentCsarDefinition = collectComponentCsarDefinition(component);
246         if (collectedComponentCsarDefinition.isRight()) {
247             return Either.right(collectedComponentCsarDefinition.right().value());
248         }
249         if (generators != null) {
250             for (CsarEntryGenerator generator : generators) {
251                 LOGGER.debug("Invoking CsarEntryGenerator: {}", generator.getClass().getName());
252                 for (Map.Entry<String, byte[]> pluginGeneratedFile : generator.generateCsarEntries(component).entrySet()) {
253                     zip.putNextEntry(new ZipEntry(pluginGeneratedFile.getKey()));
254                     zip.write(pluginGeneratedFile.getValue());
255                 }
256             }
257         }
258         return writeAllFilesToCsar(component, collectedComponentCsarDefinition.left().value(), zip, isInCertificationRequest);
259     }
260
261     private String createCsarBlock0(String metaFileVersion, String toscaConformanceLevel) {
262         return String.format(BLOCK_0_TEMPLATE, metaFileVersion, toscaConformanceLevel);
263     }
264
265     private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(Component component, boolean getFromCS,
266                                                                                  ArtifactDefinition artifactDef, boolean isSkipImports) {
267         LifecycleStateEnum lifecycleState = component.getLifecycleState();
268         boolean shouldBeFetchedFromCassandra =
269             getFromCS || !(lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN || lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT);
270         Either<ToscaRepresentation, ResponseFormat> toscaRepresentation =
271             shouldBeFetchedFromCassandra ? fetchToscaRepresentation(artifactDef) : generateToscaRepresentation(component, isSkipImports);
272         return toscaRepresentation.left()
273             .bind(iff(myd -> !myd.getDependencies().isDefined(), myd -> fetchToscaTemplateDependencies(myd.getMainYaml(), component)));
274     }
275
276     private Either<ToscaRepresentation, ResponseFormat> fetchToscaTemplateDependencies(byte[] mainYml, Component component) {
277         return toscaExportUtils.getDependencies(component).right().map(toscaError -> {
278             LOGGER.debug("Failed to retrieve dependencies for component {}, error {}", component.getUniqueId(), toscaError);
279             return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
280         }).left().map(tt -> ToscaRepresentation.make(mainYml, tt));
281     }
282
283     private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(ArtifactDefinition artifactDef) {
284         return getFromCassandra(artifactDef.getEsId()).right().map(as -> {
285             LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, artifactDef.getArtifactName(), artifactDef.getUniqueId());
286             return componentsUtils.getResponseFormat(as);
287         }).left().map(ToscaRepresentation::make);
288     }
289
290     private Either<byte[], ActionStatus> getFromCassandra(String cassandraId) {
291         return artifactCassandraDao.getArtifact(cassandraId).right().map(operationstatus -> {
292             LOGGER.info("Failed to fetch artifact from Cassandra by id {} error {}.", cassandraId, operationstatus);
293             StorageOperationStatus storageStatus = DaoStatusConverter.convertCassandraStatusToStorageStatus(operationstatus);
294             return componentsUtils.convertFromStorageResponse(storageStatus);
295         }).left().map(DAOArtifactData::getDataAsArray);
296     }
297
298     private static <L, R> F<L, Either<L, R>> iff(Predicate<L> p, Function<L, Either<L, R>> ifTrue) {
299         return l -> p.test(l) ? ifTrue.apply(l) : Either.left(l);
300     }
301
302     private static <A, B> F<A, B> iff(Predicate<A> p, Supplier<B> s, Function<A, B> orElse) {
303         return a -> p.test(a) ? s.get() : orElse.apply(a);
304     }
305
306     private void addServiceMf(Component component, ZipOutputStream zip, LifecycleStateEnum lifecycleState, boolean isInCertificationRequest,
307                               String fileName, byte[] mainYaml, String definitionsPath) throws IOException {
308         // add mf
309         if ((component.getComponentType() == ComponentTypeEnum.SERVICE) && (lifecycleState != LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT)) {
310             String serviceName = component.getName();
311             String createdBy = component.getCreatorUserId();
312             String serviceVersion;
313             if (isInCertificationRequest) {
314                 int tmp = Integer.valueOf(component.getVersion().split("\\.")[0]) + 1;
315                 serviceVersion = String.valueOf(tmp) + ".0";
316             } else {
317                 serviceVersion = component.getVersion();
318             }
319             SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
320             format.setTimeZone(TimeZone.getTimeZone("UTC"));
321             Date date = new Date();
322             String releaseTime = format.format(date);
323             if (component.getCategories() == null || component.getCategories().get(0) == null) {
324                 return;
325             }
326             String serviceType = component.getCategories().get(0).getName();
327             String description = component.getDescription();
328             String serviceTemplate = definitionsPath + fileName;
329             String hash = GeneralUtility.calculateMD5Base64EncodedByByteArray(mainYaml);
330             String nsMfBlock0 = createNsMfBlock0(serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate,
331                 hash);
332             byte[] nsMfBlock0Byte = nsMfBlock0.getBytes();
333             zip.putNextEntry(new ZipEntry(SERVICE_MANIFEST));
334             zip.write(nsMfBlock0Byte);
335         }
336     }
337
338     private String createNsMfBlock0(String serviceName, String createdBy, String serviceVersion, String releaseTime, String serviceType,
339                                     String description, String serviceTemplate, String hash) {
340         final String block0template = "metadata??\n" + "ns_product_name: %s\n" + "ns_provider_id: %s\n" + "ns_package_version: %s\n" +
341             "ns_release_data_time: %s\n" + "ns_type: %s\n" + "ns_package_description: %s\n\n" + "Source: %s\n" + "Algorithm: MD5\n" + "Hash: %s\n\n";
342         return String.format(block0template, serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate, hash);
343     }
344
345     private boolean hasToWriteComponentSubstitutionType(final Component component) {
346         final Map<String, CategoryBaseTypeConfig> serviceNodeTypesConfig =
347             ConfigurationManager.getConfigurationManager().getConfiguration().getServiceBaseNodeTypes();
348         List<CategoryDefinition> categories = component.getCategories();
349         if (CollectionUtils.isNotEmpty(categories) && MapUtils.isNotEmpty(serviceNodeTypesConfig)
350             && serviceNodeTypesConfig.get(categories.get(0).getName()) != null) {
351             boolean doNotExtendBaseType = serviceNodeTypesConfig.get(categories.get(0).getName()).isDoNotExtendBaseType();
352             if (doNotExtendBaseType) {
353                 return false;
354             }
355         }
356         if (component instanceof Service) {
357             return !ModelConverter.isAtomicComponent(component) && ((Service) component).isSubstituteCandidate();
358         }
359         return !ModelConverter.isAtomicComponent(component);
360     }
361
362     private Either<ZipOutputStream, ResponseFormat> writeComponentInterface(Either<ToscaRepresentation, ToscaError> interfaceRepresentation,
363                                                                             ZipOutputStream zip, String fileName, String definitionsPath) {
364         // TODO: This should not be done but we need this to keep the refactoring small enough to be easily reviewable
365         return writeComponentInterface(interfaceRepresentation, fileName, ZipWriter.live(zip), definitionsPath)
366             .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)).recover(th -> {
367                 LOGGER.error("#writeComponentInterface - zip writing failed with error: ", th);
368                 return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
369             }).get();
370     }
371
372     private Try<Void> writeComponentInterface(
373         Either<ToscaRepresentation, ToscaError> interfaceRepresentation, String fileName, ZipWriter zw, String definitionsPath) {
374         Either<byte[], ToscaError> yml = interfaceRepresentation.left()
375             .map(ToscaRepresentation::getMainYaml);
376         return fromEither(yml, ToscaErrorException::new).flatMap(zw.write(definitionsPath + ToscaExportHandler.getInterfaceFilename(fileName)));
377     }
378
379     private List<Triple<String, String, Component>> writeComponentInterface(final Component component, final ZipOutputStream zip,
380                                                                             final String fileName, final String definitionsPath) {
381         final Either<ToscaRepresentation, ToscaError> interfaceRepresentation = toscaExportUtils.exportComponentInterface(component, false);
382         writeComponentInterface(interfaceRepresentation, zip, fileName, definitionsPath);
383         return interfaceRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
384     }
385
386     private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither(ZipOutputStream zip,
387                                                                                            List<Triple<String, String, Component>> dependencies,
388                                                                                            String definitionsPath)
389         throws IOException {
390         ComponentCache
391             innerComponentsCache = ComponentCache.overwritable(overwriteIfSameVersions()).onMerge((oldValue, newValue) ->
392             LOGGER.warn("Overwriting component invariantID {} of version {} with a newer version {}", oldValue.getId(),
393                 oldValue.getComponentVersion(),
394                 newValue.getComponentVersion()));
395         if (dependencies != null && !dependencies.isEmpty()) {
396             for (Triple<String, String, Component> d : dependencies) {
397                 String cassandraId = d.getMiddle();
398                 Component childComponent = d.getRight();
399                 Either<byte[], ResponseFormat> entryData = getEntryData(cassandraId, childComponent).right()
400                     .map(componentsUtils::getResponseFormat);
401                 if (entryData.isRight()) {
402                     return Either.right(entryData.right().value());
403                 }
404                 //fill innerComponentsCache
405                 String fileName = d.getLeft();
406                 innerComponentsCache.put(cassandraId, fileName, childComponent);
407                 addInnerComponentsToCache(innerComponentsCache, childComponent);
408             }
409             //add inner components to CSAR
410             return addInnerComponentsToCSAR(zip, innerComponentsCache, definitionsPath);
411         }
412         return null;
413     }
414
415     private Either<ZipOutputStream, ResponseFormat> addInnerComponentsToCSAR(ZipOutputStream zip, ComponentCache innerComponentsCache,
416                                                                              String definitionsPath)
417         throws IOException {
418         for (ImmutableTriple<String, String, Component> ict : innerComponentsCache.iterable()) {
419             Component innerComponent = ict.getRight();
420             String icFileName = ict.getMiddle();
421             // add component to zip
422             Either<Tuple2<byte[], ZipEntry>, ResponseFormat> zipEntry = toZipEntry(ict, definitionsPath);
423             // TODO: this should not be done, we should instead compose this either further,
424
425             // but in order to keep this refactoring small, we'll stop here.
426             if (zipEntry.isRight()) {
427                 return Either.right(zipEntry.right().value());
428             }
429             Tuple2<byte[], ZipEntry> value = zipEntry.left().value();
430             zip.putNextEntry(value._2);
431             zip.write(value._1);
432             // add component interface to zip
433             if (hasToWriteComponentSubstitutionType(innerComponent)) {
434                 writeComponentInterface(innerComponent, zip, icFileName, definitionsPath);
435             }
436         }
437         return null;
438     }
439
440     private Either<Tuple2<byte[], ZipEntry>, ResponseFormat> toZipEntry(ImmutableTriple<String, String, Component> cachedEntry,
441                                                                         String definitionsPath) {
442         String cassandraId = cachedEntry.getLeft();
443         String fileName = cachedEntry.getMiddle();
444         Component innerComponent = cachedEntry.getRight();
445         return getEntryData(cassandraId, innerComponent).right().map(status -> {
446             LOGGER.debug("Failed adding to zip component {}, error {}", cassandraId, status);
447             return componentsUtils.getResponseFormat(status);
448         }).left().map(content -> new Tuple2<>(content, new ZipEntry(definitionsPath + fileName)));
449     }
450
451     private void addInnerComponentsToCache(ComponentCache componentCache, Component childComponent) {
452         javaListToVavrList(childComponent.getComponentInstances()).filter(ci -> componentCache.notCached(ci.getComponentUid())).forEach(ci -> {
453             // all resource must be only once!
454             Either<Resource, StorageOperationStatus> resource = toscaOperationFacade.getToscaElement(ci.getComponentUid());
455             Component componentRI = checkAndAddComponent(componentCache, ci, resource);
456             //if not atomic - insert inner components as well
457
458             // TODO: This could potentially create a StackOverflowException if the call stack
459
460             // happens to be too large. Tail-recursive optimization should be used here.
461             if (!ModelConverter.isAtomicComponent(componentRI)) {
462                 addInnerComponentsToCache(componentCache, componentRI);
463             }
464         });
465     }
466
467     private Component checkAndAddComponent(ComponentCache componentCache, ComponentInstance ci, Either<Resource, StorageOperationStatus> resource) {
468         if (resource.isRight()) {
469             LOGGER.debug("Failed to fetch resource with id {} for instance {}", ci.getComponentUid(), ci.getName());
470         }
471         Component componentRI = resource.left().value();
472         Map<String, ArtifactDefinition> childToscaArtifacts = componentRI.getToscaArtifacts();
473         ArtifactDefinition childArtifactDefinition = childToscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
474         if (childArtifactDefinition != null) {
475             //add to cache
476             componentCache.put(childArtifactDefinition.getEsId(), childArtifactDefinition.getArtifactName(), componentRI);
477         }
478         return componentRI;
479     }
480
481     private Either<byte[], ActionStatus> getEntryData(String cassandraId, Component childComponent) {
482         if (cassandraId == null || cassandraId.isEmpty()) {
483             return toscaExportUtils.exportComponent(childComponent).right().map(toscaErrorToActionStatus(childComponent)).left()
484                 .map(ToscaRepresentation::getMainYaml);
485         } else {
486             return getFromCassandra(cassandraId);
487         }
488     }
489
490     private F<ToscaError, ActionStatus> toscaErrorToActionStatus(Component childComponent) {
491         return toscaError -> {
492             LOGGER.debug("Failed to export tosca template for child component {} error {}", childComponent.getUniqueId(), toscaError);
493             return componentsUtils.convertFromToscaError(toscaError);
494         };
495     }
496
497     private Either<byte[], ResponseFormat> getLatestSchemaFilesFromCassandra() {
498         String fto = versionFirstThreeOctets;
499         return sdcSchemaFilesCassandraDao.getSpecificSchemaFiles(fto, CONFORMANCE_LEVEL).right().map(schemaFilesFetchDBError(fto)).left()
500             .bind(iff(List::isEmpty, () -> schemaFileFetchError(fto), s -> Either.left(s.iterator().next().getPayloadAsArray())));
501     }
502
503     private F<CassandraOperationStatus, ResponseFormat> schemaFilesFetchDBError(String firstThreeOctets) {
504         return cos -> {
505             LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}. Please fix DB table accordingly.", firstThreeOctets,
506                 CONFORMANCE_LEVEL);
507             StorageOperationStatus sos = DaoStatusConverter.convertCassandraStatusToStorageStatus(cos);
508             return componentsUtils.getResponseFormat(componentsUtils.convertFromStorageResponse(sos));
509         };
510     }
511
512     private Either<byte[], ResponseFormat> schemaFileFetchError(String firstThreeOctets) {
513         LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}", firstThreeOctets, CONFORMANCE_LEVEL);
514         return Either.right(componentsUtils.getResponseFormat(ActionStatus.TOSCA_SCHEMA_FILES_NOT_FOUND, firstThreeOctets, CONFORMANCE_LEVEL));
515     }
516
517     /**
518      * Create a list of all derived nodes found on the package
519      *
520      * @param dependencies all node dependencies
521      * @return a list of nodes
522      */
523     private List<String> findNonRootNodesFromPackage(final List<Triple<String, String, Component>> dependencies) {
524         final List<String> nodes = new ArrayList<>();
525         if (CollectionUtils.isNotEmpty(dependencies)) {
526             final String NATIVE_ROOT = "tosca.nodes.Root";
527             dependencies.forEach(dependency -> {
528                 if (dependency.getRight() instanceof Resource) {
529                     final Resource resource = (Resource) dependency.getRight();
530                     if (CollectionUtils.isNotEmpty(resource.getDerivedList())) {
531                         resource.getDerivedList().stream().filter(node -> !nodes.contains(node) && !NATIVE_ROOT.equalsIgnoreCase(node))
532                             .forEach(node -> nodes.add(node));
533                     }
534                 }
535             });
536         }
537         return nodes;
538     }
539
540     /**
541      * Writes to a CSAR zip from casandra schema data
542      *
543      * @param zipOutputStream  stores the input stream content
544      * @param schemaFileZip    zip data from Cassandra
545      * @param nodesFromPackage list of all nodes found on the onboarded package
546      */
547     private void addSchemaFilesFromCassandra(final ZipOutputStream zipOutputStream, final byte[] schemaFileZip, final List<String> nodesFromPackage,
548                                              final String definitionsPath) {
549         final int initSize = 2048;
550         LOGGER.debug("Starting copy from Schema file zip to CSAR zip");
551         try (final ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(
552             schemaFileZip)); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(
553             byteArrayOutputStream, initSize)) {
554             ZipEntry entry;
555             while ((entry = zipInputStream.getNextEntry()) != null) {
556                 ZipUtils.checkForZipSlipInRead(entry);
557                 final String entryName = entry.getName();
558                 int readSize = initSize;
559                 final byte[] entryData = new byte[initSize];
560                 if (shouldZipEntryBeHandled(entryName)) {
561                     if (NODES_YML.equalsIgnoreCase(entryName)) {
562                         handleNode(zipInputStream, byteArrayOutputStream, nodesFromPackage);
563                     } else {
564                         while ((readSize = zipInputStream.read(entryData, 0, readSize)) != -1) {
565                             bufferedOutputStream.write(entryData, 0, readSize);
566                         }
567                         bufferedOutputStream.flush();
568                     }
569                     byteArrayOutputStream.flush();
570                     zipOutputStream.putNextEntry(new ZipEntry(definitionsPath + entryName));
571                     zipOutputStream.write(byteArrayOutputStream.toByteArray());
572                     zipOutputStream.flush();
573                     byteArrayOutputStream.reset();
574                 }
575             }
576         } catch (final Exception e) {
577             LOGGER.error("Error while writing the SDC schema file to the CSAR", e);
578             throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
579         }
580         LOGGER.debug("Finished copy from Schema file zip to CSAR zip");
581     }
582
583     /**
584      * Handles the nodes.yml zip entry, updating the nodes.yml to avoid duplicated nodes on it.
585      *
586      * @param zipInputStream        the zip entry to be read
587      * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
588      * @param nodesFromPackage      list of all nodes found on the onboarded package
589      */
590     private void handleNode(final ZipInputStream zipInputStream, final ByteArrayOutputStream byteArrayOutputStream,
591                             final List<String> nodesFromPackage) throws IOException {
592         final Map<String, Object> nodesFromArtifactFile = readYamlZipEntry(zipInputStream);
593         final Map<String, Object> nodesYaml = updateNodeYml(nodesFromPackage, nodesFromArtifactFile);
594         updateZipEntry(byteArrayOutputStream, nodesYaml);
595     }
596
597     /**
598      * Updates the zip entry from the given parameters
599      *
600      * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
601      * @param nodesYaml             a Map of nodes to be written
602      */
603     private void updateZipEntry(final ByteArrayOutputStream byteArrayOutputStream, final Map<String, Object> nodesYaml) throws IOException {
604         if (MapUtils.isNotEmpty(nodesYaml)) {
605             byteArrayOutputStream.write(new YamlUtil().objectToYaml(nodesYaml).getBytes());
606         }
607     }
608
609     /**
610      * Filters and removes all duplicated nodes found
611      *
612      * @param nodesFromPackage      a List of all derived nodes found on the given package
613      * @param nodesFromArtifactFile represents the nodes.yml file stored in Cassandra
614      * @return a nodes Map updated
615      */
616     private Map<String, Object> updateNodeYml(final List<String> nodesFromPackage, final Map<String, Object> nodesFromArtifactFile) {
617         if (MapUtils.isNotEmpty(nodesFromArtifactFile)) {
618             final String nodeTypeBlock = TypeUtils.ToscaTagNamesEnum.NODE_TYPES.getElementName();
619             final Map<String, Object> nodeTypes = (Map<String, Object>) nodesFromArtifactFile.get(nodeTypeBlock);
620             nodesFromPackage.stream().filter(nodeTypes::containsKey).forEach(nodeTypes::remove);
621             nodesFromArtifactFile.replace(nodeTypeBlock, nodeTypes);
622         }
623         return nodesFromArtifactFile;
624     }
625
626     /**
627      * Writes a new zip entry
628      *
629      * @param zipInputStream the zip entry to be read
630      * @return a map of the given zip entry
631      */
632     private Map<String, Object> readYamlZipEntry(final ZipInputStream zipInputStream) throws IOException {
633         final int initSize = 2048;
634         final StringBuilder zipEntry = new StringBuilder();
635         final byte[] buffer = new byte[initSize];
636         int read = 0;
637         while ((read = zipInputStream.read(buffer, 0, initSize)) >= 0) {
638             zipEntry.append(new String(buffer, 0, read));
639         }
640         return (Map<String, Object>) new Yaml().load(zipEntry.toString());
641     }
642
643     /**
644      * Checks if the zip entry should or should not be added to the CSAR based on the given global type list
645      *
646      * @param entryName the zip entry name
647      * @return true if the zip entry should be handled
648      */
649     private boolean shouldZipEntryBeHandled(final String entryName) {
650         return ConfigurationManager.getConfigurationManager().getConfiguration().getGlobalCsarImports().stream()
651             .anyMatch(entry -> entry.contains(entryName));
652     }
653
654     private void addSchemaFilesByModel(final ZipOutputStream zipOutputStream, final String modelName,
655                                        final String definitionsPath, final boolean isSingleImportsFile) {
656         try {
657             final List<ToscaImportByModel> modelDefaultImportList = modelOperation.findAllModelImports(modelName, true);
658             final Set<Path> writtenEntryPathList = new HashSet<>();
659             final var defsPath = Path.of(definitionsPath);
660             Map<Path, byte[]> contentToMerge = new HashMap<>();
661             for (final ToscaImportByModel toscaImportByModel : modelDefaultImportList) {
662                 var importPath = Path.of(toscaImportByModel.getFullPath());
663                 if (!isSingleImportsFile) {
664                     if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(importPath)).normalize().toString())) {
665                         final Path entryPath = defsPath.resolve(importPath);
666                         contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
667                     } else {
668                         if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
669                             importPath = ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
670                         }
671                         final Path entryPath = defsPath.resolve(importPath);
672                         writtenEntryPathList.add(entryPath);
673                         contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
674                     }
675                 } else {
676                     if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
677                         importPath = ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
678                     }
679                     final Path entryPath = defsPath.resolve(importPath);
680                     zipOutputStream.putNextEntry(new ZipEntry(entryPath.toString()));
681                     writtenEntryPathList.add(entryPath);
682                     final byte[] content = toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8);
683                     zipOutputStream.write(content, 0, content.length);
684                     zipOutputStream.closeEntry();
685                 }
686             }
687             if (!isSingleImportsFile) {
688                 byte[] mergingContent = new byte[0];
689                 for (Map.Entry<Path, byte[]> entry : contentToMerge.entrySet()) {
690                     if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(entry.getKey())).normalize().toString())) {
691                         mergingContent = mergeContent(mergingContent, entry.getValue());
692                     } else {
693                         final var zipEntry = new ZipEntry(entry.getKey().toString());
694                         zipOutputStream.putNextEntry(zipEntry);
695                         writtenEntryPathList.add(entry.getKey());
696                         mergingContent = mergeContent(mergingContent, entry.getValue());
697                         zipOutputStream.write(mergingContent, 0, mergingContent.length);
698                         zipOutputStream.closeEntry();
699                     }
700                 }
701             }
702         } catch (final IOException e) {
703             LOGGER.error(String.valueOf(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR), CsarUtils.class.getName(),
704                 "Error while writing the schema files by model to the CSAR", e);
705             throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.CSAR_TOSCA_IMPORTS_ERROR));
706         }
707     }
708
709     private byte[] mergeContent(final byte[] first, final byte[] second) {
710         byte[] merged = new byte[0];
711         final Map<String, Object> firstMap = new Yaml().load(new String(first));
712         final Map<String, Object> secondMap = new Yaml().load(new String(second));
713         if (MapUtils.isNotEmpty(secondMap)) {
714             final DumperOptions options = new DumperOptions();
715             options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
716             final Yaml yaml = new Yaml(options);
717             for (final Entry<String, Object> secondMapEntry : secondMap.entrySet()) {
718                 final Map<String, Object> newMap = new HashMap<>();
719                 if (secondMapEntry.getKey().endsWith("_types")) {
720                     if (MapUtils.isNotEmpty(firstMap) && firstMap.containsKey(secondMapEntry.getKey())) {
721                         final Map<String, Object> secondMapEntryValue = (Map<String, Object>) secondMapEntry.getValue();
722                         final Map<String, Object> firstMapValue = (Map<String, Object>) firstMap.get(secondMapEntry.getKey());
723                         secondMapEntryValue.putAll(firstMapValue);
724                         newMap.put(secondMapEntry.getKey(), secondMapEntryValue);
725                     } else {
726                         newMap.put(secondMapEntry.getKey(), secondMapEntry.getValue());
727                     }
728                 } else {
729                     newMap.put(secondMapEntry.getKey(), secondMapEntry.getValue());
730                 }
731                 merged = Bytes.concat(merged, yaml.dumpAsMap(newMap).getBytes());
732             }
733         }
734         return merged;
735     }
736
737     private Either<CsarDefinition, ResponseFormat> collectComponentCsarDefinition(Component component) {
738         ComponentArtifacts componentArtifacts = new ComponentArtifacts();
739         Component updatedComponent = component;
740
741         //get service to receive the AII artifacts uploaded to the service
742         if (updatedComponent.getComponentType() == ComponentTypeEnum.SERVICE) {
743             Either<Service, StorageOperationStatus> getServiceResponse = toscaOperationFacade.getToscaElement(updatedComponent.getUniqueId());
744
745             if (getServiceResponse.isRight()) {
746                 ActionStatus actionStatus = componentsUtils.convertFromStorageResponse(getServiceResponse.right().value());
747                 return Either.right(componentsUtils.getResponseFormat(actionStatus));
748             }
749
750             updatedComponent = getServiceResponse.left().value();
751         }
752
753         //find the artifacts of the main component, it would have its composed instances artifacts in a separate folder
754         ComponentTypeArtifacts componentInstanceArtifacts = new ComponentTypeArtifacts();
755         ArtifactsInfo artifactsInfo = collectComponentArtifacts(updatedComponent);
756         componentInstanceArtifacts.setComponentArtifacts(artifactsInfo);
757         componentArtifacts.setMainTypeAndCIArtifacts(componentInstanceArtifacts);
758
759         Map<String, ComponentTypeArtifacts> resourceTypeArtifacts = componentArtifacts
760             .getComponentTypeArtifacts();    //artifacts mapped by the component type(tosca name+version)
761         //get the component instances
762         List<ComponentInstance> componentInstances = updatedComponent.getComponentInstances();
763         if (componentInstances != null) {
764             for (ComponentInstance componentInstance : componentInstances) {
765                 //call recursive to find artifacts for all the path
766                 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
767                     updatedComponent, componentInstance, resourceTypeArtifacts, componentInstanceArtifacts);
768                 if (collectComponentInstanceArtifacts.isRight()) {
769                     return Either.right(collectComponentInstanceArtifacts.right().value());
770                 }
771             }
772         }
773
774         if (LOGGER.isDebugEnabled()) {
775             printResult(componentArtifacts, updatedComponent.getName());
776         }
777
778         return Either.left(new CsarDefinition(componentArtifacts));
779     }
780
781     private void printResult(ComponentArtifacts componentArtifacts, String name) {
782         StringBuilder result = new StringBuilder();
783         result.append("Artifacts of main component " + name + "\n");
784         ComponentTypeArtifacts componentInstanceArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
785         printArtifacts(componentInstanceArtifacts);
786         result.append("Type Artifacts\n");
787         for (Map.Entry<String, ComponentTypeArtifacts> typeArtifacts : componentArtifacts.getComponentTypeArtifacts().entrySet()) {
788             result.append("Folder " + typeArtifacts.getKey() + "\n");
789             result.append(printArtifacts(typeArtifacts.getValue()));
790         }
791
792         if (LOGGER.isDebugEnabled()) {
793             LOGGER.debug(result.toString());
794         }
795     }
796
797     private String printArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
798         StringBuilder result = new StringBuilder();
799         ArtifactsInfo artifactsInfo = componentInstanceArtifacts.getComponentArtifacts();
800         Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentArtifacts = artifactsInfo.getArtifactsInfo();
801         printArtifacts(componentArtifacts);
802         result = result.append("Resources\n");
803         for (Map.Entry<String, ArtifactsInfo> resourceInstance : componentInstanceArtifacts.getComponentInstancesArtifacts().entrySet()) {
804             result.append("Folder" + resourceInstance.getKey() + "\n");
805             result.append(printArtifacts(resourceInstance.getValue().getArtifactsInfo()));
806         }
807
808         return result.toString();
809     }
810
811     private String printArtifacts(Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componetArtifacts) {
812         StringBuilder result = new StringBuilder();
813         for (Map.Entry<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactGroup : componetArtifacts.entrySet()) {
814             result.append("    " + artifactGroup.getKey().getType());
815             for (Map.Entry<String, List<ArtifactDefinition>> groupArtifacts : artifactGroup.getValue().entrySet()) {
816                 result.append("        " + groupArtifacts.getKey());
817                 for (ArtifactDefinition artifact : groupArtifacts.getValue()) {
818                     result.append("            " + artifact.getArtifactDisplayName());
819                 }
820             }
821         }
822
823         return result.toString();
824     }
825
826     private Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts(Component parentComponent, ComponentInstance componentInstance,
827                                                                               Map<String, ComponentTypeArtifacts> resourcesTypeArtifacts,
828                                                                               ComponentTypeArtifacts instanceArtifactsLocation) {
829         //1. get the component instance component
830         String componentUid;
831         if (componentInstance.getOriginType() == OriginTypeEnum.ServiceProxy) {
832             componentUid = componentInstance.getSourceModelUid();
833         } else {
834             componentUid = componentInstance.getComponentUid();
835         }
836         Either<Component, StorageOperationStatus> component = toscaOperationFacade.getToscaElement(componentUid);
837         if (component.isRight()) {
838             LOGGER.error("Failed to fetch resource with id {} for instance {}", componentUid, parentComponent.getUUID());
839             return Either.right(componentsUtils.getResponseFormat(ActionStatus.ASSET_NOT_FOUND_DURING_CSAR_CREATION,
840                 parentComponent.getComponentType().getValue(), parentComponent.getUUID(),
841                 componentInstance.getOriginType().getComponentType().getValue(), componentUid));
842         }
843         Component fetchedComponent = component.left().value();
844
845         //2. fill the artifacts for the current component parent type
846         String toscaComponentName =
847             componentInstance.getToscaComponentName() + "_v" + componentInstance.getComponentVersion();
848
849         // if there are no artifacts for this component type we need to fetch and build them
850         ComponentTypeArtifacts componentParentArtifacts = Optional
851             .ofNullable(resourcesTypeArtifacts.get(toscaComponentName))
852             .orElseGet(() -> collectComponentTypeArtifacts(fetchedComponent));
853
854         if (componentParentArtifacts.getComponentArtifacts().isNotEmpty()) {
855             resourcesTypeArtifacts.put(toscaComponentName, componentParentArtifacts);
856         }
857
858         //3. find the artifacts specific to the instance
859         Map<String, List<ArtifactDefinition>> componentInstanceSpecificInformationalArtifacts =
860             getComponentInstanceSpecificArtifacts(componentInstance.getArtifacts(),
861                 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.INFORMATIONAL);
862         Map<String, List<ArtifactDefinition>> componentInstanceSpecificDeploymentArtifacts =
863             getComponentInstanceSpecificArtifacts(componentInstance.getDeploymentArtifacts(),
864                 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.DEPLOYMENT);
865
866         //4. add the instances artifacts to the component type
867         ArtifactsInfo artifactsInfo = new ArtifactsInfo();
868         if (!componentInstanceSpecificInformationalArtifacts.isEmpty()) {
869             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, componentInstanceSpecificInformationalArtifacts);
870         }
871         if (!componentInstanceSpecificDeploymentArtifacts.isEmpty()) {
872             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, componentInstanceSpecificDeploymentArtifacts);
873         }
874         if (!artifactsInfo.isEmpty()) {
875             instanceArtifactsLocation.addComponentInstancesArtifacts(componentInstance.getNormalizedName(), artifactsInfo);
876         }
877
878         //5. do the same for all the component instances
879         List<ComponentInstance> componentInstances = fetchedComponent.getComponentInstances();
880         if (componentInstances != null) {
881             for (ComponentInstance childComponentInstance : componentInstances) {
882                 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
883                     fetchedComponent, childComponentInstance, resourcesTypeArtifacts, componentParentArtifacts);
884                 if (collectComponentInstanceArtifacts.isRight()) {
885                     return collectComponentInstanceArtifacts;
886                 }
887             }
888         }
889
890         return Either.left(true);
891     }
892
893     private Map<String, List<ArtifactDefinition>> getComponentInstanceSpecificArtifacts(Map<String, ArtifactDefinition> componentArtifacts,
894                                                                                         Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentTypeArtifacts,
895                                                                                         ArtifactGroupTypeEnum artifactGroupTypeEnum) {
896         Map<String, List<ArtifactDefinition>> parentArtifacts = componentTypeArtifacts
897             .get(artifactGroupTypeEnum);    //the artfiacts of the component itself and not the instance
898
899         Map<String, List<ArtifactDefinition>> artifactsByTypeOfComponentInstance = new HashMap<>();
900         if (componentArtifacts != null) {
901             for (ArtifactDefinition artifact : componentArtifacts.values()) {
902                 List<ArtifactDefinition> parentArtifactsByType = null;
903                 if (parentArtifacts != null) {
904                     parentArtifactsByType = parentArtifacts.get(artifact.getArtifactType());
905                 }
906                 //the artifact is of instance
907                 if (parentArtifactsByType == null || !parentArtifactsByType.contains(artifact)) {
908                     List<ArtifactDefinition> typeArtifacts = artifactsByTypeOfComponentInstance.get(artifact.getArtifactType());
909                     if (typeArtifacts == null) {
910                         typeArtifacts = new ArrayList<>();
911                         artifactsByTypeOfComponentInstance.put(artifact.getArtifactType(), typeArtifacts);
912                     }
913                     typeArtifacts.add(artifact);
914                 }
915             }
916         }
917
918         return artifactsByTypeOfComponentInstance;
919     }
920
921     private ComponentTypeArtifacts collectComponentTypeArtifacts(Component fetchedComponent) {
922         ArtifactsInfo componentArtifacts = collectComponentArtifacts(fetchedComponent);
923         ComponentTypeArtifacts componentArtifactsInfo = new ComponentTypeArtifacts();
924         if (componentArtifacts.isNotEmpty()) {
925             componentArtifactsInfo.setComponentArtifacts(componentArtifacts);
926         }
927         return componentArtifactsInfo;
928     }
929
930     private ArtifactsInfo collectComponentArtifacts(Component component) {
931         Map<String, ArtifactDefinition> informationalArtifacts = component.getArtifacts();
932         Map<String, List<ArtifactDefinition>> informationalArtifactsByType = collectGroupArtifacts(informationalArtifacts);
933         Map<String, ArtifactDefinition> deploymentArtifacts = component.getDeploymentArtifacts();
934         Map<String, List<ArtifactDefinition>> deploymentArtifactsByType = collectGroupArtifacts(deploymentArtifacts);
935         ArtifactsInfo artifactsInfo = new ArtifactsInfo();
936         if (!informationalArtifactsByType.isEmpty()) {
937             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, informationalArtifactsByType);
938         }
939         if (!deploymentArtifactsByType.isEmpty()) {
940             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, deploymentArtifactsByType);
941         }
942
943         return artifactsInfo;
944     }
945
946     private Map<String, List<ArtifactDefinition>> collectGroupArtifacts(
947         final Map<String, ArtifactDefinition> componentArtifacts) {
948         final Map<String, List<ArtifactDefinition>> artifactsByType = new HashMap<>();
949         for (final ArtifactDefinition artifact : componentArtifacts.values()) {
950             if (artifact.getArtifactUUID() != null) {
951                 artifactsByType.putIfAbsent(artifact.getArtifactType(), new ArrayList<>());
952                 final List<ArtifactDefinition> typeArtifacts = artifactsByType.get(artifact.getArtifactType());
953                 typeArtifacts.add(artifact);
954             }
955         }
956         return artifactsByType;
957     }
958
959     private Either<ZipOutputStream, ResponseFormat> writeAllFilesToCsar(Component mainComponent, CsarDefinition csarDefinition,
960                                                                         ZipOutputStream zipstream, boolean isInCertificationRequest)
961         throws IOException {
962         ComponentArtifacts componentArtifacts = csarDefinition.getComponentArtifacts();
963         Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath = writeComponentArtifactsToSpecifiedPath(mainComponent,
964             componentArtifacts, zipstream, ARTIFACTS_PATH, isInCertificationRequest);
965         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
966             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
967         }
968         ComponentTypeArtifacts mainTypeAndCIArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
969         writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, mainTypeAndCIArtifacts.getComponentArtifacts(),
970             zipstream, ARTIFACTS_PATH, isInCertificationRequest);
971         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
972             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
973         }
974         Map<String, ArtifactsInfo> componentInstancesArtifacts = mainTypeAndCIArtifacts.getComponentInstancesArtifacts();
975         String currentPath = ARTIFACTS_PATH + RESOURCES_PATH;
976         for (String keyAssetName : componentInstancesArtifacts.keySet()) {
977             ArtifactsInfo artifactsInfo = componentInstancesArtifacts.get(keyAssetName);
978             String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
979             writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, artifactsInfo, zipstream, pathWithAssetName,
980                 isInCertificationRequest);
981             if (writeComponentArtifactsToSpecifiedPath.isRight()) {
982                 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
983             }
984         }
985         writeComponentArtifactsToSpecifiedPath = writeOperationsArtifactsToCsar(mainComponent, zipstream);
986         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
987             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
988         }
989         return Either.left(zipstream);
990     }
991
992     private Either<ZipOutputStream, ResponseFormat> writeOperationsArtifactsToCsar(Component component, ZipOutputStream zipstream) {
993         if (checkComponentBeforeOperation(component)) {
994             return Either.left(zipstream);
995         }
996         for (Map.Entry<String, InterfaceDefinition> interfaceEntry : ((Resource) component).getInterfaces().entrySet()) {
997             for (OperationDataDefinition operation : interfaceEntry.getValue().getOperations().values()) {
998                 try {
999                     if (checkComponentBeforeWrite(component, interfaceEntry, operation)) {
1000                         continue;
1001                     }
1002                     final String artifactUUID = operation.getImplementation().getArtifactUUID();
1003                     if (artifactUUID == null) {
1004                         continue;
1005                     }
1006                     final Either<byte[], ActionStatus> artifactFromCassandra = getFromCassandra(artifactUUID);
1007                     final String artifactName = operation.getImplementation().getArtifactName();
1008                     if (artifactFromCassandra.isRight()) {
1009                         LOGGER.error(ARTIFACT_NAME_UNIQUE_ID, artifactName, artifactUUID);
1010                         LOGGER.error("Failed to get {} payload from DB reason: {}", artifactName, artifactFromCassandra.right().value());
1011                         return Either.right(componentsUtils.getResponseFormat(
1012                             ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION, "Resource", component.getUniqueId(), artifactName, artifactUUID));
1013                     }
1014                     zipstream.putNextEntry(new ZipEntry(OperationArtifactUtil.createOperationArtifactPath(component, null, operation, true)));
1015                     zipstream.write(artifactFromCassandra.left().value());
1016                 } catch (IOException e) {
1017                     LOGGER.error("Component Name {},  Interface Name {}, Operation Name {}", component.getNormalizedName(), interfaceEntry.getKey(),
1018                         operation.getName());
1019                     LOGGER.error("Error while writing the operation's artifacts to the CSAR", e);
1020                     return Either.right(componentsUtils.getResponseFormat(ERROR_DURING_CSAR_CREATION, "Resource", component.getUniqueId()));
1021                 }
1022             }
1023         }
1024         return Either.left(zipstream);
1025     }
1026
1027     private boolean checkComponentBeforeWrite(Component component, Map.Entry<String, InterfaceDefinition> interfaceEntry,
1028                                               OperationDataDefinition operation) {
1029         final ArtifactDataDefinition implementation = operation.getImplementation();
1030         if (implementation == null) {
1031             LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no Operation Implementation found", component.getNormalizedName(),
1032                 interfaceEntry.getValue().getUniqueId(), operation.getName());
1033             return true;
1034         }
1035         final String artifactName = implementation.getArtifactName();
1036         if (artifactName == null) {
1037             LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no artifact found", component.getNormalizedName(),
1038                 interfaceEntry.getValue().getUniqueId(), operation.getName());
1039             return true;
1040         }
1041         if (OperationArtifactUtil.artifactNameIsALiteralValue(artifactName)) {
1042             LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - artifact name is a literal value rather than an SDC artifact",
1043                 component.getNormalizedName(), interfaceEntry.getValue().getUniqueId(), operation.getName());
1044             return true;
1045         }
1046         return false;
1047     }
1048
1049     private boolean checkComponentBeforeOperation(Component component) {
1050         if (component instanceof Service) {
1051             return true;
1052         }
1053         if (Objects.isNull(((Resource) component).getInterfaces())) {
1054             LOGGER.debug("Component Name {}- no interfaces found", component.getNormalizedName());
1055             return true;
1056         }
1057         return false;
1058     }
1059
1060     private Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath(final Component mainComponent,
1061                                                                                       final ArtifactsInfo currArtifactsInfo,
1062                                                                                       final ZipOutputStream zip, final String path,
1063                                                                                       final boolean isInCertificationRequest) throws IOException {
1064         final Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfo = currArtifactsInfo.getArtifactsInfo();
1065         for (final ArtifactGroupTypeEnum artifactGroupTypeEnum : artifactsInfo.keySet()) {
1066             final String groupTypeFolder = path + WordUtils.capitalizeFully(artifactGroupTypeEnum.getType()) + PATH_DELIMITER;
1067             final Map<String, List<ArtifactDefinition>> artifactTypesMap = artifactsInfo.get(artifactGroupTypeEnum);
1068             for (final String artifactType : artifactTypesMap.keySet()) {
1069                 final List<ArtifactDefinition> artifactDefinitionList = artifactTypesMap.get(artifactType);
1070                 String artifactTypeFolder = groupTypeFolder + artifactType + PATH_DELIMITER;
1071                 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType) && path.contains(ARTIFACTS_PATH + RESOURCES_PATH)) {
1072                     // Ignore this packaging as BPMN artifacts needs to be packaged in different manner
1073                     continue;
1074                 }
1075                 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType)) {
1076                     artifactTypeFolder += OperationArtifactUtil.BPMN_ARTIFACT_PATH + File.separator;
1077                 } else if (ArtifactTypeEnum.ONBOARDED_PACKAGE.getType().equals(artifactType)) {
1078                     // renaming legacy folder ONBOARDED_PACKAGE to the new folder ETSI_PACKAGE
1079                     artifactTypeFolder = artifactTypeFolder
1080                         .replace(ArtifactTypeEnum.ONBOARDED_PACKAGE.getType(), ArtifactTypeEnum.ETSI_PACKAGE.getType());
1081                 }
1082                 // TODO: We should not do this but in order to keep this refactoring small enough,
1083
1084                 // we'll leave this as is for now
1085                 List<ArtifactDefinition> collect = filterArtifactDefinitionToZip(mainComponent, artifactDefinitionList, isInCertificationRequest)
1086                     .collect(Collectors.toList());
1087                 for (ArtifactDefinition ad : collect) {
1088                     zip.putNextEntry(new ZipEntry(artifactTypeFolder + ad.getArtifactName()));
1089                     zip.write(ad.getPayloadData());
1090                 }
1091             }
1092         }
1093         return Either.left(zip);
1094     }
1095
1096     private Stream<ArtifactDefinition> filterArtifactDefinitionToZip(Component mainComponent, List<ArtifactDefinition> artifactDefinitionList,
1097                                                                      boolean isInCertificationRequest) {
1098         return artifactDefinitionList.stream().filter(shouldBeInZip(isInCertificationRequest, mainComponent)).map(this::fetchPayLoadData)
1099             .filter(Either::isLeft).map(e -> e.left().value());
1100     }
1101
1102     private Predicate<ArtifactDefinition> shouldBeInZip(boolean isInCertificationRequest, Component component) {
1103         return artifactDefinition -> !(!isInCertificationRequest && component.isService() && artifactDefinition.isHeatEnvType() || artifactDefinition
1104             .hasNoMandatoryEsId());
1105     }
1106
1107     private Either<ArtifactDefinition, ActionStatus> fetchPayLoadData(ArtifactDefinition ad) {
1108         byte[] payloadData = ad.getPayloadData();
1109         if (payloadData == null) {
1110             return getFromCassandra(ad.getEsId()).left().map(pd -> {
1111                 ad.setPayload(pd);
1112                 return ad;
1113             }).right().map(as -> {
1114                 LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, ad.getArtifactName(), ad.getUniqueId());
1115                 LOGGER.debug("Failed to get {} payload from DB reason: {}", ad.getArtifactName(), as);
1116                 return as;
1117             });
1118         } else {
1119             return Either.left(ad);
1120         }
1121     }
1122
1123     private Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath(Component mainComponent,
1124                                                                                            ComponentArtifacts componentArtifacts,
1125                                                                                            ZipOutputStream zipstream, String currentPath,
1126                                                                                            boolean isInCertificationRequest) throws IOException {
1127         Map<String, ComponentTypeArtifacts> componentTypeArtifacts = componentArtifacts.getComponentTypeArtifacts();
1128         //Keys are defined:
1129
1130         //<Inner Asset TOSCA name (e.g. VFC name)> folder name: <Inner Asset TOSCA name (e.g. VFC name)>_v<version>.
1131
1132         //E.g. "org.openecomp.resource.vf.vipr_atm_v1.0"
1133         Set<String> componentTypeArtifactsKeys = componentTypeArtifacts.keySet();
1134         for (String keyAssetName : componentTypeArtifactsKeys) {
1135             ComponentTypeArtifacts componentInstanceArtifacts = componentTypeArtifacts.get(keyAssetName);
1136             ArtifactsInfo componentArtifacts2 = componentInstanceArtifacts.getComponentArtifacts();
1137             String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
1138             Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent,
1139                 componentArtifacts2, zipstream, pathWithAssetName, isInCertificationRequest);
1140             if (writeArtifactsInfoToSpecifiedPath.isRight()) {
1141                 return writeArtifactsInfoToSpecifiedPath;
1142             }
1143         }
1144         return Either.left(zipstream);
1145     }
1146
1147     private Either<ToscaRepresentation, ResponseFormat> generateToscaRepresentation(Component component, boolean isSkipImports) {
1148         return toscaExportUtils.exportComponent(component, isSkipImports).right().map(toscaError -> {
1149             LOGGER.debug("exportComponent failed {}", toscaError);
1150             return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
1151         });
1152     }
1153
1154     private String createToscaBlock0(String metaFileVersion, String csarVersion, String createdBy, String entryDef, boolean isAsdPackage,
1155                                      String definitionsPath, boolean isSkipImports) {
1156         final String block0template = "TOSCA-Meta-File-Version: %s\nCSAR-Version: %s\nCreated-By: %s\nEntry-Definitions: "
1157             + definitionsPath + "%s\n%s\n" + (!isSkipImports ? "Name: csar.meta\nContent-Type: text/plain\n" : "");
1158         return String.format(block0template, metaFileVersion, csarVersion, createdBy, entryDef, isAsdPackage ? "entry_definition_type: asd" : "");
1159     }
1160
1161     private class CsarDefinition {
1162
1163         private ComponentArtifacts componentArtifacts;
1164
1165         // add list of tosca artifacts and meta describes CSAR zip root
1166         public CsarDefinition(ComponentArtifacts componentArtifacts) {
1167             this.componentArtifacts = componentArtifacts;
1168         }
1169
1170         public ComponentArtifacts getComponentArtifacts() {
1171             return componentArtifacts;
1172         }
1173     }
1174
1175     private class ComponentArtifacts {
1176
1177         //artifacts of the component and CI's artifacts contained in it's composition (represents Informational, Deployment & Resource folders of main component)
1178         private ComponentTypeArtifacts mainTypeAndCIArtifacts;
1179         //artifacts of all component types mapped by their tosca name
1180         private Map<String, ComponentTypeArtifacts> componentTypeArtifacts;
1181
1182         public ComponentArtifacts() {
1183             mainTypeAndCIArtifacts = new ComponentTypeArtifacts();
1184             componentTypeArtifacts = new HashMap<>();
1185         }
1186
1187         public ComponentTypeArtifacts getMainTypeAndCIArtifacts() {
1188             return mainTypeAndCIArtifacts;
1189         }
1190
1191         public void setMainTypeAndCIArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
1192             this.mainTypeAndCIArtifacts = componentInstanceArtifacts;
1193         }
1194
1195         public Map<String, ComponentTypeArtifacts> getComponentTypeArtifacts() {
1196             return componentTypeArtifacts;
1197         }
1198     }
1199
1200     /**
1201      * The artifacts of the component and of all its composed instances
1202      */
1203     private class ComponentTypeArtifacts {
1204
1205         private ArtifactsInfo componentArtifacts;    //component artifacts (describes the Informational Deployment folders)
1206
1207         private Map<String, ArtifactsInfo> componentInstancesArtifacts;        //artifacts of the composed instances mapped by the resourceInstance normalized name (describes the Resources folder)
1208
1209         public ComponentTypeArtifacts() {
1210             componentArtifacts = new ArtifactsInfo();
1211             componentInstancesArtifacts = new HashMap<>();
1212         }
1213
1214         public ArtifactsInfo getComponentArtifacts() {
1215             return componentArtifacts;
1216         }
1217
1218         public void setComponentArtifacts(ArtifactsInfo artifactsInfo) {
1219             this.componentArtifacts = artifactsInfo;
1220         }
1221
1222         public Map<String, ArtifactsInfo> getComponentInstancesArtifacts() {
1223             return componentInstancesArtifacts;
1224         }
1225
1226         public void addComponentInstancesArtifacts(String normalizedName, ArtifactsInfo artifactsInfo) {
1227             componentInstancesArtifacts.put(normalizedName, artifactsInfo);
1228         }
1229     }
1230
1231     /**
1232      * The artifacts Definition saved by their structure
1233      */
1234     private class ArtifactsInfo {
1235         //Key is the type of artifacts(Informational/Deployment)
1236
1237         //Value is a map between an artifact type and a list of all artifacts of this type
1238         private Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfoField;
1239
1240         public ArtifactsInfo() {
1241             this.artifactsInfoField = new EnumMap<>(ArtifactGroupTypeEnum.class);
1242         }
1243
1244         public Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> getArtifactsInfo() {
1245             return artifactsInfoField;
1246         }
1247
1248         public void addArtifactsToGroup(ArtifactGroupTypeEnum artifactGroup, Map<String, List<ArtifactDefinition>> artifactsDefinition) {
1249             if (artifactsInfoField.get(artifactGroup) == null) {
1250                 artifactsInfoField.put(artifactGroup, artifactsDefinition);
1251             } else {
1252                 Map<String, List<ArtifactDefinition>> artifactTypeEnumListMap = artifactsInfoField.get(artifactGroup);
1253                 artifactTypeEnumListMap.putAll(artifactsDefinition);
1254                 artifactsInfoField.put(artifactGroup, artifactTypeEnumListMap);
1255             }
1256         }
1257
1258         public boolean isEmpty() {
1259             return artifactsInfoField.isEmpty();
1260         }
1261
1262         public boolean isNotEmpty() {
1263             return !isEmpty();
1264         }
1265     }
1266
1267     public static class ToscaErrorException extends Exception {
1268
1269         ToscaErrorException(ToscaError error) {
1270             super("Error while exporting component's interface (toscaError:" + error + ")");
1271         }
1272     }
1273
1274 }