Catalog alignment
[sdc.git] / catalog-be / src / main / java / org / openecomp / sdc / be / tosca / CsarUtils.java
1 /*-
2  * ============LICENSE_START=======================================================
3  * SDC
4  * ================================================================================
5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6  * ================================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ============LICENSE_END=========================================================
19  */
20
21 package org.openecomp.sdc.be.tosca;
22
23
24 import fj.data.Either;
25 import org.apache.commons.codec.binary.Base64;
26 import org.apache.commons.collections.MapUtils;
27 import org.apache.commons.io.output.ByteArrayOutputStream;
28 import org.apache.commons.lang.WordUtils;
29 import org.apache.commons.lang3.tuple.ImmutablePair;
30 import org.apache.commons.lang3.tuple.ImmutableTriple;
31 import org.apache.commons.lang3.tuple.Triple;
32 import org.openecomp.sdc.be.components.impl.ImportUtils;
33 import org.openecomp.sdc.be.config.Configuration.ArtifactTypeConfig;
34 import org.openecomp.sdc.be.config.ConfigurationManager;
35 import org.openecomp.sdc.be.dao.api.ActionStatus;
36 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
37 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
38 import org.openecomp.sdc.be.dao.cassandra.SdcSchemaFilesCassandraDao;
39 import org.openecomp.sdc.be.datatypes.elements.OperationDataDefinition;
40 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
41 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
42 import org.openecomp.sdc.be.impl.ComponentsUtils;
43 import org.openecomp.sdc.be.model.ArtifactDefinition;
44 import org.openecomp.sdc.be.model.Component;
45 import org.openecomp.sdc.be.model.ComponentInstance;
46 import org.openecomp.sdc.be.model.InterfaceDefinition;
47 import org.openecomp.sdc.be.model.LifecycleStateEnum;
48 import org.openecomp.sdc.be.model.Resource;
49 import org.openecomp.sdc.be.model.Service;
50 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
51 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
52 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
53 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
54 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
55 import org.openecomp.sdc.be.resources.data.SdcSchemaFilesData;
56 import org.openecomp.sdc.be.tosca.model.ToscaTemplate;
57 import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil;
58 import org.openecomp.sdc.be.utils.CommonBeUtils;
59 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
60 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
61 import org.openecomp.sdc.common.impl.ExternalConfiguration;
62 import org.openecomp.sdc.common.log.elements.LoggerSupportability;
63 import org.openecomp.sdc.common.log.enums.LoggerSupportabilityActions;
64 import org.openecomp.sdc.common.log.enums.StatusCode;
65 import org.openecomp.sdc.common.log.wrappers.Logger;
66 import org.openecomp.sdc.common.util.GeneralUtility;
67 import org.openecomp.sdc.common.util.ValidationUtils;
68 import org.openecomp.sdc.common.zip.ZipUtils;
69 import org.openecomp.sdc.exception.ResponseFormat;
70 import org.springframework.beans.factory.annotation.Autowired;
71
72 import java.io.BufferedOutputStream;
73 import java.io.ByteArrayInputStream;
74 import java.io.File;
75 import java.io.IOException;
76 import java.util.ArrayList;
77 import java.util.Arrays;
78 import java.util.EnumMap;
79 import java.util.HashMap;
80 import java.util.HashSet;
81 import java.util.List;
82 import java.util.Map;
83 import java.util.Map.Entry;
84 import java.util.Objects;
85 import java.util.Set;
86 import java.util.regex.Matcher;
87 import java.util.regex.Pattern;
88 import java.util.zip.ZipEntry;
89 import java.util.zip.ZipInputStream;
90 import java.util.zip.ZipOutputStream;
91
92 /**
93  * @author tg851x
94  *
95  */
96 @org.springframework.stereotype.Component("csar-utils")
97 public class CsarUtils {
98     private static final Logger log = Logger.getLogger(CsarUtils.class);
99     private static final LoggerSupportability loggerSupportability = LoggerSupportability.getLogger(CsarUtils.class.getName());
100     private static final String PATH_DELIMITER = "/";
101     @Autowired
102     private SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
103     @Autowired
104     private ArtifactCassandraDao artifactCassandraDao;
105     @Autowired
106     private ComponentsUtils componentsUtils;
107     @Autowired
108     private ToscaExportHandler toscaExportUtils;
109     @Autowired
110     protected ToscaOperationFacade toscaOperationFacade;
111
112     private static final String CONFORMANCE_LEVEL = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
113     private static final String SDC_VERSION = ExternalConfiguration.getAppVersion();
114     public static final String ARTIFACTS_PATH = "Artifacts/";
115     private static final String RESOURCES_PATH = "Resources/";
116     private static final String DEFINITIONS_PATH = "Definitions/";
117     public static final String WORKFLOW_ARTIFACT_DIR = "Workflows"+File.separator+"BPMN"+File.separator;
118     public static final String DEPLOYMENT_ARTIFACTS_DIR = "Deployment"+File.separator;
119     private static final String CSAR_META_VERSION = "1.0";
120     private static final String CSAR_META_PATH_FILE_NAME = "csar.meta";
121     private static final String TOSCA_META_PATH_FILE_NAME = "TOSCA-Metadata/TOSCA.meta";
122     private static final String TOSCA_META_VERSION = "1.0";
123     private static final String CSAR_VERSION = "1.1";
124     public static final String ARTIFACTS = "Artifacts";
125     private static final String DEFINITION = "Definitions";
126     private static final String DEL_PATTERN = "([/\\\\]+)";
127     private static final String WORD_PATTERN = "\\w\\_\\-\\.\\s]+)";
128     public static final String VALID_ENGLISH_ARTIFACT_NAME = "([" + WORD_PATTERN;
129     private static final String VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS = "([\\d" + WORD_PATTERN;
130     private static final String ARTIFACT_NAME_UNIQUE_ID = "ArtifactName {}, unique ID {}";
131
132     private static final String VFC_NODE_TYPE_ARTIFACTS_PATH_PATTERN = ARTIFACTS + DEL_PATTERN +
133                                                                               ImportUtils.Constants.USER_DEFINED_RESOURCE_NAMESPACE_PREFIX +
134                                                                               VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS + DEL_PATTERN +
135                                                                               VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS + DEL_PATTERN +
136                                                                               VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS + DEL_PATTERN +
137                                                                               VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS;
138
139     public static final String VF_NODE_TYPE_ARTIFACTS_PATH_PATTERN = ARTIFACTS + DEL_PATTERN+
140                                                                              // Artifact Group (i.e Deployment/Informational)
141                                                                              VALID_ENGLISH_ARTIFACT_NAME + DEL_PATTERN +
142                                                                              // Artifact Type
143                                                                              VALID_ENGLISH_ARTIFACT_NAME  + DEL_PATTERN +
144                                                                              // Artifact Any File Name
145                                                                              ".+";
146
147     public static final String SERVICE_TEMPLATE_PATH_PATTERN = DEFINITION + DEL_PATTERN+
148                                                                        // Service Template File Name
149                                                                        VALID_ENGLISH_ARTIFACT_NAME;
150
151     public static final String ARTIFACT_CREATED_FROM_CSAR = "Artifact created from csar";
152     private static final String BLOCK_0_TEMPLATE = "SDC-TOSCA-Meta-File-Version: %s\nSDC-TOSCA-Definitions-Version: %s\n";
153
154     private String versionFirstThreeOctets;
155
156     public CsarUtils() {
157         if(SDC_VERSION != null && !SDC_VERSION.isEmpty()){
158             Matcher matcher = Pattern.compile("(?!\\.)(\\d+(\\.\\d+)+)(?![\\d\\.])").matcher(SDC_VERSION);
159             matcher.find();
160             setVersionFirstThreeOctets(matcher.group(0));
161         } else {
162             setVersionFirstThreeOctets("");
163         }
164     }
165
166     /**
167      *
168      * @param component
169      * @param getFromCS
170      * @param isInCertificationRequest
171      * @return
172      */
173     public Either<byte[], ResponseFormat> createCsar(Component component, boolean getFromCS, boolean isInCertificationRequest) {
174         loggerSupportability.log(LoggerSupportabilityActions.GENERATE_CSAR, StatusCode.STARTED,"Starting to create Csar for component {} ",component.getName());
175         final String createdBy = component.getCreatorFullName();
176         String fileName;
177         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
178         ArtifactDefinition artifactDefinition = toscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
179         fileName = artifactDefinition.getArtifactName();
180
181         String toscaConformanceLevel = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
182         String csarBlock0 = createCsarBlock0(CSAR_META_VERSION, toscaConformanceLevel);
183         byte[] csarBlock0Byte = csarBlock0.getBytes();
184
185         final String toscaBlock0 = createToscaBlock0(TOSCA_META_VERSION, CSAR_VERSION, createdBy, fileName);
186         byte[] toscaBlock0Byte = toscaBlock0.getBytes();
187
188         Either<byte[], ResponseFormat> generateCsarZipResponse = generateCsarZip(csarBlock0Byte, toscaBlock0Byte, component, getFromCS, isInCertificationRequest);
189
190         if (generateCsarZipResponse.isRight()) {
191             return Either.right(generateCsarZipResponse.right().value());
192         }
193         loggerSupportability.log(LoggerSupportabilityActions.GENERATE_CSAR, StatusCode.COMPLETE,"Ended create Csar for component {} ",component.getName());
194         return Either.left(generateCsarZipResponse.left().value());
195     }
196
197     private Either<byte[], ResponseFormat> generateCsarZip(byte[] csarBlock0Byte, byte[] toscaBlock0Byte, Component component, boolean getFromCS, boolean isInCertificationRequest) {
198         try (ByteArrayOutputStream out = new ByteArrayOutputStream();
199              ZipOutputStream zip = new ZipOutputStream(out)) {
200             zip.putNextEntry(new ZipEntry(CSAR_META_PATH_FILE_NAME));
201             zip.write(csarBlock0Byte);
202             zip.putNextEntry(new ZipEntry(TOSCA_META_PATH_FILE_NAME));
203             zip.write(toscaBlock0Byte);
204             Either<ZipOutputStream, ResponseFormat> populateZip = populateZip(component, getFromCS, zip, isInCertificationRequest);
205             if (populateZip.isRight()) {
206                 log.debug("Failed to populate CSAR zip file {}. Please fix DB table accordingly ", populateZip.right().value());
207                 return Either.right(populateZip.right().value());
208             }
209
210             zip.finish();
211             byte[] byteArray = out.toByteArray();
212
213             return Either.left(byteArray);
214         } catch (IOException e) {
215             log.debug("Failed with IOexception to create CSAR zip for component {}. Please fix DB table accordingly ", component.getUniqueId(), e);
216
217             ResponseFormat responseFormat = componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR);
218             return Either.right(responseFormat);
219         }
220 }
221
222     private Either<ZipOutputStream, ResponseFormat> populateZip(Component component, boolean getFromCS, ZipOutputStream zip, boolean isInCertificationRequest) throws IOException {
223
224         LifecycleStateEnum lifecycleState = component.getLifecycleState();
225         String componentYaml;
226         Either<ToscaRepresentation, ToscaError> exportComponent;
227         byte[] mainYaml;
228         // <file name, cassandraId, component>
229         List<Triple<String, String, Component>> dependencies = null;
230
231         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
232         ArtifactDefinition artifactDefinition = toscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
233         String fileName = artifactDefinition.getArtifactName();
234
235         if (getFromCS || !(lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN || lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT)) {
236             String cassandraId = artifactDefinition.getEsId();
237             Either<byte[], ActionStatus> fromCassandra = getFromCassandra(cassandraId);
238             if (fromCassandra.isRight()) {
239                 log.debug(ARTIFACT_NAME_UNIQUE_ID, artifactDefinition.getArtifactName(), artifactDefinition.getUniqueId());
240                 ResponseFormat responseFormat = componentsUtils.getResponseFormat(fromCassandra.right().value());
241                 return Either.right(responseFormat);
242             }
243             mainYaml = fromCassandra.left().value();
244
245         } else {
246             exportComponent = toscaExportUtils.exportComponent(component);
247             if (exportComponent.isRight()) {
248                 log.debug("exportComponent failed", exportComponent.right().value());
249                 ActionStatus convertedFromToscaError = componentsUtils.convertFromToscaError(exportComponent.right().value());
250                 ResponseFormat responseFormat = componentsUtils.getResponseFormat(convertedFromToscaError);
251                 return Either.right(responseFormat);
252             }
253             ToscaRepresentation exportResult = exportComponent.left().value();
254             componentYaml = exportResult.getMainYaml();
255             mainYaml = componentYaml.getBytes();
256             dependencies = exportResult.getDependencies();
257         }
258
259         zip.putNextEntry(new ZipEntry(DEFINITIONS_PATH + fileName));
260         zip.write(mainYaml);
261         //US798487 - Abstraction of complex types
262         if (!ModelConverter.isAtomicComponent(component)){
263             log.debug("Component {} is complex - generating abstract type for it..", component.getName());
264                         writeComponentInterface(component, zip, fileName, false);
265         }
266
267         if (dependencies == null) {
268             Either<ToscaTemplate, ToscaError> dependenciesRes = toscaExportUtils.getDependencies(component);
269             if (dependenciesRes.isRight()) {
270                 log.debug("Failed to retrieve dependencies for component {}, error {}", component.getUniqueId(),
271                         dependenciesRes.right().value());
272                 ActionStatus convertFromToscaError = componentsUtils.convertFromToscaError(dependenciesRes.right().value());
273                 ResponseFormat responseFormat = componentsUtils.getResponseFormat(convertFromToscaError);
274                 return Either.right(responseFormat);
275             }
276             dependencies = dependenciesRes.left().value().getDependencies();
277         }
278
279         //UID <cassandraId,filename,component>
280         Map<String, ImmutableTriple<String,String, Component>> innerComponentsCache = new HashMap<>();
281
282         Either<ZipOutputStream, ResponseFormat> responseFormat = getZipOutputStreamResponseFormatEither(zip, dependencies, innerComponentsCache);
283         if (responseFormat != null) return responseFormat;
284
285         //retrieve SDC.zip from Cassandra
286         Either<byte[], ResponseFormat> latestSchemaFilesFromCassandra = getLatestSchemaFilesFromCassandra();
287
288         if(latestSchemaFilesFromCassandra.isRight()){
289             log.error("Error retrieving SDC Schema files from cassandra" );
290             return Either.right(latestSchemaFilesFromCassandra.right().value());
291         }
292
293         //add files from retrieved SDC.zip to Definitions folder in CSAR
294         Either<ZipOutputStream, ResponseFormat> addSchemaFilesFromCassandra = addSchemaFilesFromCassandra(zip, latestSchemaFilesFromCassandra.left().value());
295
296         if(addSchemaFilesFromCassandra.isRight()){
297             return addSchemaFilesFromCassandra;
298         }
299
300         Either<CsarDefinition, ResponseFormat> collectedComponentCsarDefinition = collectComponentCsarDefinition(component);
301
302         if (collectedComponentCsarDefinition.isRight()) {
303             return Either.right(collectedComponentCsarDefinition.right().value());
304         }
305
306         return writeAllFilesToCsar(component, collectedComponentCsarDefinition.left().value(), zip, isInCertificationRequest);
307     }
308
309     private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither(ZipOutputStream zip, List<Triple<String, String, Component>> dependencies, Map<String, ImmutableTriple<String, String, Component>> innerComponentsCache) throws IOException {
310         String fileName;
311         if (dependencies != null && !dependencies.isEmpty()) {
312             for (Triple<String, String, Component> d : dependencies) {
313                 String cassandraId = d.getMiddle();
314                 Component childComponent = d.getRight();
315                 Either<byte[], ActionStatus> entryData = getEntryData(cassandraId, childComponent);
316
317                 if (entryData.isRight()) {
318                     ResponseFormat responseFormat = componentsUtils.getResponseFormat(entryData.right().value());
319                     return Either.right(responseFormat);
320                 }
321
322                 //fill innerComponentsCache
323                 fileName = d.getLeft();
324                 addComponentToCache(innerComponentsCache, cassandraId, fileName, childComponent);
325                 addInnerComponentsToCache(innerComponentsCache, childComponent);
326             }
327
328             //add inner components to CSAR
329             Either<ZipOutputStream, ResponseFormat> responseFormat = addInnerComponentsToCSAR(zip, innerComponentsCache);
330             if (responseFormat != null) return responseFormat;
331         }
332         return null;
333     }
334
335     private Either<ZipOutputStream, ResponseFormat> addInnerComponentsToCSAR(ZipOutputStream zip, Map<String, ImmutableTriple<String, String, Component>> innerComponentsCache) throws IOException {
336         for (Entry<String, ImmutableTriple<String, String, Component>> innerComponentTripleEntry : innerComponentsCache.entrySet()) {
337
338             ImmutableTriple<String, String, Component> innerComponentTriple = innerComponentTripleEntry.getValue();
339
340             Component innerComponent = innerComponentTriple.getRight();
341             String icFileName = innerComponentTriple.getMiddle();
342
343             // add component to zip
344             Either<byte[], ActionStatus> entryData = getEntryData(innerComponentTriple.getLeft(), innerComponent);
345             if (entryData.isRight()) {
346                 ResponseFormat responseFormat = componentsUtils.getResponseFormat(entryData.right().value());
347                 log.debug("Failed adding to zip component {}, error {}", innerComponentTriple.getLeft(),
348                         entryData.right().value());
349                 return Either.right(responseFormat);
350             }
351             byte[] content = entryData.left().value();
352             zip.putNextEntry(new ZipEntry(DEFINITIONS_PATH + icFileName));
353             zip.write(content);
354
355             // add component interface to zip
356             if (!ModelConverter.isAtomicComponent(innerComponent)) {
357                                         writeComponentInterface(innerComponent, zip, icFileName, true);
358             }
359         }
360         return null;
361     }
362
363     private Either<ZipOutputStream, ResponseFormat> addSchemaFilesFromCassandra(ZipOutputStream zip, byte[] schemaFileZip) {
364
365             final int initSize = 2048;
366
367             log.debug("Starting copy from Schema file zip to CSAR zip");
368             try (final ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(schemaFileZip));
369                  final ByteArrayOutputStream out = new ByteArrayOutputStream();
370                  final BufferedOutputStream bos = new BufferedOutputStream(out, initSize)) {
371
372                 ZipEntry entry;
373                 while ((entry = zipStream.getNextEntry()) != null) {
374                     ZipUtils.checkForZipSlipInRead(entry);
375                     final String entryName = entry.getName();
376                     int readSize = initSize;
377                     final byte[] entryData = new byte[initSize];
378
379                     while ((readSize = zipStream.read(entryData, 0, readSize)) != -1) {
380                         bos.write(entryData, 0, readSize);
381                     }
382
383                     bos.flush();
384                     out.flush();
385                     zip.putNextEntry(new ZipEntry(DEFINITIONS_PATH + entryName));
386                     zip.write(out.toByteArray());
387                     zip.flush();
388                     out.reset();
389                 }
390             } catch (final Exception e) {
391                 log.error("Error while writing the SDC schema file to the CSAR", e);
392                 return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
393             }
394
395             log.debug("Finished coppy from Schema file zip to CSAR zip");
396             return Either.left(zip);
397         }
398
399
400
401     private void addInnerComponentsToCache(Map<String, ImmutableTriple<String, String, Component>> componentCache,
402             Component childComponent) {
403
404         List<ComponentInstance> instances = childComponent.getComponentInstances();
405
406         if(instances != null) {
407             instances.forEach(ci -> {
408                 ImmutableTriple<String, String, Component> componentRecord = componentCache.get(ci.getComponentUid());
409                 if (componentRecord == null) {
410                     // all resource must be only once!
411                     Either<Resource, StorageOperationStatus> resource = toscaOperationFacade.getToscaElement(ci.getComponentUid());
412                     Component componentRI = checkAndAddComponent(componentCache, ci, resource);
413
414                     //if not atomic - insert inner components as well
415                     if(!ModelConverter.isAtomicComponent(componentRI)) {
416                         addInnerComponentsToCache(componentCache, componentRI);
417                     }
418                 }
419             });
420         }
421     }
422
423     private Component checkAndAddComponent(Map<String, ImmutableTriple<String, String, Component>> componentCache, ComponentInstance ci, Either<Resource, StorageOperationStatus> resource) {
424         if (resource.isRight()) {
425             log.debug("Failed to fetch resource with id {} for instance {}", ci.getComponentUid(), ci.getName());
426         }
427         Component componentRI = resource.left().value();
428
429         Map<String, ArtifactDefinition> childToscaArtifacts = componentRI.getToscaArtifacts();
430         ArtifactDefinition childArtifactDefinition = childToscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
431         if (childArtifactDefinition != null) {
432             //add to cache
433             addComponentToCache(componentCache, childArtifactDefinition.getEsId(), childArtifactDefinition.getArtifactName(), componentRI);
434         }
435         return componentRI;
436     }
437
438     private void addComponentToCache(Map<String, ImmutableTriple<String, String, Component>> componentCache,
439             String id, String fileName, Component component) {
440
441         ImmutableTriple<String, String, Component> cachedComponent = componentCache.get(component.getInvariantUUID());
442         if (cachedComponent == null || CommonBeUtils.compareAsdcComponentVersions(component.getVersion(), cachedComponent.getRight().getVersion())) {
443             componentCache.put(component.getInvariantUUID(),
444                     new ImmutableTriple<>(id, fileName, component));
445
446             if(cachedComponent != null) {
447                 //overwriting component with newer version
448                 log.warn("Overwriting component invariantID {} of version {} with a newer version {}", id, cachedComponent.getRight().getVersion(), component.getVersion());
449             }
450         }
451     }
452         private Either<ZipOutputStream, ResponseFormat> writeComponentInterface(Component component, ZipOutputStream zip,
453                         String fileName, boolean isAssociatedComponent) {
454                 try {
455                         Either<ToscaRepresentation, ToscaError> componentInterface = toscaExportUtils
456                                         .exportComponentInterface(component, isAssociatedComponent);
457                         ToscaRepresentation componentInterfaceYaml = componentInterface.left().value();
458                         String mainYaml = componentInterfaceYaml.getMainYaml();
459                         String interfaceFileName = DEFINITIONS_PATH + ToscaExportHandler.getInterfaceFilename(fileName);
460
461                         zip.putNextEntry(new ZipEntry(interfaceFileName));
462                         zip.write(mainYaml.getBytes());
463
464                 } catch (Exception e) {
465                         log.error("#writeComponentInterface - zip writing failed with error: ", e);
466                         return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
467                 }
468
469                 return Either.left(zip);
470         }
471
472         private Either<byte[], ActionStatus> getEntryData(String cassandraId, Component childComponent) {
473                 byte[] content;
474                 if (cassandraId == null || cassandraId.isEmpty()) {
475                         Either<ToscaRepresentation, ToscaError> exportRes = toscaExportUtils.exportComponent(childComponent);
476                         if (exportRes.isRight()) {
477                                 log.debug("Failed to export tosca template for child component {} error {}",
478                                                 childComponent.getUniqueId(), exportRes.right().value());
479                                 return Either.right(componentsUtils.convertFromToscaError(exportRes.right().value()));
480                         }
481                         content = exportRes.left().value().getMainYaml().getBytes();
482                 } else {
483                         Either<byte[], ActionStatus> fromCassandra = getFromCassandra(cassandraId);
484                         if (fromCassandra.isRight()) {
485                                 return Either.right(fromCassandra.right().value());
486                         } else {
487                                 content = fromCassandra.left().value();
488                         }
489                 }
490                 return Either.left(content);
491         }
492
493     private Either<byte[], ResponseFormat> getLatestSchemaFilesFromCassandra() {
494         Either<List<SdcSchemaFilesData>, CassandraOperationStatus> specificSchemaFiles = sdcSchemaFilesCassandraDao.getSpecificSchemaFiles(getVersionFirstThreeOctets(), CONFORMANCE_LEVEL);
495
496         if(specificSchemaFiles.isRight()){
497             log.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}. Please fix DB table accordingly.", getVersionFirstThreeOctets(), CONFORMANCE_LEVEL);
498             StorageOperationStatus storageStatus = DaoStatusConverter.convertCassandraStatusToStorageStatus(specificSchemaFiles.right().value());
499             ActionStatus convertedFromStorageResponse = componentsUtils.convertFromStorageResponse(storageStatus);
500             return Either.right(componentsUtils.getResponseFormat(convertedFromStorageResponse));
501         }
502
503         List<SdcSchemaFilesData> listOfSchemas = specificSchemaFiles.left().value();
504
505         if(listOfSchemas.isEmpty()){
506             log.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}", getVersionFirstThreeOctets(), CONFORMANCE_LEVEL);
507             return Either.right(componentsUtils.getResponseFormat(ActionStatus.TOSCA_SCHEMA_FILES_NOT_FOUND, getVersionFirstThreeOctets(), CONFORMANCE_LEVEL));
508         }
509
510         SdcSchemaFilesData schemaFile = listOfSchemas.iterator().next();
511
512         return Either.left(schemaFile.getPayloadAsArray());
513     }
514
515     private Either<byte[], ActionStatus> getFromCassandra(String cassandraId) {
516         Either<DAOArtifactData, CassandraOperationStatus> artifactResponse = artifactCassandraDao.getArtifact(cassandraId);
517
518         if (artifactResponse.isRight()) {
519             log.debug("Failed to fetch artifact from Cassandra by id {} error {} ", cassandraId, artifactResponse.right().value());
520
521             StorageOperationStatus storageStatus = DaoStatusConverter.convertCassandraStatusToStorageStatus(artifactResponse.right().value());
522             ActionStatus convertedFromStorageResponse = componentsUtils.convertFromStorageResponse(storageStatus);
523             return Either.right(convertedFromStorageResponse);
524         }
525         DAOArtifactData artifactData = artifactResponse.left().value();
526         return Either.left(artifactData.getDataAsArray());
527     }
528
529     private String createCsarBlock0(String metaFileVersion, String toscaConformanceLevel) {
530         return String.format(BLOCK_0_TEMPLATE, metaFileVersion, toscaConformanceLevel);
531     }
532
533     private String createToscaBlock0(String metaFileVersion, String csarVersion, String createdBy, String entryDef) {
534         final String block0template = "TOSCA-Meta-File-Version: %s\nCSAR-Version: %s\nCreated-By: %s\nEntry-Definitions: Definitions/%s\n\nName: csar.meta\nContent-Type: text/plain\n";
535         return String.format(block0template, metaFileVersion, csarVersion, createdBy, entryDef);
536     }
537
538     /**
539      * Extracts artifacts of VFCs from CSAR
540      *
541      * @param csar
542      * @return Map of <String, List<ArtifactDefinition>> the contains Lists of artifacts according vfcToscaNamespace
543      */
544     public static Map<String, List<ArtifactDefinition>> extractVfcsArtifactsFromCsar(Map<String, byte[]> csar) {
545
546         Map<String, List<ArtifactDefinition>> artifacts = new HashMap<>();
547         if (csar != null) {
548             log.debug("************* Going to extract VFCs artifacts from Csar. ");
549             Map<String, Set<List<String>>> collectedWarningMessages = new HashMap<>();
550             csar.entrySet().stream()
551                 // filter CSAR entry by node type artifact path
552                 .filter(e -> Pattern.compile(VFC_NODE_TYPE_ARTIFACTS_PATH_PATTERN).matcher(e.getKey()).matches())
553                 // extract ArtifactDefinition from CSAR entry for each entry with matching artifact path
554                 .forEach(e -> addExtractedVfcArtifact(extractVfcArtifact(e, collectedWarningMessages), artifacts));
555             // add counter suffix to artifact labels
556             handleWarningMessages(collectedWarningMessages);
557
558         }
559         return artifacts;
560     }
561
562     /**
563      * Print warnings to log
564      *
565      * @param collectedWarningMessages
566      */
567     public static void handleWarningMessages(Map<String, Set<List<String>>> collectedWarningMessages) {
568         collectedWarningMessages.entrySet().stream()
569                                 // for each vfc
570                                 .forEach(e -> e.getValue().stream()
571                                                // add each warning message to log
572                                                .forEach(args -> log.warn(e.getKey(), args.toArray())));
573
574     }
575
576     private static void addExtractedVfcArtifact(ImmutablePair<String, ArtifactDefinition> extractedVfcArtifact, Map<String, List<ArtifactDefinition>> artifacts) {
577         if (extractedVfcArtifact != null) {
578             List<ArtifactDefinition> currArtifactsList;
579             String vfcToscaNamespace = extractedVfcArtifact.getKey();
580             if (artifacts.containsKey(vfcToscaNamespace)) {
581                 currArtifactsList = artifacts.get(vfcToscaNamespace);
582             } else {
583                 currArtifactsList = new ArrayList<>();
584                 artifacts.put(vfcToscaNamespace, currArtifactsList);
585             }
586             currArtifactsList.add(extractedVfcArtifact.getValue());
587         }
588     }
589
590     private static ImmutablePair<String, ArtifactDefinition> extractVfcArtifact(Entry<String, byte[]> entry, Map<String, Set<List<String>>> collectedWarningMessages) {
591         ArtifactDefinition artifact;
592         String[] parsedCsarArtifactPath = entry.getKey().split(PATH_DELIMITER);
593         Either<ArtifactGroupTypeEnum, Boolean> eitherArtifactGroupType = detectArtifactGroupType(parsedCsarArtifactPath[2].toUpperCase(), collectedWarningMessages);
594         if (eitherArtifactGroupType.isLeft()) {
595             artifact = buildArtifactDefinitionFromCsarArtifactPath(entry, collectedWarningMessages, parsedCsarArtifactPath, eitherArtifactGroupType.left().value());
596         } else {
597             return null;
598         }
599         return new ImmutablePair<>(parsedCsarArtifactPath[1], artifact);
600     }
601
602     private static Either<ArtifactGroupTypeEnum, Boolean> detectArtifactGroupType(String groupType, Map<String, Set<List<String>>> collectedWarningMessages) {
603         Either<ArtifactGroupTypeEnum, Boolean> result;
604         try {
605             ArtifactGroupTypeEnum artifactGroupType = ArtifactGroupTypeEnum.findType(groupType.toUpperCase());
606             if (artifactGroupType == null || (artifactGroupType != ArtifactGroupTypeEnum.INFORMATIONAL && artifactGroupType != ArtifactGroupTypeEnum.DEPLOYMENT)) {
607                 String warningMessage = "Warning - unrecognized artifact group type {} was received.";
608                 List<String> messageArguments = new ArrayList<>();
609                 messageArguments.add(groupType);
610                 if (!collectedWarningMessages.containsKey(warningMessage)) {
611                     Set<List<String>> messageArgumentLists = new HashSet<>();
612                     messageArgumentLists.add(messageArguments);
613                     collectedWarningMessages.put(warningMessage, messageArgumentLists);
614                 } else {
615                     collectedWarningMessages.get(warningMessage).add(messageArguments);
616                 }
617
618                 result = Either.right(false);
619             } else {
620
621                 result = Either.left(artifactGroupType);
622             }
623         } catch (Exception e) {
624             log.debug("detectArtifactGroupType failed with exception", e);
625             result = Either.right(false);
626         }
627         return result;
628     }
629
630     private static ArtifactDefinition buildArtifactDefinitionFromCsarArtifactPath(Entry<String, byte[]> entry, Map<String, Set<List<String>>> collectedWarningMessages, String[] parsedCsarArtifactPath, ArtifactGroupTypeEnum artifactGroupType) {
631         ArtifactDefinition artifact;
632         artifact = new ArtifactDefinition();
633         artifact.setArtifactGroupType(artifactGroupType);
634         artifact.setArtifactType(detectArtifactTypeVFC(artifactGroupType, parsedCsarArtifactPath[3], parsedCsarArtifactPath[1], collectedWarningMessages));
635         artifact.setArtifactName(ValidationUtils.normalizeFileName(parsedCsarArtifactPath[parsedCsarArtifactPath.length - 1]));
636         artifact.setPayloadData(Base64.encodeBase64String(entry.getValue()));
637         artifact.setArtifactDisplayName(artifact.getArtifactName().lastIndexOf('.') > 0 ? artifact.getArtifactName().substring(0, artifact.getArtifactName().lastIndexOf('.')) : artifact.getArtifactName());
638         artifact.setArtifactLabel(ValidationUtils.normalizeArtifactLabel(artifact.getArtifactName()));
639         artifact.setDescription(ARTIFACT_CREATED_FROM_CSAR);
640         artifact.setIsFromCsar(true);
641         artifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(entry.getValue()));
642         return artifact;
643     }
644
645     public static final class NonMetaArtifactInfo {
646         private final String path;
647         private final String artifactName;
648         private final String displayName;
649         private final String artifactLabel;
650         private final ArtifactTypeEnum artifactType;
651         private final ArtifactGroupTypeEnum artifactGroupType;
652         private String payloadData;
653         private String artifactChecksum;
654         private String artifactUniqueId;
655         private final boolean isFromCsar;
656
657         public NonMetaArtifactInfo(String artifactName, String path, ArtifactTypeEnum artifactType, ArtifactGroupTypeEnum artifactGroupType, byte[] payloadData, String artifactUniqueId, boolean isFromCsar) {
658             super();
659             this.path = path;
660             this.isFromCsar = isFromCsar;
661             this.artifactName = ValidationUtils.normalizeFileName(artifactName);
662             this.artifactType = artifactType;
663             this.artifactGroupType = artifactGroupType;
664             final int pointIndex = artifactName.lastIndexOf('.');
665             if (pointIndex > 0) {
666                 displayName = artifactName.substring(0, pointIndex);
667             } else {
668                 displayName = artifactName;
669             }
670             this.artifactLabel = ValidationUtils.normalizeArtifactLabel(artifactName);
671             if (payloadData != null) {
672                 this.payloadData = Base64.encodeBase64String(payloadData);
673                 this.artifactChecksum = GeneralUtility.calculateMD5Base64EncodedByByteArray(payloadData);
674             }
675             this.artifactUniqueId = artifactUniqueId;
676         }
677
678         public String getPath() {
679             return path;
680         }
681
682         public String getArtifactName() {
683             return artifactName;
684         }
685
686         public ArtifactTypeEnum getArtifactType() {
687             return artifactType;
688         }
689
690         public String getDisplayName() {
691             return displayName;
692         }
693
694         public ArtifactGroupTypeEnum getArtifactGroupType() {
695             return artifactGroupType;
696         }
697
698         public String getArtifactLabel() {
699             return artifactLabel;
700         }
701
702         public boolean isFromCsar(){
703             return isFromCsar;
704         }
705
706         public String getPayloadData() {
707             return payloadData;
708         }
709
710         public String getArtifactChecksum() {
711             return artifactChecksum;
712         }
713
714         public String getArtifactUniqueId() {
715             return artifactUniqueId;
716         }
717
718         public void setArtifactUniqueId(String artifactUniqueId) {
719             this.artifactUniqueId = artifactUniqueId;
720         }
721
722     }
723
724     /**
725      * This method checks the artifact GroupType & Artifact Type. <br>
726      * if there is any problem warning messages are added to collectedWarningMessages
727      *
728      * @param artifactPath
729      * @param collectedWarningMessages
730      * @return
731      */
732     public static Either<NonMetaArtifactInfo, Boolean> validateNonMetaArtifact(String artifactPath, byte[] payloadData, Map<String, Set<List<String>>> collectedWarningMessages) {
733         Either<NonMetaArtifactInfo, Boolean> ret;
734         try {
735             String[] parsedArtifactPath = artifactPath.split(PATH_DELIMITER);
736             // Validate Artifact Group Type
737             Either<ArtifactGroupTypeEnum, Boolean> eitherGroupType = detectArtifactGroupType(parsedArtifactPath[1], collectedWarningMessages);
738             if (eitherGroupType.isLeft()) {
739                 final ArtifactGroupTypeEnum groupTypeEnum = eitherGroupType.left().value();
740
741                 // Validate Artifact Type
742                 String artifactType = parsedArtifactPath[2];
743                 artifactType = detectArtifactTypeVF(groupTypeEnum, artifactType, collectedWarningMessages);
744
745                 String artifactFileNameType = parsedArtifactPath[3];
746                 ret = Either.left(new NonMetaArtifactInfo(artifactFileNameType, artifactPath, ArtifactTypeEnum.findType(artifactType), groupTypeEnum, payloadData, null, true));
747
748             } else {
749                 ret = Either.right(eitherGroupType.right().value());
750             }
751         } catch (Exception e) {
752             log.debug("detectArtifactGroupType failed with exception", e);
753             ret = Either.right(false);
754         }
755         return ret;
756
757     }
758
759     private static String detectArtifactTypeVFC(ArtifactGroupTypeEnum artifactGroupType, String receivedTypeName, String parentVfName, Map<String, Set<List<String>>> collectedWarningMessages) {
760         String warningMessage = "Warning - artifact type {} that was provided for VFC {} is not recognized.";
761         return detectArtifactType(artifactGroupType, receivedTypeName, warningMessage, collectedWarningMessages, parentVfName);
762     }
763
764     private static String detectArtifactTypeVF(ArtifactGroupTypeEnum artifactGroupType, String receivedTypeName, Map<String, Set<List<String>>> collectedWarningMessages) {
765         String warningMessage = "Warning - artifact type {} that was provided for VF is not recognized.";
766         return detectArtifactType(artifactGroupType, receivedTypeName, warningMessage, collectedWarningMessages);
767     }
768
769     private static String detectArtifactType(ArtifactGroupTypeEnum artifactGroupType, String receivedTypeName, String warningMessage, Map<String, Set<List<String>>> collectedWarningMessages, String... arguments) {
770
771         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(receivedTypeName);
772         Map<String, ArtifactTypeConfig> resourceValidTypeArtifacts = null;
773
774         if(artifactGroupType != null){
775             switch (artifactGroupType) {
776                 case INFORMATIONAL:
777                     resourceValidTypeArtifacts = ConfigurationManager.getConfigurationManager().getConfiguration()
778                                                                      .getResourceInformationalArtifacts();
779                     break;
780                 case DEPLOYMENT:
781                     resourceValidTypeArtifacts = ConfigurationManager.getConfigurationManager().getConfiguration()
782                                                                      .getResourceDeploymentArtifacts();
783                     break;
784                 default:
785                     break;
786             }
787         }
788
789         Set<String> validArtifactTypes = null;
790         if(resourceValidTypeArtifacts != null){
791             validArtifactTypes = resourceValidTypeArtifacts.keySet();
792         }
793
794         if (validArtifactTypes == null || artifactType == null || !validArtifactTypes.contains(artifactType.getType())) {
795             List<String> messageArguments = new ArrayList<>();
796             messageArguments.add(receivedTypeName);
797             messageArguments.addAll(Arrays.asList(arguments));
798             if (!collectedWarningMessages.containsKey(warningMessage)) {
799                 Set<List<String>> messageArgumentLists = new HashSet<>();
800                 messageArgumentLists.add(messageArguments);
801                 collectedWarningMessages.put(warningMessage, messageArgumentLists);
802             } else {
803                 collectedWarningMessages.get(warningMessage).add(messageArguments);
804             }
805         }
806
807         return artifactType == null ? ArtifactTypeEnum.OTHER.getType() : artifactType.getType();
808     }
809
810     private Either<ZipOutputStream, ResponseFormat> writeAllFilesToCsar(Component mainComponent, CsarDefinition csarDefinition, ZipOutputStream zipstream, boolean isInCertificationRequest) throws IOException{
811         ComponentArtifacts componentArtifacts = csarDefinition.getComponentArtifacts();
812
813         Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath = writeComponentArtifactsToSpecifiedPath(mainComponent, componentArtifacts, zipstream, ARTIFACTS_PATH, isInCertificationRequest);
814
815         if(writeComponentArtifactsToSpecifiedPath.isRight()){
816             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
817         }
818
819         ComponentTypeArtifacts mainTypeAndCIArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
820         writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedtPath(mainComponent, mainTypeAndCIArtifacts.getComponentArtifacts(), zipstream, ARTIFACTS_PATH, isInCertificationRequest);
821
822         if(writeComponentArtifactsToSpecifiedPath.isRight()){
823             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
824         }
825
826         Map<String, ArtifactsInfo> componentInstancesArtifacts = mainTypeAndCIArtifacts.getComponentInstancesArtifacts();
827         Set<String> keySet = componentInstancesArtifacts.keySet();
828
829         String currentPath = ARTIFACTS_PATH + RESOURCES_PATH;
830         for (String keyAssetName : keySet) {
831             ArtifactsInfo artifactsInfo = componentInstancesArtifacts.get(keyAssetName);
832             String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
833             writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedtPath(mainComponent, artifactsInfo, zipstream, pathWithAssetName, isInCertificationRequest);
834
835             if(writeComponentArtifactsToSpecifiedPath.isRight()){
836                 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
837             }
838         }
839         writeComponentArtifactsToSpecifiedPath = writeOperationsArtifactsToCsar(mainComponent, zipstream);
840
841         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
842             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
843         }
844         return Either.left(zipstream);
845     }
846
847     private Either<ZipOutputStream, ResponseFormat> writeOperationsArtifactsToCsar(Component component,
848             ZipOutputStream zipstream) {
849         if (checkComponentBeforeOperation(component)) return Either.left(zipstream);
850         final Map<String, InterfaceDefinition> interfaces = ((Resource) component).getInterfaces();
851         for (Map.Entry<String, InterfaceDefinition> interfaceEntry : interfaces.entrySet()) {
852             for (OperationDataDefinition operation : interfaceEntry.getValue().getOperations().values()) {
853                 try {
854                     if (checkComponentBeforeWrite(component, interfaceEntry, operation)) continue;
855                     final String artifactUUID = operation.getImplementation().getArtifactUUID();
856                     final Either<byte[], ActionStatus> artifactFromCassandra = getFromCassandra(artifactUUID);
857                     final String artifactName = operation.getImplementation().getArtifactName();
858                     if (artifactFromCassandra.isRight()) {
859                         log.error(ARTIFACT_NAME_UNIQUE_ID, artifactName, artifactUUID);
860                         log.error("Failed to get {} payload from DB reason: {}", artifactName,
861                                 artifactFromCassandra.right().value());
862                         return Either.right(componentsUtils.getResponseFormat(
863                                 ActionStatus.ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION, "Resource",
864                                 component.getUniqueId(), artifactName, artifactUUID));
865                     }
866                     final byte[] payloadData = artifactFromCassandra.left().value();
867                     zipstream.putNextEntry(new ZipEntry(OperationArtifactUtil.createOperationArtifactPath(
868                             component, null, operation,true)));
869                     zipstream.write(payloadData);
870                 } catch (IOException e) {
871                     log.error("Component Name {},  Interface Name {}, Operation Name {}", component.getNormalizedName(),
872                             interfaceEntry.getKey(), operation.getName());
873                     log.error("Error while writing the operation's artifacts to the CSAR " + "{}", e);
874                     return Either.right(componentsUtils
875                                                 .getResponseFormat(ActionStatus.ERROR_DURING_CSAR_CREATION, "Resource",
876                                                         component.getUniqueId()));
877                 }
878             }
879         }
880         return Either.left(zipstream);
881     }
882
883     private boolean checkComponentBeforeWrite(Component component, Entry<String, InterfaceDefinition> interfaceEntry, OperationDataDefinition operation) {
884         if (Objects.isNull(operation.getImplementation())) {
885             log.debug("Component Name {}, Interface Id {}, Operation Name {} - no Operation Implementation found",
886                     component.getNormalizedName(), interfaceEntry.getValue().getUniqueId(),
887                     operation.getName());
888             return true;
889         }
890         if (Objects.isNull(operation.getImplementation().getArtifactName())) {
891             log.debug("Component Name {}, Interface Id {}, Operation Name {} - no artifact found",
892                     component.getNormalizedName(), interfaceEntry.getValue().getUniqueId(),
893                     operation.getName());
894             return true;
895         }
896         return false;
897     }
898
899     private boolean checkComponentBeforeOperation(Component component) {
900         if (component instanceof Service) {
901             return true;
902         }
903         if (Objects.isNull(((Resource) component).getInterfaces())) {
904             log.debug("Component Name {}- no interfaces found", component.getNormalizedName());
905             return true;
906         }
907         return false;
908     }
909
910     private Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath(Component mainComponent, ComponentArtifacts componentArtifacts, ZipOutputStream zipstream,
911             String currentPath, boolean isInCertificationRequest) throws IOException {
912         Map<String, ComponentTypeArtifacts> componentTypeArtifacts = componentArtifacts.getComponentTypeArtifacts();
913         //Keys are defined:
914         //<Inner Asset TOSCA name (e.g. VFC name)> folder name: <Inner Asset TOSCA name (e.g. VFC name)>_v<version>.
915         //E.g. "org.openecomp.resource.vf.vipr_atm_v1.0"
916         Set<String> componentTypeArtifactsKeys = componentTypeArtifacts.keySet();
917         for (String keyAssetName : componentTypeArtifactsKeys) {
918             ComponentTypeArtifacts componentInstanceArtifacts = componentTypeArtifacts.get(keyAssetName);
919             ArtifactsInfo componentArtifacts2 = componentInstanceArtifacts.getComponentArtifacts();
920             String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
921             Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath = writeArtifactsInfoToSpecifiedtPath(mainComponent, componentArtifacts2, zipstream, pathWithAssetName, isInCertificationRequest);
922
923             if(writeArtifactsInfoToSpecifiedPath.isRight()){
924                 return writeArtifactsInfoToSpecifiedPath;
925             }
926         }
927
928         return Either.left(zipstream);
929     }
930
931     private Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedtPath(Component mainComponent, ArtifactsInfo currArtifactsInfo, ZipOutputStream zip, String path, boolean isInCertificationRequest) throws IOException {
932         Map<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> artifactsInfo = currArtifactsInfo
933                                                                                                             .getArtifactsInfo();
934         Set<ArtifactGroupTypeEnum> groupTypeEnumKeySet = artifactsInfo.keySet();
935
936         for (ArtifactGroupTypeEnum artifactGroupTypeEnum : groupTypeEnumKeySet) {
937             String groupTypeFolder = path + WordUtils.capitalizeFully(artifactGroupTypeEnum.getType()) + PATH_DELIMITER;
938
939             Map<ArtifactTypeEnum, List<ArtifactDefinition>> artifactTypesMap = artifactsInfo.get(artifactGroupTypeEnum);
940             Set<ArtifactTypeEnum> artifactTypeEnumKeySet = artifactTypesMap.keySet();
941
942             for (ArtifactTypeEnum artifactTypeEnum : artifactTypeEnumKeySet) {
943                 List<ArtifactDefinition> artifactDefinitionList = artifactTypesMap.get(artifactTypeEnum);
944                                 String artifactTypeFolder = groupTypeFolder + artifactTypeEnum.toString() + PATH_DELIMITER;
945
946                                 if(artifactTypeEnum == ArtifactTypeEnum.WORKFLOW && path.contains(ARTIFACTS_PATH + RESOURCES_PATH)){
947                                         // Ignore this packaging as BPMN artifacts needs to be packaged in different manner
948                                         continue;
949                                 }
950                                 if (artifactTypeEnum == ArtifactTypeEnum.WORKFLOW) {
951                                         artifactTypeFolder += OperationArtifactUtil.BPMN_ARTIFACT_PATH + File.separator;
952                                 }
953
954                 Either<ZipOutputStream, ResponseFormat> writeArtifactDefinition = writeArtifactDefinition(mainComponent, zip, artifactDefinitionList, artifactTypeFolder, isInCertificationRequest);
955
956                 if(writeArtifactDefinition.isRight()){
957                     return writeArtifactDefinition;
958                 }
959             }
960         }
961
962         return Either.left(zip);
963     }
964
965     private Either<ZipOutputStream, ResponseFormat> writeArtifactDefinition(Component mainComponent, ZipOutputStream zip, List<ArtifactDefinition> artifactDefinitionList,
966             String artifactPathAndFolder, boolean isInCertificationRequest) throws IOException {
967
968         ComponentTypeEnum componentType = mainComponent.getComponentType();
969         String heatEnvType = ArtifactTypeEnum.HEAT_ENV.getType();
970
971         for (ArtifactDefinition artifactDefinition : artifactDefinitionList) {
972             if (!isInCertificationRequest && componentType == ComponentTypeEnum.SERVICE
973                         && artifactDefinition.getArtifactType().equals(heatEnvType) ||
974                         //this is placeholder
975                         (artifactDefinition.getEsId() == null && artifactDefinition.getMandatory())){
976                 continue;
977             }
978
979             byte[] payloadData = artifactDefinition.getPayloadData();
980             String artifactFileName = artifactDefinition.getArtifactName();
981
982             if (payloadData == null) {
983                 Either<byte[], ActionStatus> fromCassandra = getFromCassandra(artifactDefinition.getEsId());
984
985                 if (fromCassandra.isRight()) {
986                     log.debug(ARTIFACT_NAME_UNIQUE_ID, artifactDefinition.getArtifactName(), artifactDefinition.getUniqueId());
987                     log.debug("Failed to get {} payload from DB reason: {}", artifactFileName, fromCassandra.right().value());
988                     continue;
989                 }
990                 payloadData = fromCassandra.left().value();
991             }
992             zip.putNextEntry(new ZipEntry(artifactPathAndFolder + artifactFileName));
993             zip.write(payloadData);
994         }
995
996         return Either.left(zip);
997     }
998
999     /************************************ Artifacts Structure ******************************************************************/
1000     /**
1001      * The artifacts Definition saved by their structure
1002      */
1003     private class ArtifactsInfo {
1004         //Key is the type of artifacts(Informational/Deployment)
1005         //Value is a map between an artifact type and a list of all artifacts of this type
1006         private Map<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> artifactsInfoField;
1007
1008         public ArtifactsInfo() {
1009             this.artifactsInfoField = new EnumMap<>(ArtifactGroupTypeEnum.class);
1010         }
1011
1012         public Map<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> getArtifactsInfo() {
1013             return artifactsInfoField;
1014         }
1015
1016         public List<ArtifactDefinition> getFlatArtifactsListByType(ArtifactTypeEnum artifactType){
1017             List<ArtifactDefinition> artifacts = new ArrayList<>();
1018             for (List<ArtifactDefinition> artifactsByType:artifactsInfoField.get(artifactType).values()){
1019                 artifacts.addAll(artifactsByType);
1020             }
1021             return artifacts;
1022         }
1023
1024         public void addArtifactsToGroup(ArtifactGroupTypeEnum artifactGroup,Map<ArtifactTypeEnum, List<ArtifactDefinition>> artifactsDefinition){
1025                         if (artifactsInfoField.get(artifactGroup) == null) {
1026                                 artifactsInfoField.put(artifactGroup, artifactsDefinition);
1027                         } else {
1028                                 Map<ArtifactTypeEnum, List<ArtifactDefinition>> artifactTypeEnumListMap =
1029                                                 artifactsInfoField.get(artifactGroup);
1030                                 artifactTypeEnumListMap.putAll(artifactsDefinition);
1031                                 artifactsInfoField.put(artifactGroup, artifactTypeEnumListMap);
1032                         }
1033
1034         }
1035
1036         public boolean isEmpty() {
1037             return artifactsInfoField.isEmpty();
1038         }
1039
1040     }
1041
1042     /**
1043      * The artifacts of the component and of all its composed instances
1044      *
1045      */
1046     private class ComponentTypeArtifacts {
1047         private ArtifactsInfo componentArtifacts;    //component artifacts (describes the Informational Deployment folders)
1048         private Map<String, ArtifactsInfo> componentInstancesArtifacts;        //artifacts of the composed instances mapped by the resourceInstance normalized name (describes the Resources folder)
1049
1050         public ComponentTypeArtifacts() {
1051             componentArtifacts = new ArtifactsInfo();
1052             componentInstancesArtifacts = new HashMap<>();
1053         }
1054
1055         public ArtifactsInfo getComponentArtifacts() {
1056             return componentArtifacts;
1057         }
1058         public void setComponentArtifacts(ArtifactsInfo artifactsInfo) {
1059             this.componentArtifacts = artifactsInfo;
1060         }
1061         public Map<String, ArtifactsInfo> getComponentInstancesArtifacts() {
1062             return componentInstancesArtifacts;
1063         }
1064         public void setComponentInstancesArtifacts(Map<String, ArtifactsInfo> componentInstancesArtifacts) {
1065             this.componentInstancesArtifacts = componentInstancesArtifacts;
1066         }
1067
1068         public void addComponentInstancesArtifacts(String normalizedName, ArtifactsInfo artifactsInfo) {
1069             componentInstancesArtifacts.put(normalizedName, artifactsInfo);
1070         }
1071
1072     }
1073
1074     private class ComponentArtifacts {
1075         //artifacts of the component and CI's artifacts contained in it's composition (represents Informational, Deployment & Resource folders of main component)
1076         private ComponentTypeArtifacts mainTypeAndCIArtifacts;
1077         //artifacts of all component types mapped by their tosca name
1078         private Map<String, ComponentTypeArtifacts> componentTypeArtifacts;
1079
1080         public ComponentArtifacts(){
1081             mainTypeAndCIArtifacts = new ComponentTypeArtifacts();
1082             componentTypeArtifacts = new HashMap<>();
1083         }
1084
1085         public ComponentTypeArtifacts getMainTypeAndCIArtifacts() {
1086             return mainTypeAndCIArtifacts;
1087         }
1088
1089         public void setMainTypeAndCIArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
1090             this.mainTypeAndCIArtifacts = componentInstanceArtifacts;
1091         }
1092
1093         public Map<String, ComponentTypeArtifacts> getComponentTypeArtifacts() {
1094             return componentTypeArtifacts;
1095         }
1096
1097         public void setComponentTypeArtifacts(Map<String, ComponentTypeArtifacts> componentTypeArtifacts) {
1098             this.componentTypeArtifacts = componentTypeArtifacts;
1099         }
1100     }
1101
1102     private class CsarDefinition {
1103         private ComponentArtifacts componentArtifacts;
1104
1105         // add list of tosca artifacts and meta describes CSAR zip root
1106
1107         public CsarDefinition(ComponentArtifacts componentArtifacts) {
1108             this.componentArtifacts = componentArtifacts;
1109         }
1110
1111         public ComponentArtifacts getComponentArtifacts() {
1112             return componentArtifacts;
1113         }
1114     }
1115
1116     /************************************ Artifacts Structure END******************************************************************/
1117
1118     private Either<CsarDefinition,ResponseFormat> collectComponentCsarDefinition(Component component){
1119         ComponentArtifacts componentArtifacts = new ComponentArtifacts();
1120         Component updatedComponent = component;
1121
1122         //get service to receive the AII artifacts uploaded to the service
1123         if (updatedComponent.getComponentType() == ComponentTypeEnum.SERVICE) {
1124             Either<Service, StorageOperationStatus> getServiceResponse = toscaOperationFacade.getToscaElement(updatedComponent.getUniqueId());
1125
1126             if(getServiceResponse.isRight()){
1127                 ActionStatus actionStatus = componentsUtils.convertFromStorageResponse(getServiceResponse.right().value());
1128                 return Either.right(componentsUtils.getResponseFormat(actionStatus));
1129             }
1130
1131             updatedComponent = getServiceResponse.left().value();
1132         }
1133
1134         //find the artifacts of the main component, it would have its composed instances artifacts in a separate folder
1135         ComponentTypeArtifacts componentInstanceArtifacts = new ComponentTypeArtifacts();
1136         ArtifactsInfo artifactsInfo = collectComponentArtifacts(updatedComponent);
1137         componentInstanceArtifacts.setComponentArtifacts(artifactsInfo);
1138         componentArtifacts.setMainTypeAndCIArtifacts(componentInstanceArtifacts);
1139
1140         Map<String,ComponentTypeArtifacts> resourceTypeArtifacts = componentArtifacts.getComponentTypeArtifacts();    //artifacts mapped by the component type(tosca name+version)
1141         //get the component instances
1142         List<ComponentInstance> componentInstances = updatedComponent.getComponentInstances();
1143         if (componentInstances!=null){
1144             for (ComponentInstance componentInstance:componentInstances){
1145                 //call recursive to find artifacts for all the path
1146                 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
1147                         updatedComponent, componentInstance, resourceTypeArtifacts, componentInstanceArtifacts);
1148                 if (collectComponentInstanceArtifacts.isRight()){
1149                     return Either.right(collectComponentInstanceArtifacts.right().value());
1150                 }
1151             }
1152         }
1153
1154         if(log.isDebugEnabled()){
1155             printResult(componentArtifacts,updatedComponent.getName());
1156         }
1157
1158         return Either.left(new CsarDefinition(componentArtifacts));
1159     }
1160
1161     private void printResult(ComponentArtifacts componentArtifacts, String name) {
1162         StringBuilder result = new StringBuilder();
1163         result.append("Artifacts of main component " + name + "\n");
1164         ComponentTypeArtifacts componentInstanceArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
1165         printArtifacts(componentInstanceArtifacts);
1166         result.append("Type Artifacts\n");
1167         for (Map.Entry<String, ComponentTypeArtifacts> typeArtifacts:componentArtifacts.getComponentTypeArtifacts().entrySet()){
1168             result.append("Folder " + typeArtifacts.getKey() + "\n");
1169             result.append(printArtifacts(typeArtifacts.getValue()));
1170         }
1171
1172         if(log.isDebugEnabled()){
1173             log.debug(result.toString());
1174         }
1175     }
1176
1177     private String printArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
1178         StringBuilder result = new StringBuilder();
1179         ArtifactsInfo artifactsInfo = componentInstanceArtifacts.getComponentArtifacts();
1180         Map<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> componetArtifacts = artifactsInfo.getArtifactsInfo();
1181         printArtifacts(componetArtifacts);
1182         result = result.append("Resources\n");
1183         for (Map.Entry<String, ArtifactsInfo> resourceInstance:componentInstanceArtifacts.getComponentInstancesArtifacts().entrySet()){
1184             result.append("Folder" + resourceInstance.getKey() + "\n");
1185             result.append(printArtifacts(resourceInstance.getValue().getArtifactsInfo()));
1186         }
1187
1188         return result.toString();
1189     }
1190
1191     private String  printArtifacts(Map<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> componetArtifacts) {
1192         StringBuilder result = new StringBuilder();
1193         for (Map.Entry<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> artifactGroup:componetArtifacts.entrySet()){
1194             result.append("    " + artifactGroup.getKey().getType());
1195             for (Map.Entry<ArtifactTypeEnum, List<ArtifactDefinition>> groupArtifacts:artifactGroup.getValue().entrySet()){
1196                 result.append("        " + groupArtifacts.getKey().getType());
1197                 for (ArtifactDefinition artifact:groupArtifacts.getValue()){
1198                     result.append("            " + artifact.getArtifactDisplayName());
1199                 }
1200             }
1201         }
1202
1203         return result.toString();
1204     }
1205
1206     private ComponentTypeArtifacts collectComponentTypeArtifacts(Map<String, ComponentTypeArtifacts> resourcesArtifacts, ComponentInstance componentInstance,
1207             Component fetchedComponent) {
1208         String toscaComponentName = componentInstance.getToscaComponentName() + "_v" + componentInstance.getComponentVersion();
1209
1210         ComponentTypeArtifacts componentArtifactsInfo = resourcesArtifacts.get(toscaComponentName);
1211         //if there are no artifacts for this component type we need to fetch and build them
1212         if (componentArtifactsInfo==null){
1213             ArtifactsInfo componentArtifacts = collectComponentArtifacts(fetchedComponent);
1214             componentArtifactsInfo = new ComponentTypeArtifacts();
1215             if (!componentArtifacts.isEmpty()){
1216                 componentArtifactsInfo.setComponentArtifacts(componentArtifacts);
1217                 resourcesArtifacts.put(toscaComponentName, componentArtifactsInfo);
1218             }
1219         }
1220         return componentArtifactsInfo;
1221     }
1222
1223     private Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts(Component parentComponent,ComponentInstance componentInstance,
1224             Map<String, ComponentTypeArtifacts> resourcesTypeArtifacts,ComponentTypeArtifacts instanceArtifactsLocation) {
1225         //1. get the component instance component
1226         String componentUid;
1227         if (componentInstance.getOriginType() == OriginTypeEnum.ServiceProxy) {
1228                         componentUid = componentInstance.getSourceModelUid();
1229                 }
1230                 else {
1231                         componentUid = componentInstance.getComponentUid();
1232                 }
1233         Either<Component, StorageOperationStatus> component = toscaOperationFacade.getToscaElement(componentUid);
1234                 if (component.isRight()) {
1235             log.error("Failed to fetch resource with id {} for instance {}",componentUid, parentComponent.getUUID());
1236             return Either.right(componentsUtils.getResponseFormat(ActionStatus.ASSET_NOT_FOUND_DURING_CSAR_CREATION,
1237                     parentComponent.getComponentType().getValue(), parentComponent.getUUID(),
1238                     componentInstance.getOriginType().getComponentType().getValue(), componentUid));
1239         }
1240                 Component fetchedComponent = component.left().value();
1241
1242         //2. fill the artifacts for the current component parent type
1243         ComponentTypeArtifacts componentParentArtifacts = collectComponentTypeArtifacts(resourcesTypeArtifacts, componentInstance, fetchedComponent);
1244
1245         //3. find the artifacts specific to the instance
1246         Map<ArtifactTypeEnum, List<ArtifactDefinition>> componentInstanceSpecificInformationalArtifacts =
1247                 getComponentInstanceSpecificArtifacts(componentInstance.getArtifacts(),
1248                         componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.INFORMATIONAL);
1249         Map<ArtifactTypeEnum, List<ArtifactDefinition>> componentInstanceSpecificDeploymentArtifacts =
1250                 getComponentInstanceSpecificArtifacts(componentInstance.getDeploymentArtifacts(),
1251                         componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.DEPLOYMENT);
1252
1253         //4. add the instances artifacts to the component type
1254         ArtifactsInfo artifactsInfo = new ArtifactsInfo();
1255         if (!componentInstanceSpecificInformationalArtifacts.isEmpty()){
1256             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, componentInstanceSpecificInformationalArtifacts);
1257         }
1258         if (!componentInstanceSpecificDeploymentArtifacts.isEmpty()){
1259             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, componentInstanceSpecificDeploymentArtifacts);
1260         }
1261         if (!artifactsInfo.isEmpty()){
1262             instanceArtifactsLocation.addComponentInstancesArtifacts(componentInstance.getNormalizedName(), artifactsInfo);
1263         }
1264
1265         //5. do the same for all the component instances
1266         List<ComponentInstance> componentInstances = fetchedComponent.getComponentInstances();
1267         if (componentInstances!=null){
1268             for (ComponentInstance childComponentInstance:componentInstances){
1269                 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
1270                         fetchedComponent, childComponentInstance, resourcesTypeArtifacts, componentParentArtifacts);
1271                 if (collectComponentInstanceArtifacts.isRight()){
1272                     return collectComponentInstanceArtifacts;
1273                 }
1274             }
1275         }
1276
1277         return Either.left(true);
1278     }
1279
1280     public String getVersionFirstThreeOctets() {
1281         return versionFirstThreeOctets;
1282     }
1283
1284     public void setVersionFirstThreeOctets(String versionFirstThreeOctetes) {
1285         this.versionFirstThreeOctets = versionFirstThreeOctetes;
1286     }
1287     private Map<ArtifactTypeEnum, List<ArtifactDefinition>> getComponentInstanceSpecificArtifacts(Map<String, ArtifactDefinition> componentArtifacts,
1288             Map<ArtifactGroupTypeEnum, Map<ArtifactTypeEnum, List<ArtifactDefinition>>> componentTypeArtifacts, ArtifactGroupTypeEnum artifactGroupTypeEnum) {
1289         Map<ArtifactTypeEnum, List<ArtifactDefinition>> parentArtifacts = componentTypeArtifacts.get(artifactGroupTypeEnum);    //the artfiacts of the component itself and not the instance
1290
1291         Map<ArtifactTypeEnum, List<ArtifactDefinition>> artifactsByTypeOfComponentInstance = new EnumMap<>(ArtifactTypeEnum.class);
1292         if (componentArtifacts!=null){
1293             for (ArtifactDefinition artifact:componentArtifacts.values()){
1294                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
1295                 List<ArtifactDefinition> parentArtifactsByType = null;
1296                 if (parentArtifacts!=null){
1297                     parentArtifactsByType = parentArtifacts.get(artifactType);
1298                 }
1299                 //the artifact is of instance
1300                 if (parentArtifactsByType == null || !parentArtifactsByType.contains(artifact)){
1301                     List<ArtifactDefinition> typeArtifacts = artifactsByTypeOfComponentInstance.get(artifactType);
1302                     if (typeArtifacts == null){
1303                         typeArtifacts = new ArrayList<>();
1304                         artifactsByTypeOfComponentInstance.put(artifactType, typeArtifacts);
1305                     }
1306                     typeArtifacts.add(artifact);
1307                 }
1308             }
1309         }
1310
1311         return artifactsByTypeOfComponentInstance;
1312     }
1313
1314     private ArtifactsInfo collectComponentArtifacts(Component component) {
1315         Map<String, ArtifactDefinition> informationalArtifacts = component.getArtifacts();
1316         Map<ArtifactTypeEnum, List<ArtifactDefinition>> informationalArtifactsByType = collectGroupArtifacts(informationalArtifacts);
1317         Map<String, ArtifactDefinition> deploymentArtifacts = component.getDeploymentArtifacts();
1318         Map<ArtifactTypeEnum, List<ArtifactDefinition>> deploymentArtifactsByType = collectGroupArtifacts(deploymentArtifacts);
1319                 Map<String, ArtifactDefinition> interfaceOperationArtifacts =
1320                                 OperationArtifactUtil.getDistinctInterfaceOperationArtifactsByName(component);
1321                 Map<ArtifactTypeEnum, List<ArtifactDefinition>> interfaceOperationArtifactsByType = collectGroupArtifacts(
1322                                 interfaceOperationArtifacts);
1323         ArtifactsInfo artifactsInfo = new ArtifactsInfo();
1324         if (!informationalArtifactsByType.isEmpty()){
1325             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, informationalArtifactsByType);
1326         }
1327         if (!deploymentArtifactsByType.isEmpty() ){
1328             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, deploymentArtifactsByType);
1329                 }
1330                 //Add component interface operation artifacts
1331                 if(MapUtils.isNotEmpty(interfaceOperationArtifacts)) {
1332                         artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, interfaceOperationArtifactsByType);
1333         }
1334
1335         return artifactsInfo;
1336     }
1337
1338     private Map<ArtifactTypeEnum, List<ArtifactDefinition>> collectGroupArtifacts(Map<String, ArtifactDefinition> componentArtifacts) {
1339         Map<ArtifactTypeEnum, List<ArtifactDefinition>> artifactsByType = new EnumMap<>(ArtifactTypeEnum.class);
1340         for (ArtifactDefinition artifact:componentArtifacts.values()){
1341             if (artifact.getArtifactUUID()!=null){
1342                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
1343                 List<ArtifactDefinition> typeArtifacts = artifactsByType.get(artifactType);
1344                 if (typeArtifacts==null){
1345                     typeArtifacts = new ArrayList<>();
1346                     artifactsByType.put(artifactType, typeArtifacts);
1347                 }
1348                 typeArtifacts.add(artifact);
1349             }
1350         }
1351         return artifactsByType;
1352     }
1353 }