catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CommonCsarGenerator.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2023 Nordix Foundation. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.openecomp.sdc.be.tosca;
  21
  22 import static org.openecomp.sdc.be.dao.api.ActionStatus.ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION;
  23 import static org.openecomp.sdc.be.dao.api.ActionStatus.ERROR_DURING_CSAR_CREATION;
  24 import static org.openecomp.sdc.be.tosca.ComponentCache.MergeStrategy.overwriteIfSameVersions;
  25 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.fromEither;
  26 import static org.openecomp.sdc.be.tosca.FJToVavrHelper.Try0.javaListToVavrList;
  27 import static org.openecomp.sdc.common.api.Constants.ADDITIONAL_TYPE_DEFINITIONS;
  28
  29 import com.google.common.primitives.Bytes;
  30 import fj.F;
  31 import fj.data.Either;
  32 import io.vavr.Tuple2;
  33 import io.vavr.control.Try;
  34 import java.io.BufferedOutputStream;
  35 import java.io.ByteArrayInputStream;
  36 import java.io.File;
  37 import java.io.IOException;
  38 import java.nio.charset.StandardCharsets;
  39 import java.nio.file.Path;
  40 import java.nio.file.Paths;
  41 import java.text.SimpleDateFormat;
  42 import java.util.ArrayList;
  43 import java.util.Date;
  44 import java.util.EnumMap;
  45 import java.util.HashMap;
  46 import java.util.HashSet;
  47 import java.util.List;
  48 import java.util.Map;
  49 import java.util.Objects;
  50 import java.util.Optional;
  51 import java.util.Set;
  52 import java.util.TimeZone;
  53 import java.util.function.Function;
  54 import java.util.function.Predicate;
  55 import java.util.function.Supplier;
  56 import java.util.regex.Matcher;
  57 import java.util.regex.Pattern;
  58 import java.util.stream.Collectors;
  59 import java.util.stream.Stream;
  60 import java.util.zip.ZipEntry;
  61 import java.util.zip.ZipInputStream;
  62 import java.util.zip.ZipOutputStream;
  63 import org.apache.commons.collections.CollectionUtils;
  64 import org.apache.commons.collections.MapUtils;
  65 import org.apache.commons.io.output.ByteArrayOutputStream;
  66 import org.apache.commons.lang.StringUtils;
  67 import org.apache.commons.lang3.tuple.ImmutableTriple;
  68 import org.apache.commons.lang3.tuple.Triple;
  69 import org.apache.commons.text.WordUtils;
  70 import org.onap.sdc.tosca.services.YamlUtil;
  71 import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException;
  72 import org.openecomp.sdc.be.config.CategoryBaseTypeConfig;
  73 import org.openecomp.sdc.be.config.ConfigurationManager;
  74 import org.openecomp.sdc.be.dao.api.ActionStatus;
  75 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  76 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  77 import org.openecomp.sdc.be.dao.cassandra.SdcSchemaFilesCassandraDao;
  78 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
  79 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
  80 import org.openecomp.sdc.be.datatypes.elements.OperationDataDefinition;
  81 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
  82 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
  83 import org.openecomp.sdc.be.impl.ComponentsUtils;
  84 import org.openecomp.sdc.be.model.ArtifactDefinition;
  85 import org.openecomp.sdc.be.model.Component;
  86 import org.openecomp.sdc.be.model.ComponentInstance;
  87 import org.openecomp.sdc.be.model.InterfaceDefinition;
  88 import org.openecomp.sdc.be.model.LifecycleStateEnum;
  89 import org.openecomp.sdc.be.model.Resource;
  90 import org.openecomp.sdc.be.model.Service;
  91 import org.openecomp.sdc.be.model.category.CategoryDefinition;
  92 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
  93 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
  94 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  95 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
  96 import org.openecomp.sdc.be.model.operations.impl.ModelOperation;
  97 import org.openecomp.sdc.be.plugins.CsarEntryGenerator;
  98 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
  99 import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil;
 100 import org.openecomp.sdc.be.utils.TypeUtils;
 101 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
 102 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
 103 import org.openecomp.sdc.common.impl.ExternalConfiguration;
 104 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
 105 import org.openecomp.sdc.common.util.GeneralUtility;
 106 import org.openecomp.sdc.common.zip.ZipUtils;
 107 import org.openecomp.sdc.exception.ResponseFormat;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110 import org.springframework.beans.factory.annotation.Autowired;
 111 import org.yaml.snakeyaml.Yaml;
 112
 113 /**
 114  * Generates a Network Service CSAR based on a SERVICE component and wraps it in a SDC CSAR entry.
 115  */
 116 @org.springframework.stereotype.Component("commonCsarGenerator")
 117 public class CommonCsarGenerator {
 118
 119     private static final Logger LOGGER = LoggerFactory.getLogger(CommonCsarGenerator.class);
 120     public static final String ARTIFACTS_PATH = "Artifacts/";
 121     private static final String RESOURCES_PATH = "Resources/";
 122     private static final String PATH_DELIMITER = "/";
 123     private static final String SERVICE_MANIFEST = "NS.mf";
 124     private static final String ARTIFACT_NAME_UNIQUE_ID = "ArtifactName {}, unique ID {}";
 125     private static final String TOSCA_META_PATH_FILE_NAME = "TOSCA-Metadata/TOSCA.meta";
 126     private static final String TOSCA_META_VERSION = "1.0";
 127     private static final String CSAR_VERSION = "1.1";
 128     private static final String SDC_VERSION = ExternalConfiguration.getAppVersion();
 129     public static final String NODES_YML = "nodes.yml";
 130     private static final String CONFORMANCE_LEVEL = ConfigurationManager.getConfigurationManager().getConfiguration().getToscaConformanceLevel();
 131     private final ToscaOperationFacade toscaOperationFacade;
 132     private final ComponentsUtils componentsUtils;
 133     private final ToscaExportHandler toscaExportUtils;
 134     private final List<CsarEntryGenerator> generators;
 135     private final ArtifactCassandraDao artifactCassandraDao;
 136     private final String versionFirstThreeOctets;
 137     private final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
 138     private final ModelOperation modelOperation;
 139
 140     @Autowired
 141     public CommonCsarGenerator(
 142         final ToscaOperationFacade toscaOperationFacade,
 143         final ComponentsUtils componentsUtils,
 144         final ToscaExportHandler toscaExportUtils,
 145         final List<CsarEntryGenerator> generators,
 146         final ArtifactCassandraDao artifactCassandraDao,
 147         final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao,
 148         final ModelOperation modelOperation) {
 149         this.toscaOperationFacade = toscaOperationFacade;
 150         this.componentsUtils = componentsUtils;
 151         this.toscaExportUtils = toscaExportUtils;
 152         this.generators = generators;
 153         this.artifactCassandraDao = artifactCassandraDao;
 154         this.versionFirstThreeOctets = readVersionFirstThreeOctets();
 155         this.sdcSchemaFilesCassandraDao = sdcSchemaFilesCassandraDao;
 156         this.modelOperation = modelOperation;
 157     }
 158
 159     private String readVersionFirstThreeOctets() {
 160         if (StringUtils.isEmpty(SDC_VERSION)) {
 161             return "";
 162         }
 163         // change regex to avoid DoS sonar issue
 164         Matcher matcher = Pattern.compile("(?!\\.)(\\d{1,9}(\\.\\d{1,9}){1,9})(?![\\d\\.])").matcher(SDC_VERSION);
 165         matcher.find();
 166         return matcher.group(0);
 167     }
 168
 169     /**
 170      * Generates a Network Service CSAR based on a SERVICE component that has category configured in
 171      * CategoriesToGenerateNsd enum and wraps it in a SDC CSAR entry.
 172      *
 173      * @param component the component to create the NS CSAR from
 174      * @return an entry to be added in the Component CSAR by SDC
 175      */
 176
 177     public Either<ZipOutputStream, ResponseFormat> generateCsarZip(Component component,
 178                                                                    boolean getFromCS,
 179                                                                    ZipOutputStream zip,
 180                                                                    boolean isInCertificationRequest,
 181                                                                    boolean isAsdPackage,
 182                                                                    String definitionsPath,
 183                                                                    boolean addDependencies,
 184                                                                    boolean isSkipImports) throws IOException {
 185         ArtifactDefinition artifactDef = component.getToscaArtifacts().get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
 186         Either<ToscaRepresentation, ResponseFormat> toscaRepresentation = fetchToscaRepresentation(component, getFromCS, artifactDef, isSkipImports);
 187
 188         // This should not be done but in order to keep the refactoring small enough we stop here.
 189         // TODO: Refactor the rest of this function
 190         byte[] mainYaml;
 191         List<Triple<String, String, Component>> dependencies;
 192         if (toscaRepresentation.isLeft()) {
 193             mainYaml = toscaRepresentation.left().value().getMainYaml();
 194             dependencies = toscaRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
 195         } else {
 196             return Either.right(toscaRepresentation.right().value());
 197         }
 198         final String fileName = artifactDef.getArtifactName();
 199         final byte[] toscaBlock0Byte =
 200             createToscaBlock0(TOSCA_META_VERSION, CSAR_VERSION, component.getCreatorFullName(), fileName, isAsdPackage, definitionsPath).getBytes();
 201         zip.putNextEntry(new ZipEntry(TOSCA_META_PATH_FILE_NAME));
 202         zip.write(toscaBlock0Byte);
 203         zip.putNextEntry(new ZipEntry(definitionsPath + fileName));
 204         zip.write(mainYaml);
 205         LifecycleStateEnum lifecycleState = component.getLifecycleState();
 206         addServiceMf(component, zip, lifecycleState, isInCertificationRequest, fileName, mainYaml, definitionsPath);
 207         if (addDependencies) {
 208             //US798487 - Abstraction of complex types
 209             if (hasToWriteComponentSubstitutionType(component)) {
 210                 LOGGER.debug("Component {} is complex - generating abstract type for it..", component.getName());
 211                 dependencies.addAll(writeComponentInterface(component, zip, fileName, definitionsPath));
 212             }
 213             //UID <cassandraId,filename,component>
 214             Either<ZipOutputStream, ResponseFormat> zipOutputStreamOrResponseFormat =
 215                 getZipOutputStreamResponseFormatEither(zip, dependencies, definitionsPath);
 216             if (zipOutputStreamOrResponseFormat != null && zipOutputStreamOrResponseFormat.isRight()) {
 217                 return zipOutputStreamOrResponseFormat;
 218             }
 219         }
 220         if (component.getModel() == null) {
 221             //retrieve SDC.zip from Cassandra
 222             Either<byte[], ResponseFormat> latestSchemaFiles = getLatestSchemaFilesFromCassandra();
 223             if (latestSchemaFiles.isRight()) {
 224                 LOGGER.error("Error retrieving SDC Schema files from cassandra");
 225                 return Either.right(latestSchemaFiles.right().value());
 226             }
 227             final byte[] schemaFileZip = latestSchemaFiles.left().value();
 228             final List<String> nodesFromPackage = findNonRootNodesFromPackage(dependencies);
 229             //add files from retrieved SDC.zip to Definitions folder in CSAR
 230             addSchemaFilesFromCassandra(zip, schemaFileZip, nodesFromPackage, definitionsPath);
 231         } else {
 232             //retrieve schema files by model from Cassandra
 233             addSchemaFilesByModel(zip, component.getModel(), definitionsPath, addDependencies);
 234         }
 235         Either<CsarDefinition, ResponseFormat> collectedComponentCsarDefinition = collectComponentCsarDefinition(component);
 236         if (collectedComponentCsarDefinition.isRight()) {
 237             return Either.right(collectedComponentCsarDefinition.right().value());
 238         }
 239         if (generators != null) {
 240             for (CsarEntryGenerator generator : generators) {
 241                 LOGGER.debug("Invoking CsarEntryGenerator: {}", generator.getClass().getName());
 242                 for (Map.Entry<String, byte[]> pluginGeneratedFile : generator.generateCsarEntries(component).entrySet()) {
 243                     zip.putNextEntry(new ZipEntry(pluginGeneratedFile.getKey()));
 244                     zip.write(pluginGeneratedFile.getValue());
 245                 }
 246             }
 247         }
 248         return writeAllFilesToCsar(component, collectedComponentCsarDefinition.left().value(), zip, isInCertificationRequest);
 249     }
 250
 251     private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(Component component, boolean getFromCS,
 252                                                                                  ArtifactDefinition artifactDef, boolean isSkipImports) {
 253         LifecycleStateEnum lifecycleState = component.getLifecycleState();
 254         boolean shouldBeFetchedFromCassandra =
 255             getFromCS || !(lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN || lifecycleState == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT);
 256         Either<ToscaRepresentation, ResponseFormat> toscaRepresentation =
 257             shouldBeFetchedFromCassandra ? fetchToscaRepresentation(artifactDef) : generateToscaRepresentation(component, isSkipImports);
 258         return toscaRepresentation.left()
 259             .bind(iff(myd -> !myd.getDependencies().isDefined(), myd -> fetchToscaTemplateDependencies(myd.getMainYaml(), component)));
 260     }
 261
 262     private Either<ToscaRepresentation, ResponseFormat> fetchToscaTemplateDependencies(byte[] mainYml, Component component) {
 263         return toscaExportUtils.getDependencies(component).right().map(toscaError -> {
 264             LOGGER.debug("Failed to retrieve dependencies for component {}, error {}", component.getUniqueId(), toscaError);
 265             return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
 266         }).left().map(tt -> ToscaRepresentation.make(mainYml, tt));
 267     }
 268
 269     private Either<ToscaRepresentation, ResponseFormat> fetchToscaRepresentation(ArtifactDefinition artifactDef) {
 270         return getFromCassandra(artifactDef.getEsId()).right().map(as -> {
 271             LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, artifactDef.getArtifactName(), artifactDef.getUniqueId());
 272             return componentsUtils.getResponseFormat(as);
 273         }).left().map(ToscaRepresentation::make);
 274     }
 275
 276     private Either<byte[], ActionStatus> getFromCassandra(String cassandraId) {
 277         return artifactCassandraDao.getArtifact(cassandraId).right().map(operationstatus -> {
 278             LOGGER.info("Failed to fetch artifact from Cassandra by id {} error {}.", cassandraId, operationstatus);
 279             StorageOperationStatus storageStatus = DaoStatusConverter.convertCassandraStatusToStorageStatus(operationstatus);
 280             return componentsUtils.convertFromStorageResponse(storageStatus);
 281         }).left().map(DAOArtifactData::getDataAsArray);
 282     }
 283
 284     private static <L, R> F<L, Either<L, R>> iff(Predicate<L> p, Function<L, Either<L, R>> ifTrue) {
 285         return l -> p.test(l) ? ifTrue.apply(l) : Either.left(l);
 286     }
 287
 288     private static <A, B> F<A, B> iff(Predicate<A> p, Supplier<B> s, Function<A, B> orElse) {
 289         return a -> p.test(a) ? s.get() : orElse.apply(a);
 290     }
 291
 292     private void addServiceMf(Component component, ZipOutputStream zip, LifecycleStateEnum lifecycleState, boolean isInCertificationRequest,
 293                               String fileName, byte[] mainYaml, String definitionsPath) throws IOException {
 294         // add mf
 295         if ((component.getComponentType() == ComponentTypeEnum.SERVICE) && (lifecycleState != LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT)) {
 296             String serviceName = component.getName();
 297             String createdBy = component.getCreatorUserId();
 298             String serviceVersion;
 299             if (isInCertificationRequest) {
 300                 int tmp = Integer.valueOf(component.getVersion().split("\\.")[0]) + 1;
 301                 serviceVersion = String.valueOf(tmp) + ".0";
 302             } else {
 303                 serviceVersion = component.getVersion();
 304             }
 305             SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
 306             format.setTimeZone(TimeZone.getTimeZone("UTC"));
 307             Date date = new Date();
 308             String releaseTime = format.format(date);
 309             if (component.getCategories() == null || component.getCategories().get(0) == null) {
 310                 return;
 311             }
 312             String serviceType = component.getCategories().get(0).getName();
 313             String description = component.getDescription();
 314             String serviceTemplate = definitionsPath + fileName;
 315             String hash = GeneralUtility.calculateMD5Base64EncodedByByteArray(mainYaml);
 316             String nsMfBlock0 = createNsMfBlock0(serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate,
 317                 hash);
 318             byte[] nsMfBlock0Byte = nsMfBlock0.getBytes();
 319             zip.putNextEntry(new ZipEntry(SERVICE_MANIFEST));
 320             zip.write(nsMfBlock0Byte);
 321         }
 322     }
 323
 324     private String createNsMfBlock0(String serviceName, String createdBy, String serviceVersion, String releaseTime, String serviceType,
 325                                     String description, String serviceTemplate, String hash) {
 326         final String block0template = "metadata??\n" + "ns_product_name: %s\n" + "ns_provider_id: %s\n" + "ns_package_version: %s\n" +
 327             "ns_release_data_time: %s\n" + "ns_type: %s\n" + "ns_package_description: %s\n\n" + "Source: %s\n" + "Algorithm: MD5\n" + "Hash: %s\n\n";
 328         return String.format(block0template, serviceName, createdBy, serviceVersion, releaseTime, serviceType, description, serviceTemplate, hash);
 329     }
 330
 331     private boolean hasToWriteComponentSubstitutionType(final Component component) {
 332         final Map<String, CategoryBaseTypeConfig> serviceNodeTypesConfig =
 333             ConfigurationManager.getConfigurationManager().getConfiguration().getServiceBaseNodeTypes();
 334         List<CategoryDefinition> categories = component.getCategories();
 335         if (CollectionUtils.isNotEmpty(categories) && MapUtils.isNotEmpty(serviceNodeTypesConfig)
 336             && serviceNodeTypesConfig.get(categories.get(0).getName()) != null) {
 337             boolean doNotExtendBaseType = serviceNodeTypesConfig.get(categories.get(0).getName()).isDoNotExtendBaseType();
 338             if (doNotExtendBaseType) {
 339                 return false;
 340             }
 341         }
 342         if (component instanceof Service) {
 343             return !ModelConverter.isAtomicComponent(component) && ((Service) component).isSubstituteCandidate();
 344         }
 345         return !ModelConverter.isAtomicComponent(component);
 346     }
 347
 348     private Either<ZipOutputStream, ResponseFormat> writeComponentInterface(Either<ToscaRepresentation, ToscaError> interfaceRepresentation,
 349                                                                             ZipOutputStream zip, String fileName, String definitionsPath) {
 350         // TODO: This should not be done but we need this to keep the refactoring small enough to be easily reviewable
 351         return writeComponentInterface(interfaceRepresentation, fileName, ZipWriter.live(zip), definitionsPath)
 352             .map(void0 -> Either.<ZipOutputStream, ResponseFormat>left(zip)).recover(th -> {
 353                 LOGGER.error("#writeComponentInterface - zip writing failed with error: ", th);
 354                 return Either.right(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
 355             }).get();
 356     }
 357
 358     private Try<Void> writeComponentInterface(
 359         Either<ToscaRepresentation, ToscaError> interfaceRepresentation, String fileName, ZipWriter zw, String definitionsPath) {
 360         Either<byte[], ToscaError> yml = interfaceRepresentation.left()
 361             .map(ToscaRepresentation::getMainYaml);
 362         return fromEither(yml, ToscaErrorException::new).flatMap(zw.write(definitionsPath + ToscaExportHandler.getInterfaceFilename(fileName)));
 363     }
 364
 365     private List<Triple<String, String, Component>> writeComponentInterface(final Component component, final ZipOutputStream zip,
 366                                                                             final String fileName, final String definitionsPath) {
 367         final Either<ToscaRepresentation, ToscaError> interfaceRepresentation = toscaExportUtils.exportComponentInterface(component, false);
 368         writeComponentInterface(interfaceRepresentation, zip, fileName, definitionsPath);
 369         return interfaceRepresentation.left().value().getDependencies().getOrElse(new ArrayList<>());
 370     }
 371
 372     private Either<ZipOutputStream, ResponseFormat> getZipOutputStreamResponseFormatEither(ZipOutputStream zip,
 373                                                                                            List<Triple<String, String, Component>> dependencies,
 374                                                                                            String definitionsPath)
 375         throws IOException {
 376         ComponentCache
 377             innerComponentsCache = ComponentCache.overwritable(overwriteIfSameVersions()).onMerge((oldValue, newValue) ->
 378             LOGGER.warn("Overwriting component invariantID {} of version {} with a newer version {}", oldValue.getId(),
 379                 oldValue.getComponentVersion(),
 380                 newValue.getComponentVersion()));
 381         if (dependencies != null && !dependencies.isEmpty()) {
 382             for (Triple<String, String, Component> d : dependencies) {
 383                 String cassandraId = d.getMiddle();
 384                 Component childComponent = d.getRight();
 385                 Either<byte[], ResponseFormat> entryData = getEntryData(cassandraId, childComponent).right()
 386                     .map(componentsUtils::getResponseFormat);
 387                 if (entryData.isRight()) {
 388                     return Either.right(entryData.right().value());
 389                 }
 390                 //fill innerComponentsCache
 391                 String fileName = d.getLeft();
 392                 innerComponentsCache.put(cassandraId, fileName, childComponent);
 393                 addInnerComponentsToCache(innerComponentsCache, childComponent);
 394             }
 395             //add inner components to CSAR
 396             return addInnerComponentsToCSAR(zip, innerComponentsCache, definitionsPath);
 397         }
 398         return null;
 399     }
 400
 401     private Either<ZipOutputStream, ResponseFormat> addInnerComponentsToCSAR(ZipOutputStream zip, ComponentCache innerComponentsCache,
 402                                                                              String definitionsPath)
 403         throws IOException {
 404         for (ImmutableTriple<String, String, Component> ict : innerComponentsCache.iterable()) {
 405             Component innerComponent = ict.getRight();
 406             String icFileName = ict.getMiddle();
 407             // add component to zip
 408             Either<Tuple2<byte[], ZipEntry>, ResponseFormat> zipEntry = toZipEntry(ict, definitionsPath);
 409             // TODO: this should not be done, we should instead compose this either further,
 410
 411             // but in order to keep this refactoring small, we'll stop here.
 412             if (zipEntry.isRight()) {
 413                 return Either.right(zipEntry.right().value());
 414             }
 415             Tuple2<byte[], ZipEntry> value = zipEntry.left().value();
 416             zip.putNextEntry(value._2);
 417             zip.write(value._1);
 418             // add component interface to zip
 419             if (hasToWriteComponentSubstitutionType(innerComponent)) {
 420                 writeComponentInterface(innerComponent, zip, icFileName, definitionsPath);
 421             }
 422         }
 423         return null;
 424     }
 425
 426     private Either<Tuple2<byte[], ZipEntry>, ResponseFormat> toZipEntry(ImmutableTriple<String, String, Component> cachedEntry,
 427                                                                         String definitionsPath) {
 428         String cassandraId = cachedEntry.getLeft();
 429         String fileName = cachedEntry.getMiddle();
 430         Component innerComponent = cachedEntry.getRight();
 431         return getEntryData(cassandraId, innerComponent).right().map(status -> {
 432             LOGGER.debug("Failed adding to zip component {}, error {}", cassandraId, status);
 433             return componentsUtils.getResponseFormat(status);
 434         }).left().map(content -> new Tuple2<>(content, new ZipEntry(definitionsPath + fileName)));
 435     }
 436
 437     private void addInnerComponentsToCache(ComponentCache componentCache, Component childComponent) {
 438         javaListToVavrList(childComponent.getComponentInstances()).filter(ci -> componentCache.notCached(ci.getComponentUid())).forEach(ci -> {
 439             // all resource must be only once!
 440             Either<Resource, StorageOperationStatus> resource = toscaOperationFacade.getToscaElement(ci.getComponentUid());
 441             Component componentRI = checkAndAddComponent(componentCache, ci, resource);
 442             //if not atomic - insert inner components as well
 443
 444             // TODO: This could potentially create a StackOverflowException if the call stack
 445
 446             // happens to be too large. Tail-recursive optimization should be used here.
 447             if (!ModelConverter.isAtomicComponent(componentRI)) {
 448                 addInnerComponentsToCache(componentCache, componentRI);
 449             }
 450         });
 451     }
 452
 453     private Component checkAndAddComponent(ComponentCache componentCache, ComponentInstance ci, Either<Resource, StorageOperationStatus> resource) {
 454         if (resource.isRight()) {
 455             LOGGER.debug("Failed to fetch resource with id {} for instance {}", ci.getComponentUid(), ci.getName());
 456         }
 457         Component componentRI = resource.left().value();
 458         Map<String, ArtifactDefinition> childToscaArtifacts = componentRI.getToscaArtifacts();
 459         ArtifactDefinition childArtifactDefinition = childToscaArtifacts.get(ToscaExportHandler.ASSET_TOSCA_TEMPLATE);
 460         if (childArtifactDefinition != null) {
 461             //add to cache
 462             componentCache.put(childArtifactDefinition.getEsId(), childArtifactDefinition.getArtifactName(), componentRI);
 463         }
 464         return componentRI;
 465     }
 466
 467     private Either<byte[], ActionStatus> getEntryData(String cassandraId, Component childComponent) {
 468         if (cassandraId == null || cassandraId.isEmpty()) {
 469             return toscaExportUtils.exportComponent(childComponent).right().map(toscaErrorToActionStatus(childComponent)).left()
 470                 .map(ToscaRepresentation::getMainYaml);
 471         } else {
 472             return getFromCassandra(cassandraId);
 473         }
 474     }
 475
 476     private F<ToscaError, ActionStatus> toscaErrorToActionStatus(Component childComponent) {
 477         return toscaError -> {
 478             LOGGER.debug("Failed to export tosca template for child component {} error {}", childComponent.getUniqueId(), toscaError);
 479             return componentsUtils.convertFromToscaError(toscaError);
 480         };
 481     }
 482
 483     private Either<byte[], ResponseFormat> getLatestSchemaFilesFromCassandra() {
 484         String fto = versionFirstThreeOctets;
 485         return sdcSchemaFilesCassandraDao.getSpecificSchemaFiles(fto, CONFORMANCE_LEVEL).right().map(schemaFilesFetchDBError(fto)).left()
 486             .bind(iff(List::isEmpty, () -> schemaFileFetchError(fto), s -> Either.left(s.iterator().next().getPayloadAsArray())));
 487     }
 488
 489     private F<CassandraOperationStatus, ResponseFormat> schemaFilesFetchDBError(String firstThreeOctets) {
 490         return cos -> {
 491             LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}. Please fix DB table accordingly.", firstThreeOctets,
 492                 CONFORMANCE_LEVEL);
 493             StorageOperationStatus sos = DaoStatusConverter.convertCassandraStatusToStorageStatus(cos);
 494             return componentsUtils.getResponseFormat(componentsUtils.convertFromStorageResponse(sos));
 495         };
 496     }
 497
 498     private Either<byte[], ResponseFormat> schemaFileFetchError(String firstThreeOctets) {
 499         LOGGER.debug("Failed to get the schema files SDC-Version: {} Conformance-Level {}", firstThreeOctets, CONFORMANCE_LEVEL);
 500         return Either.right(componentsUtils.getResponseFormat(ActionStatus.TOSCA_SCHEMA_FILES_NOT_FOUND, firstThreeOctets, CONFORMANCE_LEVEL));
 501     }
 502
 503     /**
 504      * Create a list of all derived nodes found on the package
 505      *
 506      * @param dependencies all node dependencies
 507      * @return a list of nodes
 508      */
 509     private List<String> findNonRootNodesFromPackage(final List<Triple<String, String, Component>> dependencies) {
 510         final List<String> nodes = new ArrayList<>();
 511         if (CollectionUtils.isNotEmpty(dependencies)) {
 512             final String NATIVE_ROOT = "tosca.nodes.Root";
 513             dependencies.forEach(dependency -> {
 514                 if (dependency.getRight() instanceof Resource) {
 515                     final Resource resource = (Resource) dependency.getRight();
 516                     if (CollectionUtils.isNotEmpty(resource.getDerivedList())) {
 517                         resource.getDerivedList().stream().filter(node -> !nodes.contains(node) && !NATIVE_ROOT.equalsIgnoreCase(node))
 518                             .forEach(node -> nodes.add(node));
 519                     }
 520                 }
 521             });
 522         }
 523         return nodes;
 524     }
 525
 526     /**
 527      * Writes to a CSAR zip from casandra schema data
 528      *
 529      * @param zipOutputStream  stores the input stream content
 530      * @param schemaFileZip    zip data from Cassandra
 531      * @param nodesFromPackage list of all nodes found on the onboarded package
 532      */
 533     private void addSchemaFilesFromCassandra(final ZipOutputStream zipOutputStream, final byte[] schemaFileZip, final List<String> nodesFromPackage,
 534                                              final String definitionsPath) {
 535         final int initSize = 2048;
 536         LOGGER.debug("Starting copy from Schema file zip to CSAR zip");
 537         try (final ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(
 538             schemaFileZip)); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(
 539             byteArrayOutputStream, initSize)) {
 540             ZipEntry entry;
 541             while ((entry = zipInputStream.getNextEntry()) != null) {
 542                 ZipUtils.checkForZipSlipInRead(entry);
 543                 final String entryName = entry.getName();
 544                 int readSize = initSize;
 545                 final byte[] entryData = new byte[initSize];
 546                 if (shouldZipEntryBeHandled(entryName)) {
 547                     if (NODES_YML.equalsIgnoreCase(entryName)) {
 548                         handleNode(zipInputStream, byteArrayOutputStream, nodesFromPackage);
 549                     } else {
 550                         while ((readSize = zipInputStream.read(entryData, 0, readSize)) != -1) {
 551                             bufferedOutputStream.write(entryData, 0, readSize);
 552                         }
 553                         bufferedOutputStream.flush();
 554                     }
 555                     byteArrayOutputStream.flush();
 556                     zipOutputStream.putNextEntry(new ZipEntry(definitionsPath + entryName));
 557                     zipOutputStream.write(byteArrayOutputStream.toByteArray());
 558                     zipOutputStream.flush();
 559                     byteArrayOutputStream.reset();
 560                 }
 561             }
 562         } catch (final Exception e) {
 563             LOGGER.error("Error while writing the SDC schema file to the CSAR", e);
 564             throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.GENERAL_ERROR));
 565         }
 566         LOGGER.debug("Finished copy from Schema file zip to CSAR zip");
 567     }
 568
 569     /**
 570      * Handles the nodes.yml zip entry, updating the nodes.yml to avoid duplicated nodes on it.
 571      *
 572      * @param zipInputStream        the zip entry to be read
 573      * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
 574      * @param nodesFromPackage      list of all nodes found on the onboarded package
 575      */
 576     private void handleNode(final ZipInputStream zipInputStream, final ByteArrayOutputStream byteArrayOutputStream,
 577                             final List<String> nodesFromPackage) throws IOException {
 578         final Map<String, Object> nodesFromArtifactFile = readYamlZipEntry(zipInputStream);
 579         final Map<String, Object> nodesYaml = updateNodeYml(nodesFromPackage, nodesFromArtifactFile);
 580         updateZipEntry(byteArrayOutputStream, nodesYaml);
 581     }
 582
 583     /**
 584      * Updates the zip entry from the given parameters
 585      *
 586      * @param byteArrayOutputStream an output stream in which the data is written into a byte array.
 587      * @param nodesYaml             a Map of nodes to be written
 588      */
 589     private void updateZipEntry(final ByteArrayOutputStream byteArrayOutputStream, final Map<String, Object> nodesYaml) throws IOException {
 590         if (MapUtils.isNotEmpty(nodesYaml)) {
 591             byteArrayOutputStream.write(new YamlUtil().objectToYaml(nodesYaml).getBytes());
 592         }
 593     }
 594
 595     /**
 596      * Filters and removes all duplicated nodes found
 597      *
 598      * @param nodesFromPackage      a List of all derived nodes found on the given package
 599      * @param nodesFromArtifactFile represents the nodes.yml file stored in Cassandra
 600      * @return a nodes Map updated
 601      */
 602     private Map<String, Object> updateNodeYml(final List<String> nodesFromPackage, final Map<String, Object> nodesFromArtifactFile) {
 603         if (MapUtils.isNotEmpty(nodesFromArtifactFile)) {
 604             final String nodeTypeBlock = TypeUtils.ToscaTagNamesEnum.NODE_TYPES.getElementName();
 605             final Map<String, Object> nodeTypes = (Map<String, Object>) nodesFromArtifactFile.get(nodeTypeBlock);
 606             nodesFromPackage.stream().filter(nodeTypes::containsKey).forEach(nodeTypes::remove);
 607             nodesFromArtifactFile.replace(nodeTypeBlock, nodeTypes);
 608         }
 609         return nodesFromArtifactFile;
 610     }
 611
 612     /**
 613      * Writes a new zip entry
 614      *
 615      * @param zipInputStream the zip entry to be read
 616      * @return a map of the given zip entry
 617      */
 618     private Map<String, Object> readYamlZipEntry(final ZipInputStream zipInputStream) throws IOException {
 619         final int initSize = 2048;
 620         final StringBuilder zipEntry = new StringBuilder();
 621         final byte[] buffer = new byte[initSize];
 622         int read = 0;
 623         while ((read = zipInputStream.read(buffer, 0, initSize)) >= 0) {
 624             zipEntry.append(new String(buffer, 0, read));
 625         }
 626         return (Map<String, Object>) new Yaml().load(zipEntry.toString());
 627     }
 628
 629     /**
 630      * Checks if the zip entry should or should not be added to the CSAR based on the given global type list
 631      *
 632      * @param entryName the zip entry name
 633      * @return true if the zip entry should be handled
 634      */
 635     private boolean shouldZipEntryBeHandled(final String entryName) {
 636         return ConfigurationManager.getConfigurationManager().getConfiguration().getGlobalCsarImports().stream()
 637             .anyMatch(entry -> entry.contains(entryName));
 638     }
 639
 640     private void addSchemaFilesByModel(final ZipOutputStream zipOutputStream, final String modelName,
 641                                        final String definitionsPath, final boolean isSingleImportsFile) {
 642         try {
 643             final List<ToscaImportByModel> modelDefaultImportList = modelOperation.findAllModelImports(modelName, true);
 644             final Set<Path> writtenEntryPathList = new HashSet<>();
 645             final var defsPath = Path.of(definitionsPath);
 646             Map<Path, byte[]> contentToMerge = new HashMap<>();
 647             for (final ToscaImportByModel toscaImportByModel : modelDefaultImportList) {
 648                 var importPath = Path.of(toscaImportByModel.getFullPath());
 649                 if (!isSingleImportsFile) {
 650                     if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(importPath)).normalize().toString())) {
 651                         final Path entryPath = defsPath.resolve(importPath);
 652                         contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
 653                     } else {
 654                         if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
 655                             importPath =
 656                                 ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
 657                         }
 658                         final Path entryPath = defsPath.resolve(importPath);
 659                         writtenEntryPathList.add(entryPath);
 660                         contentToMerge.put(entryPath, toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8));
 661                     }
 662                 } else {
 663                     if (writtenEntryPathList.contains(defsPath.resolve(importPath))) {
 664                         importPath =
 665                             ToscaDefaultImportHelper.addModelAsFilePrefix(importPath, toscaImportByModel.getModelId());
 666                     }
 667                     final Path entryPath = defsPath.resolve(importPath);
 668                     final var zipEntry = new ZipEntry(entryPath.toString());
 669                     zipOutputStream.putNextEntry(zipEntry);
 670                     writtenEntryPathList.add(entryPath);
 671                     final byte[] content = toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8);
 672                     zipOutputStream.write(content, 0, content.length);
 673                     zipOutputStream.closeEntry();
 674                 }
 675             }
 676             if (!isSingleImportsFile) {
 677                 byte[] mergingContent = new byte[0];
 678                 for (Map.Entry<Path, byte[]> entry : contentToMerge.entrySet()) {
 679                     if (ADDITIONAL_TYPE_DEFINITIONS.equals(Paths.get(String.valueOf(entry.getKey())).normalize().toString())) {
 680                         mergingContent = Bytes.concat(mergingContent, entry.getValue());
 681                     } else {
 682                         final var zipEntry = new ZipEntry(entry.getKey().toString());
 683                         zipOutputStream.putNextEntry(zipEntry);
 684                         writtenEntryPathList.add(entry.getKey());
 685                         final var concat = Bytes.concat(mergingContent, entry.getValue());
 686                         zipOutputStream.write(concat, 0, concat.length);
 687                         zipOutputStream.closeEntry();
 688                     }
 689                 }
 690             }
 691         } catch (final IOException e) {
 692             LOGGER.error(String.valueOf(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR), CsarUtils.class.getName(),
 693                 "Error while writing the schema files by model to the CSAR", e);
 694             throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.CSAR_TOSCA_IMPORTS_ERROR));
 695         }
 696     }
 697
 698     private Either<CsarDefinition, ResponseFormat> collectComponentCsarDefinition(Component component) {
 699         ComponentArtifacts componentArtifacts = new ComponentArtifacts();
 700         Component updatedComponent = component;
 701
 702         //get service to receive the AII artifacts uploaded to the service
 703         if (updatedComponent.getComponentType() == ComponentTypeEnum.SERVICE) {
 704             Either<Service, StorageOperationStatus> getServiceResponse = toscaOperationFacade.getToscaElement(updatedComponent.getUniqueId());
 705
 706             if (getServiceResponse.isRight()) {
 707                 ActionStatus actionStatus = componentsUtils.convertFromStorageResponse(getServiceResponse.right().value());
 708                 return Either.right(componentsUtils.getResponseFormat(actionStatus));
 709             }
 710
 711             updatedComponent = getServiceResponse.left().value();
 712         }
 713
 714         //find the artifacts of the main component, it would have its composed instances artifacts in a separate folder
 715         ComponentTypeArtifacts componentInstanceArtifacts = new ComponentTypeArtifacts();
 716         ArtifactsInfo artifactsInfo = collectComponentArtifacts(updatedComponent);
 717         componentInstanceArtifacts.setComponentArtifacts(artifactsInfo);
 718         componentArtifacts.setMainTypeAndCIArtifacts(componentInstanceArtifacts);
 719
 720         Map<String, ComponentTypeArtifacts> resourceTypeArtifacts = componentArtifacts
 721             .getComponentTypeArtifacts();    //artifacts mapped by the component type(tosca name+version)
 722         //get the component instances
 723         List<ComponentInstance> componentInstances = updatedComponent.getComponentInstances();
 724         if (componentInstances != null) {
 725             for (ComponentInstance componentInstance : componentInstances) {
 726                 //call recursive to find artifacts for all the path
 727                 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
 728                     updatedComponent, componentInstance, resourceTypeArtifacts, componentInstanceArtifacts);
 729                 if (collectComponentInstanceArtifacts.isRight()) {
 730                     return Either.right(collectComponentInstanceArtifacts.right().value());
 731                 }
 732             }
 733         }
 734
 735         if (LOGGER.isDebugEnabled()) {
 736             printResult(componentArtifacts, updatedComponent.getName());
 737         }
 738
 739         return Either.left(new CsarDefinition(componentArtifacts));
 740     }
 741
 742     private void printResult(ComponentArtifacts componentArtifacts, String name) {
 743         StringBuilder result = new StringBuilder();
 744         result.append("Artifacts of main component " + name + "\n");
 745         ComponentTypeArtifacts componentInstanceArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
 746         printArtifacts(componentInstanceArtifacts);
 747         result.append("Type Artifacts\n");
 748         for (Map.Entry<String, ComponentTypeArtifacts> typeArtifacts : componentArtifacts.getComponentTypeArtifacts().entrySet()) {
 749             result.append("Folder " + typeArtifacts.getKey() + "\n");
 750             result.append(printArtifacts(typeArtifacts.getValue()));
 751         }
 752
 753         if (LOGGER.isDebugEnabled()) {
 754             LOGGER.debug(result.toString());
 755         }
 756     }
 757
 758     private String printArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
 759         StringBuilder result = new StringBuilder();
 760         ArtifactsInfo artifactsInfo = componentInstanceArtifacts.getComponentArtifacts();
 761         Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentArtifacts = artifactsInfo.getArtifactsInfo();
 762         printArtifacts(componentArtifacts);
 763         result = result.append("Resources\n");
 764         for (Map.Entry<String, ArtifactsInfo> resourceInstance : componentInstanceArtifacts.getComponentInstancesArtifacts().entrySet()) {
 765             result.append("Folder" + resourceInstance.getKey() + "\n");
 766             result.append(printArtifacts(resourceInstance.getValue().getArtifactsInfo()));
 767         }
 768
 769         return result.toString();
 770     }
 771
 772     private String printArtifacts(Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componetArtifacts) {
 773         StringBuilder result = new StringBuilder();
 774         for (Map.Entry<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactGroup : componetArtifacts.entrySet()) {
 775             result.append("    " + artifactGroup.getKey().getType());
 776             for (Map.Entry<String, List<ArtifactDefinition>> groupArtifacts : artifactGroup.getValue().entrySet()) {
 777                 result.append("        " + groupArtifacts.getKey());
 778                 for (ArtifactDefinition artifact : groupArtifacts.getValue()) {
 779                     result.append("            " + artifact.getArtifactDisplayName());
 780                 }
 781             }
 782         }
 783
 784         return result.toString();
 785     }
 786
 787     private Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts(Component parentComponent, ComponentInstance componentInstance,
 788                                                                               Map<String, ComponentTypeArtifacts> resourcesTypeArtifacts,
 789                                                                               ComponentTypeArtifacts instanceArtifactsLocation) {
 790         //1. get the component instance component
 791         String componentUid;
 792         if (componentInstance.getOriginType() == OriginTypeEnum.ServiceProxy) {
 793             componentUid = componentInstance.getSourceModelUid();
 794         } else {
 795             componentUid = componentInstance.getComponentUid();
 796         }
 797         Either<Component, StorageOperationStatus> component = toscaOperationFacade.getToscaElement(componentUid);
 798         if (component.isRight()) {
 799             LOGGER.error("Failed to fetch resource with id {} for instance {}", componentUid, parentComponent.getUUID());
 800             return Either.right(componentsUtils.getResponseFormat(ActionStatus.ASSET_NOT_FOUND_DURING_CSAR_CREATION,
 801                 parentComponent.getComponentType().getValue(), parentComponent.getUUID(),
 802                 componentInstance.getOriginType().getComponentType().getValue(), componentUid));
 803         }
 804         Component fetchedComponent = component.left().value();
 805
 806         //2. fill the artifacts for the current component parent type
 807         String toscaComponentName =
 808             componentInstance.getToscaComponentName() + "_v" + componentInstance.getComponentVersion();
 809
 810         // if there are no artifacts for this component type we need to fetch and build them
 811         ComponentTypeArtifacts componentParentArtifacts = Optional
 812             .ofNullable(resourcesTypeArtifacts.get(toscaComponentName))
 813             .orElseGet(() -> collectComponentTypeArtifacts(fetchedComponent));
 814
 815         if (componentParentArtifacts.getComponentArtifacts().isNotEmpty()) {
 816             resourcesTypeArtifacts.put(toscaComponentName, componentParentArtifacts);
 817         }
 818
 819         //3. find the artifacts specific to the instance
 820         Map<String, List<ArtifactDefinition>> componentInstanceSpecificInformationalArtifacts =
 821             getComponentInstanceSpecificArtifacts(componentInstance.getArtifacts(),
 822                 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.INFORMATIONAL);
 823         Map<String, List<ArtifactDefinition>> componentInstanceSpecificDeploymentArtifacts =
 824             getComponentInstanceSpecificArtifacts(componentInstance.getDeploymentArtifacts(),
 825                 componentParentArtifacts.getComponentArtifacts().getArtifactsInfo(), ArtifactGroupTypeEnum.DEPLOYMENT);
 826
 827         //4. add the instances artifacts to the component type
 828         ArtifactsInfo artifactsInfo = new ArtifactsInfo();
 829         if (!componentInstanceSpecificInformationalArtifacts.isEmpty()) {
 830             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, componentInstanceSpecificInformationalArtifacts);
 831         }
 832         if (!componentInstanceSpecificDeploymentArtifacts.isEmpty()) {
 833             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, componentInstanceSpecificDeploymentArtifacts);
 834         }
 835         if (!artifactsInfo.isEmpty()) {
 836             instanceArtifactsLocation.addComponentInstancesArtifacts(componentInstance.getNormalizedName(), artifactsInfo);
 837         }
 838
 839         //5. do the same for all the component instances
 840         List<ComponentInstance> componentInstances = fetchedComponent.getComponentInstances();
 841         if (componentInstances != null) {
 842             for (ComponentInstance childComponentInstance : componentInstances) {
 843                 Either<Boolean, ResponseFormat> collectComponentInstanceArtifacts = collectComponentInstanceArtifacts(
 844                     fetchedComponent, childComponentInstance, resourcesTypeArtifacts, componentParentArtifacts);
 845                 if (collectComponentInstanceArtifacts.isRight()) {
 846                     return collectComponentInstanceArtifacts;
 847                 }
 848             }
 849         }
 850
 851         return Either.left(true);
 852     }
 853
 854     private Map<String, List<ArtifactDefinition>> getComponentInstanceSpecificArtifacts(Map<String, ArtifactDefinition> componentArtifacts,
 855                                                                                         Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentTypeArtifacts,
 856                                                                                         ArtifactGroupTypeEnum artifactGroupTypeEnum) {
 857         Map<String, List<ArtifactDefinition>> parentArtifacts = componentTypeArtifacts
 858             .get(artifactGroupTypeEnum);    //the artfiacts of the component itself and not the instance
 859
 860         Map<String, List<ArtifactDefinition>> artifactsByTypeOfComponentInstance = new HashMap<>();
 861         if (componentArtifacts != null) {
 862             for (ArtifactDefinition artifact : componentArtifacts.values()) {
 863                 List<ArtifactDefinition> parentArtifactsByType = null;
 864                 if (parentArtifacts != null) {
 865                     parentArtifactsByType = parentArtifacts.get(artifact.getArtifactType());
 866                 }
 867                 //the artifact is of instance
 868                 if (parentArtifactsByType == null || !parentArtifactsByType.contains(artifact)) {
 869                     List<ArtifactDefinition> typeArtifacts = artifactsByTypeOfComponentInstance.get(artifact.getArtifactType());
 870                     if (typeArtifacts == null) {
 871                         typeArtifacts = new ArrayList<>();
 872                         artifactsByTypeOfComponentInstance.put(artifact.getArtifactType(), typeArtifacts);
 873                     }
 874                     typeArtifacts.add(artifact);
 875                 }
 876             }
 877         }
 878
 879         return artifactsByTypeOfComponentInstance;
 880     }
 881
 882     private ComponentTypeArtifacts collectComponentTypeArtifacts(Component fetchedComponent) {
 883         ArtifactsInfo componentArtifacts = collectComponentArtifacts(fetchedComponent);
 884         ComponentTypeArtifacts componentArtifactsInfo = new ComponentTypeArtifacts();
 885         if (componentArtifacts.isNotEmpty()) {
 886             componentArtifactsInfo.setComponentArtifacts(componentArtifacts);
 887         }
 888         return componentArtifactsInfo;
 889     }
 890
 891     private ArtifactsInfo collectComponentArtifacts(Component component) {
 892         Map<String, ArtifactDefinition> informationalArtifacts = component.getArtifacts();
 893         Map<String, List<ArtifactDefinition>> informationalArtifactsByType = collectGroupArtifacts(informationalArtifacts);
 894         Map<String, ArtifactDefinition> deploymentArtifacts = component.getDeploymentArtifacts();
 895         Map<String, List<ArtifactDefinition>> deploymentArtifactsByType = collectGroupArtifacts(deploymentArtifacts);
 896         ArtifactsInfo artifactsInfo = new ArtifactsInfo();
 897         if (!informationalArtifactsByType.isEmpty()) {
 898             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.INFORMATIONAL, informationalArtifactsByType);
 899         }
 900         if (!deploymentArtifactsByType.isEmpty()) {
 901             artifactsInfo.addArtifactsToGroup(ArtifactGroupTypeEnum.DEPLOYMENT, deploymentArtifactsByType);
 902         }
 903
 904         return artifactsInfo;
 905     }
 906
 907     private Map<String, List<ArtifactDefinition>> collectGroupArtifacts(
 908         final Map<String, ArtifactDefinition> componentArtifacts) {
 909         final Map<String, List<ArtifactDefinition>> artifactsByType = new HashMap<>();
 910         for (final ArtifactDefinition artifact : componentArtifacts.values()) {
 911             if (artifact.getArtifactUUID() != null) {
 912                 artifactsByType.putIfAbsent(artifact.getArtifactType(), new ArrayList<>());
 913                 final List<ArtifactDefinition> typeArtifacts = artifactsByType.get(artifact.getArtifactType());
 914                 typeArtifacts.add(artifact);
 915             }
 916         }
 917         return artifactsByType;
 918     }
 919
 920     private Either<ZipOutputStream, ResponseFormat> writeAllFilesToCsar(Component mainComponent, CsarDefinition csarDefinition,
 921                                                                         ZipOutputStream zipstream, boolean isInCertificationRequest)
 922         throws IOException {
 923         ComponentArtifacts componentArtifacts = csarDefinition.getComponentArtifacts();
 924         Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath = writeComponentArtifactsToSpecifiedPath(mainComponent,
 925             componentArtifacts, zipstream, ARTIFACTS_PATH, isInCertificationRequest);
 926         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
 927             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
 928         }
 929         ComponentTypeArtifacts mainTypeAndCIArtifacts = componentArtifacts.getMainTypeAndCIArtifacts();
 930         writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, mainTypeAndCIArtifacts.getComponentArtifacts(),
 931             zipstream, ARTIFACTS_PATH, isInCertificationRequest);
 932         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
 933             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
 934         }
 935         Map<String, ArtifactsInfo> componentInstancesArtifacts = mainTypeAndCIArtifacts.getComponentInstancesArtifacts();
 936         String currentPath = ARTIFACTS_PATH + RESOURCES_PATH;
 937         for (String keyAssetName : componentInstancesArtifacts.keySet()) {
 938             ArtifactsInfo artifactsInfo = componentInstancesArtifacts.get(keyAssetName);
 939             String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
 940             writeComponentArtifactsToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent, artifactsInfo, zipstream, pathWithAssetName,
 941                 isInCertificationRequest);
 942             if (writeComponentArtifactsToSpecifiedPath.isRight()) {
 943                 return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
 944             }
 945         }
 946         writeComponentArtifactsToSpecifiedPath = writeOperationsArtifactsToCsar(mainComponent, zipstream);
 947         if (writeComponentArtifactsToSpecifiedPath.isRight()) {
 948             return Either.right(writeComponentArtifactsToSpecifiedPath.right().value());
 949         }
 950         return Either.left(zipstream);
 951     }
 952
 953     private Either<ZipOutputStream, ResponseFormat> writeOperationsArtifactsToCsar(Component component, ZipOutputStream zipstream) {
 954         if (checkComponentBeforeOperation(component)) {
 955             return Either.left(zipstream);
 956         }
 957         for (Map.Entry<String, InterfaceDefinition> interfaceEntry : ((Resource) component).getInterfaces().entrySet()) {
 958             for (OperationDataDefinition operation : interfaceEntry.getValue().getOperations().values()) {
 959                 try {
 960                     if (checkComponentBeforeWrite(component, interfaceEntry, operation)) {
 961                         continue;
 962                     }
 963                     final String artifactUUID = operation.getImplementation().getArtifactUUID();
 964                     if (artifactUUID == null) {
 965                         continue;
 966                     }
 967                     final Either<byte[], ActionStatus> artifactFromCassandra = getFromCassandra(artifactUUID);
 968                     final String artifactName = operation.getImplementation().getArtifactName();
 969                     if (artifactFromCassandra.isRight()) {
 970                         LOGGER.error(ARTIFACT_NAME_UNIQUE_ID, artifactName, artifactUUID);
 971                         LOGGER.error("Failed to get {} payload from DB reason: {}", artifactName, artifactFromCassandra.right().value());
 972                         return Either.right(componentsUtils.getResponseFormat(
 973                             ARTIFACT_PAYLOAD_NOT_FOUND_DURING_CSAR_CREATION, "Resource", component.getUniqueId(), artifactName, artifactUUID));
 974                     }
 975                     zipstream.putNextEntry(new ZipEntry(OperationArtifactUtil.createOperationArtifactPath(component, null, operation, true)));
 976                     zipstream.write(artifactFromCassandra.left().value());
 977                 } catch (IOException e) {
 978                     LOGGER.error("Component Name {},  Interface Name {}, Operation Name {}", component.getNormalizedName(), interfaceEntry.getKey(),
 979                         operation.getName());
 980                     LOGGER.error("Error while writing the operation's artifacts to the CSAR", e);
 981                     return Either.right(componentsUtils.getResponseFormat(ERROR_DURING_CSAR_CREATION, "Resource", component.getUniqueId()));
 982                 }
 983             }
 984         }
 985         return Either.left(zipstream);
 986     }
 987
 988     private boolean checkComponentBeforeWrite(Component component, Map.Entry<String, InterfaceDefinition> interfaceEntry,
 989                                               OperationDataDefinition operation) {
 990         final ArtifactDataDefinition implementation = operation.getImplementation();
 991         if (implementation == null) {
 992             LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no Operation Implementation found", component.getNormalizedName(),
 993                 interfaceEntry.getValue().getUniqueId(), operation.getName());
 994             return true;
 995         }
 996         final String artifactName = implementation.getArtifactName();
 997         if (artifactName == null) {
 998             LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - no artifact found", component.getNormalizedName(),
 999                 interfaceEntry.getValue().getUniqueId(), operation.getName());
1000             return true;
1001         }
1002         if (OperationArtifactUtil.artifactNameIsALiteralValue(artifactName)) {
1003             LOGGER.debug("Component Name {}, Interface Id {}, Operation Name {} - artifact name is a literal value rather than an SDC artifact",
1004                 component.getNormalizedName(), interfaceEntry.getValue().getUniqueId(), operation.getName());
1005             return true;
1006         }
1007         return false;
1008     }
1009
1010     private boolean checkComponentBeforeOperation(Component component) {
1011         if (component instanceof Service) {
1012             return true;
1013         }
1014         if (Objects.isNull(((Resource) component).getInterfaces())) {
1015             LOGGER.debug("Component Name {}- no interfaces found", component.getNormalizedName());
1016             return true;
1017         }
1018         return false;
1019     }
1020
1021     private Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath(final Component mainComponent,
1022                                                                                       final ArtifactsInfo currArtifactsInfo,
1023                                                                                       final ZipOutputStream zip, final String path,
1024                                                                                       final boolean isInCertificationRequest) throws IOException {
1025         final Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfo = currArtifactsInfo.getArtifactsInfo();
1026         for (final ArtifactGroupTypeEnum artifactGroupTypeEnum : artifactsInfo.keySet()) {
1027             final String groupTypeFolder = path + WordUtils.capitalizeFully(artifactGroupTypeEnum.getType()) + PATH_DELIMITER;
1028             final Map<String, List<ArtifactDefinition>> artifactTypesMap = artifactsInfo.get(artifactGroupTypeEnum);
1029             for (final String artifactType : artifactTypesMap.keySet()) {
1030                 final List<ArtifactDefinition> artifactDefinitionList = artifactTypesMap.get(artifactType);
1031                 String artifactTypeFolder = groupTypeFolder + artifactType + PATH_DELIMITER;
1032                 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType) && path.contains(ARTIFACTS_PATH + RESOURCES_PATH)) {
1033                     // Ignore this packaging as BPMN artifacts needs to be packaged in different manner
1034                     continue;
1035                 }
1036                 if (ArtifactTypeEnum.WORKFLOW.getType().equals(artifactType)) {
1037                     artifactTypeFolder += OperationArtifactUtil.BPMN_ARTIFACT_PATH + File.separator;
1038                 } else if (ArtifactTypeEnum.ONBOARDED_PACKAGE.getType().equals(artifactType)) {
1039                     // renaming legacy folder ONBOARDED_PACKAGE to the new folder ETSI_PACKAGE
1040                     artifactTypeFolder = artifactTypeFolder
1041                         .replace(ArtifactTypeEnum.ONBOARDED_PACKAGE.getType(), ArtifactTypeEnum.ETSI_PACKAGE.getType());
1042                 }
1043                 // TODO: We should not do this but in order to keep this refactoring small enough,
1044
1045                 // we'll leave this as is for now
1046                 List<ArtifactDefinition> collect = filterArtifactDefinitionToZip(mainComponent, artifactDefinitionList, isInCertificationRequest)
1047                     .collect(Collectors.toList());
1048                 for (ArtifactDefinition ad : collect) {
1049                     zip.putNextEntry(new ZipEntry(artifactTypeFolder + ad.getArtifactName()));
1050                     zip.write(ad.getPayloadData());
1051                 }
1052             }
1053         }
1054         return Either.left(zip);
1055     }
1056
1057     private Stream<ArtifactDefinition> filterArtifactDefinitionToZip(Component mainComponent, List<ArtifactDefinition> artifactDefinitionList,
1058                                                                      boolean isInCertificationRequest) {
1059         return artifactDefinitionList.stream().filter(shouldBeInZip(isInCertificationRequest, mainComponent)).map(this::fetchPayLoadData)
1060             .filter(Either::isLeft).map(e -> e.left().value());
1061     }
1062
1063     private Predicate<ArtifactDefinition> shouldBeInZip(boolean isInCertificationRequest, Component component) {
1064         return artifactDefinition -> !(!isInCertificationRequest && component.isService() && artifactDefinition.isHeatEnvType() || artifactDefinition
1065             .hasNoMandatoryEsId());
1066     }
1067
1068     private Either<ArtifactDefinition, ActionStatus> fetchPayLoadData(ArtifactDefinition ad) {
1069         byte[] payloadData = ad.getPayloadData();
1070         if (payloadData == null) {
1071             return getFromCassandra(ad.getEsId()).left().map(pd -> {
1072                 ad.setPayload(pd);
1073                 return ad;
1074             }).right().map(as -> {
1075                 LOGGER.debug(ARTIFACT_NAME_UNIQUE_ID, ad.getArtifactName(), ad.getUniqueId());
1076                 LOGGER.debug("Failed to get {} payload from DB reason: {}", ad.getArtifactName(), as);
1077                 return as;
1078             });
1079         } else {
1080             return Either.left(ad);
1081         }
1082     }
1083
1084     private Either<ZipOutputStream, ResponseFormat> writeComponentArtifactsToSpecifiedPath(Component mainComponent,
1085                                                                                            ComponentArtifacts componentArtifacts,
1086                                                                                            ZipOutputStream zipstream, String currentPath,
1087                                                                                            boolean isInCertificationRequest) throws IOException {
1088         Map<String, ComponentTypeArtifacts> componentTypeArtifacts = componentArtifacts.getComponentTypeArtifacts();
1089         //Keys are defined:
1090
1091         //<Inner Asset TOSCA name (e.g. VFC name)> folder name: <Inner Asset TOSCA name (e.g. VFC name)>_v<version>.
1092
1093         //E.g. "org.openecomp.resource.vf.vipr_atm_v1.0"
1094         Set<String> componentTypeArtifactsKeys = componentTypeArtifacts.keySet();
1095         for (String keyAssetName : componentTypeArtifactsKeys) {
1096             ComponentTypeArtifacts componentInstanceArtifacts = componentTypeArtifacts.get(keyAssetName);
1097             ArtifactsInfo componentArtifacts2 = componentInstanceArtifacts.getComponentArtifacts();
1098             String pathWithAssetName = currentPath + keyAssetName + PATH_DELIMITER;
1099             Either<ZipOutputStream, ResponseFormat> writeArtifactsInfoToSpecifiedPath = writeArtifactsInfoToSpecifiedPath(mainComponent,
1100                 componentArtifacts2, zipstream, pathWithAssetName, isInCertificationRequest);
1101             if (writeArtifactsInfoToSpecifiedPath.isRight()) {
1102                 return writeArtifactsInfoToSpecifiedPath;
1103             }
1104         }
1105         return Either.left(zipstream);
1106     }
1107
1108     private Either<ToscaRepresentation, ResponseFormat> generateToscaRepresentation(Component component, boolean isSkipImports) {
1109         return toscaExportUtils.exportComponent(component, isSkipImports).right().map(toscaError -> {
1110             LOGGER.debug("exportComponent failed {}", toscaError);
1111             return componentsUtils.getResponseFormat(componentsUtils.convertFromToscaError(toscaError));
1112         });
1113     }
1114
1115     private String createToscaBlock0(String metaFileVersion, String csarVersion, String createdBy, String entryDef, boolean isAsdPackage,
1116                                      String definitionsPath) {
1117         final String block0template = "TOSCA-Meta-File-Version: %s\nCSAR-Version: %s\nCreated-By: %s\nEntry-Definitions: "
1118             + definitionsPath + "%s\n%s\nName: csar.meta\nContent-Type: text/plain\n";
1119         return String.format(block0template, metaFileVersion, csarVersion, createdBy, entryDef, isAsdPackage ? "entry_definition_type: asd" : "");
1120     }
1121
1122     private class CsarDefinition {
1123
1124         private ComponentArtifacts componentArtifacts;
1125
1126         // add list of tosca artifacts and meta describes CSAR zip root
1127         public CsarDefinition(ComponentArtifacts componentArtifacts) {
1128             this.componentArtifacts = componentArtifacts;
1129         }
1130
1131         public ComponentArtifacts getComponentArtifacts() {
1132             return componentArtifacts;
1133         }
1134     }
1135
1136     private class ComponentArtifacts {
1137
1138         //artifacts of the component and CI's artifacts contained in it's composition (represents Informational, Deployment & Resource folders of main component)
1139         private ComponentTypeArtifacts mainTypeAndCIArtifacts;
1140         //artifacts of all component types mapped by their tosca name
1141         private Map<String, ComponentTypeArtifacts> componentTypeArtifacts;
1142
1143         public ComponentArtifacts() {
1144             mainTypeAndCIArtifacts = new ComponentTypeArtifacts();
1145             componentTypeArtifacts = new HashMap<>();
1146         }
1147
1148         public ComponentTypeArtifacts getMainTypeAndCIArtifacts() {
1149             return mainTypeAndCIArtifacts;
1150         }
1151
1152         public void setMainTypeAndCIArtifacts(ComponentTypeArtifacts componentInstanceArtifacts) {
1153             this.mainTypeAndCIArtifacts = componentInstanceArtifacts;
1154         }
1155
1156         public Map<String, ComponentTypeArtifacts> getComponentTypeArtifacts() {
1157             return componentTypeArtifacts;
1158         }
1159     }
1160
1161     /**
1162      * The artifacts of the component and of all its composed instances
1163      */
1164     private class ComponentTypeArtifacts {
1165
1166         private ArtifactsInfo componentArtifacts;    //component artifacts (describes the Informational Deployment folders)
1167
1168         private Map<String, ArtifactsInfo> componentInstancesArtifacts;        //artifacts of the composed instances mapped by the resourceInstance normalized name (describes the Resources folder)
1169
1170         public ComponentTypeArtifacts() {
1171             componentArtifacts = new ArtifactsInfo();
1172             componentInstancesArtifacts = new HashMap<>();
1173         }
1174
1175         public ArtifactsInfo getComponentArtifacts() {
1176             return componentArtifacts;
1177         }
1178
1179         public void setComponentArtifacts(ArtifactsInfo artifactsInfo) {
1180             this.componentArtifacts = artifactsInfo;
1181         }
1182
1183         public Map<String, ArtifactsInfo> getComponentInstancesArtifacts() {
1184             return componentInstancesArtifacts;
1185         }
1186
1187         public void addComponentInstancesArtifacts(String normalizedName, ArtifactsInfo artifactsInfo) {
1188             componentInstancesArtifacts.put(normalizedName, artifactsInfo);
1189         }
1190     }
1191
1192     /**
1193      * The artifacts Definition saved by their structure
1194      */
1195     private class ArtifactsInfo {
1196         //Key is the type of artifacts(Informational/Deployment)
1197
1198         //Value is a map between an artifact type and a list of all artifacts of this type
1199         private Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> artifactsInfoField;
1200
1201         public ArtifactsInfo() {
1202             this.artifactsInfoField = new EnumMap<>(ArtifactGroupTypeEnum.class);
1203         }
1204
1205         public Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> getArtifactsInfo() {
1206             return artifactsInfoField;
1207         }
1208
1209         public void addArtifactsToGroup(ArtifactGroupTypeEnum artifactGroup, Map<String, List<ArtifactDefinition>> artifactsDefinition) {
1210             if (artifactsInfoField.get(artifactGroup) == null) {
1211                 artifactsInfoField.put(artifactGroup, artifactsDefinition);
1212             } else {
1213                 Map<String, List<ArtifactDefinition>> artifactTypeEnumListMap = artifactsInfoField.get(artifactGroup);
1214                 artifactTypeEnumListMap.putAll(artifactsDefinition);
1215                 artifactsInfoField.put(artifactGroup, artifactTypeEnumListMap);
1216             }
1217         }
1218
1219         public boolean isEmpty() {
1220             return artifactsInfoField.isEmpty();
1221         }
1222
1223         public boolean isNotEmpty() {
1224             return !isEmpty();
1225         }
1226     }
1227
1228     public static class ToscaErrorException extends Exception {
1229
1230         ToscaErrorException(ToscaError error) {
1231             super("Error while exporting component's interface (toscaError:" + error + ")");
1232         }
1233     }
1234
1235 }