asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/ArtifactUuidFix.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23 import com.google.gson.Gson;
  24 import com.google.gson.GsonBuilder;
  25 import com.google.gson.JsonArray;
  26 import fj.data.Either;
  27 import org.openecomp.sdc.asdctool.impl.validator.utils.VfModuleArtifactPayloadEx;
  28 import org.openecomp.sdc.be.components.distribution.engine.VfModuleArtifactPayload;
  29 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  30 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  31 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
  32 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
  33 import org.openecomp.sdc.be.dao.jsongraph.JanusGraphDao;
  34 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
  35 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
  36 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
  37 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
  38 import org.openecomp.sdc.be.datatypes.elements.GroupDataDefinition;
  39 import org.openecomp.sdc.be.datatypes.elements.MapArtifactDataDefinition;
  40 import org.openecomp.sdc.be.datatypes.elements.MapGroupsDataDefinition;
  41 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
  42 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
  43 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
  44 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  45 import org.openecomp.sdc.be.datatypes.tosca.ToscaDataDefinition;
  46 import org.openecomp.sdc.be.impl.ComponentsUtils;
  47 import org.openecomp.sdc.be.model.ArtifactDefinition;
  48 import org.openecomp.sdc.be.model.Component;
  49 import org.openecomp.sdc.be.model.ComponentInstance;
  50 import org.openecomp.sdc.be.model.ComponentParametersView;
  51 import org.openecomp.sdc.be.model.DistributionStatusEnum;
  52 import org.openecomp.sdc.be.model.GroupDefinition;
  53 import org.openecomp.sdc.be.model.GroupInstance;
  54 import org.openecomp.sdc.be.model.LifecycleStateEnum;
  55 import org.openecomp.sdc.be.model.Resource;
  56 import org.openecomp.sdc.be.model.Service;
  57 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.TopologyTemplate;
  58 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.ToscaElement;
  59 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
  60 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
  61 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  62 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
  63 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
  64 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
  65 import org.openecomp.sdc.be.tosca.CsarUtils;
  66 import org.openecomp.sdc.be.tosca.ToscaError;
  67 import org.openecomp.sdc.be.tosca.ToscaExportHandler;
  68 import org.openecomp.sdc.be.tosca.ToscaRepresentation;
  69 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
  70 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
  71 import org.openecomp.sdc.common.api.Constants;
  72 import org.openecomp.sdc.common.log.wrappers.Logger;
  73 import org.openecomp.sdc.common.util.GeneralUtility;
  74 import org.openecomp.sdc.exception.ResponseFormat;
  75 import org.springframework.beans.factory.annotation.Autowired;
  76
  77 import java.io.BufferedWriter;
  78 import java.io.FileOutputStream;
  79 import java.io.IOException;
  80 import java.io.OutputStreamWriter;
  81 import java.io.Writer;
  82 import java.util.ArrayList;
  83 import java.util.Collections;
  84 import java.util.EnumMap;
  85 import java.util.HashMap;
  86 import java.util.HashSet;
  87 import java.util.List;
  88 import java.util.Map;
  89 import java.util.Optional;
  90 import java.util.Set;
  91 import java.util.stream.Collectors;
  92
  93 @org.springframework.stereotype.Component("artifactUuidFix")
  94 public class ArtifactUuidFix {
  95
  96         private static final String MIGRATION1707_ARTIFACT_UUID_FIX = "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ";
  97
  98         private static final String FAILED_TO_FETCH_VF_RESOURCES = "Failed to fetch vf resources ";
  99
 100         private static final String UTF8 = "utf-8";
 101
 102         private JanusGraphDao janusGraphDao;
 103         private ToscaOperationFacade toscaOperationFacade;
 104         private ToscaExportHandler toscaExportUtils;
 105         private ArtifactCassandraDao artifactCassandraDao;
 106         private CsarUtils csarUtils;
 107
 108         private static Logger log = Logger.getLogger(ArtifactUuidFix.class.getName());
 109
 110         @Autowired
 111         public ArtifactUuidFix(JanusGraphDao janusGraphDao,
 112                 ToscaOperationFacade toscaOperationFacade, ToscaExportHandler toscaExportUtils,
 113                 ArtifactCassandraDao artifactCassandraDao, CsarUtils csarUtils) {
 114                 this.janusGraphDao = janusGraphDao;
 115                 this.toscaOperationFacade = toscaOperationFacade;
 116                 this.toscaExportUtils = toscaExportUtils;
 117                 this.artifactCassandraDao = artifactCassandraDao;
 118                 this.csarUtils = csarUtils;
 119         }
 120
 121         public boolean doFix(String fixComponent, String runMode) {
 122                 List<Resource> vfLst = new ArrayList<>();
 123                 List<Service> serviceList = new ArrayList<>();
 124                 Map<String, List<Component>> nodeToFixTosca = new HashMap<>();
 125                 Map<String, List<Component>> vfToFixTosca = new HashMap<>();
 126                 Map<String, List<Component>> serviceToFixTosca = new HashMap<>();
 127
 128                 long time = System.currentTimeMillis();
 129
 130                 doFixTosca(nodeToFixTosca, vfToFixTosca, serviceToFixTosca);
 131
 132                 if ("vf_only".equals(fixComponent)) {
 133                         if (!fetchFaultVf(fixComponent, vfLst, time)) {
 134                                 return false;
 135                         }
 136                 } else {
 137                         if (!fetchServices(fixComponent, serviceList, time)) {
 138                                 return false;
 139                         }
 140                 }
 141                 if ("service_vf".equals(runMode) || "fix".equals(runMode)) {
 142                         log.info("Mode {}. Find problem VFs", runMode);
 143                         if (!fetchVf(serviceList, vfLst, time)) {
 144                                 log.info("Mode {}. Find problem VFs finished with failure", runMode);
 145                                 return false;
 146                         }
 147                         log.info("Mode {}. Find problem VFs finished with success", runMode);
 148                 }
 149                 if ("fix".equals(runMode) || "fix_only_services".equals(runMode)) {
 150                         log.info("Mode {}. Start fix", runMode);
 151                         if (!fix(vfLst, serviceList, nodeToFixTosca, vfToFixTosca, serviceToFixTosca)) {
 152                                 log.info("Mode {}. Fix finished with failure", runMode);
 153                                 return false;
 154                         }
 155                         log.info("Mode {}. Fix finished with success", runMode);
 156                 }
 157
 158                 return true;
 159         }
 160
 161         private boolean fetchFaultVf(String fixComponent, List<Resource> vfLst, long time) {
 162                 log.info("Find fault VF ");
 163                 String fileName = "fault_" + time + ".csv";
 164                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 165                         writer.write("vf name, vf id, state, version\n");
 166
 167                         Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
 168                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
 169                         hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF.name());
 170
 171                         Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
 172                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 173                         log.info("Try to fetch resources with properties {} and not {}", hasProps, hasNotProps);
 174
 175                         Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
 176                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 177                         if (servicesByCriteria.isRight()) {
 178                                 log.info("Failed to fetch resources {}", servicesByCriteria.right().value());
 179                                 return false;
 180                         }
 181                         List<GraphVertex> resources = servicesByCriteria.left().value();
 182                         for (GraphVertex gv : resources) {
 183                                 ComponentParametersView filter = new ComponentParametersView(true);
 184                                 filter.setIgnoreComponentInstances(false);
 185                                 filter.setIgnoreArtifacts(false);
 186                                 filter.setIgnoreGroups(false);
 187
 188                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 189                                                 .getToscaElement(gv.getUniqueId());
 190                                 if (toscaElement.isRight()) {
 191                                         log.info("Failed to fetch resources {} {}", gv.getUniqueId(), toscaElement.right().value());
 192                                         continue;
 193                                 }
 194
 195                                 Resource resource = toscaElement.left().value();
 196                                 String resourceName = resource.getName();
 197                                 Map<String, ArtifactDefinition> deploymentArtifacts = resource.getDeploymentArtifacts();
 198                                 List<GroupDefinition> groups = resource.getGroups();
 199                                 if (groups == null || groups.isEmpty()) {
 200                                         log.info("No groups for resource {} id {} ", resourceName, gv.getUniqueId());
 201                                         continue;
 202                                 }
 203                                 boolean isProblematic = false;
 204                                 for (GroupDefinition gr : groups) {
 205                                         if (gr.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 206                                                 if (isProblematicGroup(gr, resourceName, deploymentArtifacts)) {
 207                                                         isProblematic = true;
 208                                                         break;
 209                                                 }
 210                                         }
 211                                 }
 212                                 if (isProblematic) {
 213                                         vfLst.add(resource);
 214                                         writeModuleResultToFile(writer, resource, null);
 215                                         writer.flush();
 216                                 }
 217                                 janusGraphDao.commit();
 218                         }
 219
 220                 } catch (Exception e) {
 221                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
 222                         return false;
 223                 } finally {
 224                         janusGraphDao.commit();
 225                 }
 226                 return true;
 227         }
 228
 229         private boolean fetchVf(List<Service> serviceList, List<Resource> vfLst, long time) {
 230                 log.info("Find problem VF ");
 231                 if (serviceList.isEmpty()) {
 232                         log.info("No services as input");
 233                         return true;
 234                 }
 235                 String fileName = "problemVf_" + time + ".csv";
 236                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 237                         writer.write("vf name, vf id, state, version, example service name\n");
 238                         Set<String> vfIds = new HashSet<>();
 239                         for (Service service : serviceList) {
 240                                 List<ComponentInstance> componentInstances = service.getComponentInstances().stream()
 241                                                 .filter(ci -> ci.getOriginType().equals(OriginTypeEnum.VF)).collect(Collectors.toList());
 242                                 for (ComponentInstance ci : componentInstances) {
 243                                         if (!vfIds.contains(ci.getComponentUid())) {
 244                                                 vfIds.add(ci.getComponentUid());
 245                                                 ComponentParametersView filter = new ComponentParametersView(true);
 246                                                 filter.setIgnoreComponentInstances(false);
 247                                                 filter.setIgnoreArtifacts(false);
 248                                                 filter.setIgnoreGroups(false);
 249                                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 250                                                                 .getToscaElement(ci.getComponentUid(), filter);
 251                                                 if (toscaElement.isRight()) {
 252                                                         log.info("Failed to fetch resource {} {}", ci.getComponentUid(),
 253                                                                         toscaElement.right().value());
 254                                                         continue;
 255                                                 }
 256                                                 Resource resource = toscaElement.left().value();
 257                                                 if (resource.getResourceType().equals(ResourceTypeEnum.VF)) {
 258                                                         vfLst.add(resource);
 259                                                         writeModuleResultToFile(writer, resource, service);
 260                                                         writer.flush();
 261
 262                                                 }
 263                                                 janusGraphDao.commit();
 264                                         }
 265                                 }
 266                         }
 267                         log.info("output file with list of Vf : {}", fileName);
 268                 } catch (Exception e) {
 269                         log.info("Failed to fetch services ", e);
 270                         return false;
 271                 } finally {
 272                         janusGraphDao.commit();
 273                 }
 274                 return true;
 275         }
 276
 277         private boolean fetchServices(String fixServices, List<Service> serviceList, long time) {
 278                 log.info("Find problem Services {}", fixServices);
 279                 String fileName = "problemService_" + time + ".csv";
 280                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 281                         writer.write("service name, service id, state, version\n");
 282
 283                         Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
 284                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
 285                         if ("distributed_only".equals(fixServices)) {
 286                                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
 287                                 hasProps.put(GraphPropertyEnum.DISTRIBUTION_STATUS, DistributionStatusEnum.DISTRIBUTED.name());
 288                         }
 289
 290                         Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
 291                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 292                         log.info("Try to fetch services with properties {} and not {}", hasProps, hasNotProps);
 293
 294                         Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
 295                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 296                         if (servicesByCriteria.isRight()) {
 297                                 log.info("Failed to fetch services {}", servicesByCriteria.right().value());
 298                                 return false;
 299                         }
 300                         List<GraphVertex> services = servicesByCriteria.left().value();
 301                         for (GraphVertex gv : services) {
 302                                 ComponentParametersView filter = new ComponentParametersView(true);
 303                                 filter.setIgnoreComponentInstances(false);
 304                                 filter.setIgnoreArtifacts(false);
 305                                 filter.setIgnoreGroups(false);
 306
 307                                 Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade
 308                                                 .getToscaElement(gv.getUniqueId(), filter);
 309                                 if (toscaElement.isRight()) {
 310                                         log.info("Failed to fetch service {} {}", gv.getUniqueId(), toscaElement.right().value());
 311                                         continue;
 312                                 }
 313                                 Service service = toscaElement.left().value();
 314
 315                                 String serviceName = (String) gv.getMetadataProperty(GraphPropertyEnum.NAME);
 316
 317                                 boolean isProblematic = isProblematicService(service, serviceName);
 318                                 if (isProblematic) {
 319                                         serviceList.add(service);
 320                                         writeModuleResultToFile(writer, service, null);
 321                                         writer.flush();
 322
 323                                 }
 324
 325                                 janusGraphDao.commit();
 326                         }
 327                         log.info("output file with list of services : {}", fileName);
 328                 } catch (Exception e) {
 329                         log.info("Failed to fetch services ", e);
 330                         return false;
 331                 } finally {
 332                         janusGraphDao.commit();
 333                 }
 334                 return true;
 335         }
 336
 337         private boolean isProblematicService( Service service, String serviceName) {
 338
 339                 List<ComponentInstance> componentInstances = service.getComponentInstances();
 340
 341                 if (componentInstances == null) {
 342                         log.info("No instances for service {} ", service.getUniqueId());
 343                         return false;
 344                 }
 345                 boolean isCheckVFModules = true;
 346                 if(service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN ||
 347                                 service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT){
 348                         isCheckVFModules = false;
 349                 }
 350                 for (ComponentInstance ci : componentInstances) {
 351                         Map<String, ArtifactDefinition> deploymentArtifacts = ci.getDeploymentArtifacts();
 352                         List<GroupInstance> groupInstances = ci.getGroupInstances();
 353                         if (groupInstances == null || groupInstances.isEmpty()) {
 354                                 log.info("No instance groups for instance {} in service {} id {} ", ci.getName(), serviceName,
 355                                                 service.getUniqueId());
 356                                 continue;
 357                         }
 358                         List<VfModuleArtifactPayloadEx> vfModules = null;
 359                         if(isCheckVFModules){
 360                                 Optional<ArtifactDefinition> optionalVfModuleArtifact = deploymentArtifacts.values().stream()
 361                                                 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.getType())).findAny();
 362
 363                                 if(!optionalVfModuleArtifact.isPresent())
 364                                         return true;
 365
 366                                 ArtifactDefinition vfModuleArtifact = optionalVfModuleArtifact.get();
 367                                 Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> vfModulesEither = parseVFModuleJson(vfModuleArtifact);
 368                                 if(vfModulesEither.isRight()){
 369                                         log.error("Failed to parse vfModule for service {} status is {}", service.getUniqueId(), vfModulesEither.right().value());
 370                                         return true;
 371                                 }
 372                                 vfModules = vfModulesEither.left().value();
 373                                 if(vfModules == null || vfModules.isEmpty() ){
 374                                         log.info("vfModules empty for service {}", service.getUniqueId());
 375                                         return true;
 376                                 }
 377                         }
 378
 379                         for (GroupInstance gi : groupInstances) {
 380                                 if (gi.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 381                                         VfModuleArtifactPayloadEx vfModule= null;
 382                                         if(isCheckVFModules && vfModules != null){
 383                                                 Optional<VfModuleArtifactPayloadEx> op = vfModules.stream().filter(vf -> vf.getVfModuleModelName().equals(gi.getGroupName())).findAny();
 384                                                 if(!op.isPresent()){
 385                                                         log.error("Failed to find vfModule for group {}", gi.getGroupName());
 386                                                         return true;
 387                                                 }
 388                                                 vfModule = op.get();
 389                                         }
 390                                         if (isProblematicGroupInstance(gi, ci.getName(), serviceName, deploymentArtifacts, vfModule)) {
 391                                                 return true;
 392                                         }
 393                                 }
 394                         }
 395
 396                 }
 397                 return false;
 398         }
 399
 400         private boolean isProblematicGroup(GroupDefinition gr, String resourceName,
 401                                                                            Map<String, ArtifactDefinition> deploymentArtifacts) {
 402                 List<String> artifacts = gr.getArtifacts();
 403                 List<String> artifactsUuid = gr.getArtifactsUuid();
 404                 Set<String> artifactsSet = new HashSet<>();
 405
 406                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 407                         log.info("No groups in resource {} ", resourceName);
 408                         return true;
 409                 }
 410                 artifactsSet.addAll(artifacts);
 411                 if (artifactsSet.size() < artifacts.size()) {
 412                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", gr.getName(), resourceName);
 413                         return true;
 414                 }
 415
 416                 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
 417                         log.info(" artifacts.size() < artifactsUuid.size() group {} in resource {} ", gr.getName(), resourceName);
 418                         return true;
 419                 }
 420                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 421                         log.info(
 422                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() group {} in resource {} ",
 423                                         gr.getName(), resourceName);
 424                         return true;
 425                 }
 426                 if (artifactsUuid != null && artifactsUuid.contains(null)) {
 427                         log.info(" artifactsUuid.contains(null) group {} in resource {} ", gr.getName(), resourceName);
 428                         return true;
 429                 }
 430
 431                 for (String artifactId : artifacts) {
 432                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 433                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 434                         if (artifactDefinition == null) {
 435                                 log.info(" artifactDefinition == null label {} group {} in resource {} ", artifactlabel, gr.getName(),
 436                                                 resourceName);
 437                                 return true;
 438                         }
 439                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.parse(artifactDefinition.getArtifactType());
 440                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 441                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 442                                         log.info(
 443                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} group {} in resource {} ",
 444                                                         artifactlabel, artifactId, gr.getName(), resourceName);
 445                                         return true;
 446                                 }
 447                                 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 448                                         log.info(
 449                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} group {} in resource {} ",
 450                                                         artifactlabel, gr.getName(), resourceName);
 451                                         return true;
 452                                 }
 453                         }
 454                 }
 455                 for (String artifactUUID : artifactsUuid) {
 456                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 457                         if (label != null && !label.isEmpty() && !label.equals("")) {
 458                                 return true;
 459                         }
 460                 }
 461
 462                 return false;
 463         }
 464
 465         private boolean isProblematicGroupInstance(GroupInstance gi, String instName, String servicename,
 466                                                                                            Map<String, ArtifactDefinition> deploymentArtifacts, VfModuleArtifactPayloadEx vfModule) {
 467                 List<String> artifacts = gi.getArtifacts();
 468                 List<String> artifactsUuid = gi.getArtifactsUuid();
 469                 List<String> instArtifactsUuid = gi.getGroupInstanceArtifactsUuid();
 470                 List<String> instArtifactsId = gi.getGroupInstanceArtifacts();
 471                 Set<String> instArtifatIdSet = new HashSet<>();
 472                 Set<String> artifactsSet = new HashSet<>();
 473
 474                 log.info("check group {} for instance {} ", gi.getGroupName(), instName);
 475                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 476                         log.info("No instance groups for instance {} in service {} ", instName, servicename);
 477                         return true;
 478                 }
 479                 artifactsSet.addAll(artifacts);
 480                 if (artifactsSet.size() < artifacts.size()) {
 481                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", instName, servicename);
 482                         return true;
 483                 }
 484
 485                 if (instArtifactsId != null && !instArtifactsId.isEmpty()) {
 486                         instArtifatIdSet.addAll(instArtifactsId);
 487                 }
 488
 489                 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
 490                         log.info(" artifacts.size() < artifactsUuid.size() inst {} in service {} ", instName, servicename);
 491                         return true;
 492                 }
 493                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 494                         log.info(
 495                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() inst {} in service {} ",
 496                                         instName, servicename);
 497                         return true;
 498                 }
 499                 if (artifactsUuid != null && artifactsUuid.contains(null)) {
 500                         log.info(" artifactsUuid.contains(null) inst {} in service {} ", instName, servicename);
 501                         return true;
 502                 }
 503                 if (instArtifactsId != null && instArtifatIdSet.size() < instArtifactsId.size()) {
 504                         log.info(" instArtifatIdSet.size() < instArtifactsId.size() inst {} in service {} ", instName, servicename);
 505                         return true;
 506                 }
 507
 508                 if ((instArtifactsId != null && instArtifactsUuid != null)
 509                                 && instArtifactsId.size() != instArtifactsUuid.size()) {
 510                         log.info(" instArtifactsId.size() != instArtifactsUuid.size() inst {} in service {} ", instName,
 511                                         servicename);
 512                         return true;
 513                 }
 514
 515                 for (String artifactId : artifacts) {
 516                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 517                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 518                         if (artifactDefinition == null) {
 519                                 log.info(" artifactDefinition == null label {} inst {} in service {} ", artifactlabel, instName,
 520                                                 servicename);
 521                                 return true;
 522                         }
 523                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.parse(artifactDefinition.getArtifactType());
 524                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 525                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 526                                         log.info(
 527                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} inst {} in service {} ",
 528                                                         artifactlabel, artifactId, instName, servicename);
 529                                         return true;
 530                                 }
 531                                 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 532                                         log.info(
 533                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 534                                                         artifactlabel, instName, servicename);
 535                                         return true;
 536                                 }
 537                         } else {
 538                                 if (instArtifactsUuid == null || instArtifactsUuid.isEmpty()) {
 539                                         log.info(" instArtifactsUuid empty. label {} inst {} in service {} ", artifactlabel, instName,
 540                                                         servicename);
 541                                         return true;
 542                                 }
 543                                 if (!instArtifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 544                                         log.info(
 545                                                         " instArtifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 546                                                         artifactlabel, instName, servicename);
 547                                         return true;
 548                                 }
 549                         }
 550                 }
 551                 for (String artifactUUID : artifactsUuid) {
 552                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 553                         if (label != null && !label.isEmpty() && !"".equals(label)) {
 554                                 return true;
 555                         }
 556                 }
 557                 if(vfModule != null && artifactsUuid != null){
 558                         return isProblematicVFModule(vfModule, artifactsUuid, instArtifactsUuid);
 559                 }
 560
 561                 return false;
 562         }
 563
 564         private boolean isProblematicVFModule(VfModuleArtifactPayloadEx vfModule, List<String> artifactsUuid,
 565                                                                                   List<String> instArtifactsUuid) {
 566                 log.info(" isProblematicVFModule  {}  ", vfModule.getVfModuleModelName());
 567                 List<String> vfModuleArtifacts = vfModule.getArtifacts();
 568                 List<String> allArtifacts = new ArrayList<>();
 569                 allArtifacts.addAll(artifactsUuid);
 570                 if(instArtifactsUuid != null)
 571                         allArtifacts.addAll(instArtifactsUuid);
 572                 if((vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()){
 573                         log.error(" vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()");
 574                         return true;
 575                 }
 576                 if(vfModuleArtifacts!= null){
 577                         if( vfModuleArtifacts.size() != allArtifacts.size()){
 578                                 log.error(" vfModuleArtifacts.size() != allArtifacts.size()");
 579                                 return true;
 580                         }
 581                         for(String vfModuleArtifact: vfModuleArtifacts){
 582                                 Optional<String> op = allArtifacts.stream().filter(a -> a.equals(vfModuleArtifact)).findAny();
 583                                 if(!op.isPresent()){
 584                                         log.error("failed to find artifact {} in group artifacts {}", vfModuleArtifact,  allArtifacts);
 585                                         return true;
 586                                 }
 587                         }
 588                 }
 589                 return false;
 590         }
 591
 592
 593
 594         private boolean fix(List<Resource> vfLst, List<Service> serviceList, Map<String, List<Component>> nodesToFixTosca,
 595                                                 Map<String, List<Component>> vfToFixTosca, Map<String, List<Component>> servicesToFixTosca) {
 596                 boolean res = true;
 597                 log.info(" Fix started ***** ");
 598                 if (vfLst != null && !vfLst.isEmpty()) {
 599                         res = fixVf(vfLst);
 600
 601                 }
 602
 603                 if (res && serviceList != null && !serviceList.isEmpty()) {
 604                         res = fixServices(serviceList);
 605
 606                 }
 607
 608                 Set<String> fixedIds = new HashSet<>();
 609
 610                 long time = System.currentTimeMillis();
 611                 String fileName = "FailedGenerateTosca" + "_" + time + ".csv";
 612
 613                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 614                         writer.write("componentType, name, version, UID, UUID, invariantUUID, state\n");
 615                         List<Component> failedList = new ArrayList<>();
 616
 617                         if (res && nodesToFixTosca != null && !nodesToFixTosca.isEmpty()) {
 618
 619                                 generateAndSaveToscaArtifacts(nodesToFixTosca, fixedIds, null, failedList);
 620
 621                         }
 622                         if (vfToFixTosca != null && !vfToFixTosca.isEmpty()) {
 623
 624                                 generateAndSaveToscaArtifacts(vfToFixTosca, fixedIds, vfLst, failedList);
 625
 626                         }
 627
 628                         for (Component component : vfLst) {
 629                                 res = generateToscaPerComponent(fixedIds, component);
 630                                 if (res) {
 631                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 632                                         Map<String, GroupDataDefinition> groups = topologyTemplate.getGroups();
 633                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.GROUPS, EdgeLabelEnum.GROUPS, groups);
 634                                         if (res) {
 635                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getDeploymentArtifacts();
 636                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.DEPLOYMENT_ARTIFACTS,
 637                                                                 EdgeLabelEnum.DEPLOYMENT_ARTIFACTS, arifacts);
 638                                         }
 639                                         if (res) {
 640                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 641                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 642                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 643                                         }
 644                                         janusGraphDao.commit();
 645                                 } else {
 646                                         failedList.add(component);
 647                                 }
 648                         }
 649
 650                         if (servicesToFixTosca != null && !servicesToFixTosca.isEmpty()) {
 651                                 generateAndSaveToscaArtifacts(servicesToFixTosca, fixedIds, serviceList, failedList);
 652
 653                         }
 654
 655
 656                         for (Component component : serviceList) {
 657                                 res = generateToscaPerComponent(fixedIds, component);
 658                                 if (res) {
 659                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 660                                         Map<String, MapGroupsDataDefinition> groups = topologyTemplate.getInstGroups();
 661                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_GROUPS, EdgeLabelEnum.INST_GROUPS,
 662                                                         groups);
 663
 664                                         if (res) {
 665                                                 Map<String, MapArtifactDataDefinition> artifacts = topologyTemplate
 666                                                                 .getInstDeploymentArtifacts();
 667                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_DEPLOYMENT_ARTIFACTS,
 668                                                                 EdgeLabelEnum.INST_DEPLOYMENT_ARTIFACTS, artifacts);
 669                                         }
 670                                         if (res) {
 671                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 672                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 673                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 674                                         }
 675                                         janusGraphDao.commit();
 676                                 } else {
 677                                         failedList.add(component);
 678                                 }
 679
 680                         }
 681                         if (!failedList.isEmpty()) {
 682                                 for (Component component : failedList) {
 683                                         StringBuilder sb = new StringBuilder(component.getComponentType().getValue());
 684                                         sb.append(",").append(component.getName()).append(",").append(component.getVersion()).append(",")
 685                                                         .append(component.getUniqueId()).append(",").append(component.getUUID()).append(",")
 686                                                         .append(component.getInvariantUUID()).append(",").append(component.getLifecycleState());
 687
 688                                         sb.append("\n");
 689                                         writer.write(sb.toString());
 690                                 }
 691                                 writer.flush();
 692                         }
 693                 } catch (IOException e) {
 694                     log.error(e.getMessage());
 695                 } finally {
 696                         janusGraphDao.commit();
 697                 }
 698                 log.info(" Fix finished with res {} ***** ", res);
 699                 return res;
 700         }
 701
 702         private boolean generateAndSaveToscaArtifacts(Map<String, List<Component>> nodesToFixTosca, Set<String> fixedIds,
 703                                                                                                   List<? extends Component> componentsWithFailedGroups, List<Component> failedList) {
 704                 boolean res = true;
 705                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts started ");
 706                 for (Map.Entry<String, List<Component>> entry : nodesToFixTosca.entrySet()) {
 707
 708                         List<Component> component = entry.getValue();
 709                         for (Component c : component) {
 710                                 log.debug("Migration1707ArtifactUuidFix  fix tosca on component : id {},  name {} ", c.getUniqueId(),
 711                                                 c.getName());
 712                                 if (componentsWithFailedGroups != null) {
 713                                         Optional<Component> op = (Optional<Component>) componentsWithFailedGroups.stream()
 714                                                         .filter(cg -> cg.getUniqueId().equals(c.getUniqueId())).findAny();
 715                                         if (!op.isPresent())
 716                                                 res = generateToscaPerComponent(fixedIds, c);
 717                                 } else
 718                                         res = generateToscaPerComponent(fixedIds, c);
 719                                 if (res) {
 720                                         ToscaElement topologyTemplate = ModelConverter.convertToToscaElement(c);
 721                                         Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 722                                         res = fixDataOnGraph(c.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS, EdgeLabelEnum.TOSCA_ARTIFACTS,
 723                                                         arifacts);
 724                                         janusGraphDao.commit();
 725                                 } else {
 726                                         failedList.add(c);
 727                                 }
 728
 729                         }
 730                 }
 731                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts finished with res {} ", res);
 732                 return res;
 733         }
 734
 735         private boolean generateToscaPerComponent(Set<String> fixedIds, Component c) {
 736                 boolean res = true;
 737                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent started component name {} id {}",
 738                                 c.getName(), c.getUniqueId());
 739                 try {
 740                         Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
 741                                         .getToscaFullElement(c.getUniqueId());
 742                         if (toscaElement.isRight()) {
 743                                 log.info("Failed to fetch resources {} {}", c.getUniqueId(), toscaElement.right().value());
 744                                 return false;
 745                         }
 746                         Component toscaElementFull = toscaElement.left().value();
 747                         toscaElementFull.setGroups(c.getGroups());
 748                         List<ComponentInstance> ciListFull = toscaElementFull.getComponentInstances();
 749                         List<ComponentInstance> ciList = c.getComponentInstances();
 750                         if (ciListFull != null && !ciListFull.isEmpty()) {
 751                                 ciListFull.forEach(ciFull -> {
 752                                         ComponentInstance compInst = ciList.stream()
 753                                                         .filter(ci -> ci.getUniqueId().equals(ciFull.getUniqueId())).findAny().get();
 754                                         ciFull.setGroupInstances(compInst.getGroupInstances());
 755                                 });
 756                         }
 757
 758                         Either<Component, ToscaError> either = generateToscaArtifact(toscaElementFull);
 759
 760                         if (either.isRight()) {
 761                                 log.error("Couldn't generate and save tosca template component  unique id {}, name {} error: {}",
 762                                                 toscaElementFull.getUniqueId(), toscaElementFull.getName(), either.right().value());
 763                                 res = false;
 764
 765                         }
 766
 767                         if (res) {
 768                                 c.setToscaArtifacts(either.left().value().getToscaArtifacts());
 769                                 fixedIds.add(toscaElementFull.getUniqueId());
 770                         }
 771                 } finally {
 772                         if (res)
 773                                 janusGraphDao.commit();
 774                         else
 775                                 janusGraphDao.rollback();
 776                 }
 777                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent finished  component name {} id {} res {}",
 778                                 c.getName(), c.getUniqueId(), res);
 779                 return res;
 780         }
 781
 782         private <T extends ToscaDataDefinition> boolean fixDataOnGraph(String componentId, VertexTypeEnum vertexTypeEnum,
 783                                                                                                                                    EdgeLabelEnum edgeLabelEnum, Map<String, T> groups) {
 784                 log.debug("amount groups to update: VertexTypeEnum {} EdgeLabelEnum {} data size {}", vertexTypeEnum.getName(),
 785                                 edgeLabelEnum, groups.size());
 786                 boolean res = true;
 787                 Either<GraphVertex, JanusGraphOperationStatus> getResponse = janusGraphDao.getVertexById(componentId,
 788                                 JsonParseFlagEnum.NoParse);
 789                 if (getResponse.isRight()) {
 790                         log.debug("Couldn't fetch component  unique id {}, error: {}", componentId, getResponse.right().value());
 791                         res = false;
 792
 793                 }
 794                 if (res) {
 795                         GraphVertex componentVertex = getResponse.left().value();
 796
 797                         GraphVertex toscaDataVertex = null;
 798                         Either<GraphVertex, JanusGraphOperationStatus> groupVertexEither = janusGraphDao.getChildVertex(componentVertex,
 799                                         edgeLabelEnum, JsonParseFlagEnum.ParseJson);
 800                         if (groupVertexEither.isRight() && groupVertexEither.right().value() == JanusGraphOperationStatus.NOT_FOUND) {
 801                                 log.debug("no child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum, componentId,
 802                                                 groupVertexEither.right().value());
 803                                 return true;
 804                         }
 805                         if (groupVertexEither.isRight()) {
 806                                 res = false;
 807                                 log.debug("failed to get child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum,
 808                                                 componentId, groupVertexEither.right().value());
 809                         }
 810                         if (res) {
 811                                 toscaDataVertex = groupVertexEither.left().value();
 812                                 toscaDataVertex.setJson(groups);
 813                                 Either<GraphVertex, JanusGraphOperationStatus> updatevertexEither = janusGraphDao.updateVertex(toscaDataVertex);
 814                                 if (updatevertexEither.isRight()) {
 815                                         log.debug("failed to update vertex for component  unique id {}, error: {}", componentId,
 816                                                         updatevertexEither.right().value());
 817                                         janusGraphDao.rollback();
 818                                         return false;
 819                                 }
 820                         }
 821                 }
 822                 log.debug("Fix data on graph finished: VertexTypeEnum {} EdgeLabelEnum {} res {}", vertexTypeEnum.getName(),
 823                                 res);
 824                 return res;
 825         }
 826
 827         private boolean fixServices(List<Service> serviceList) {
 828                 for (Service service : serviceList) {
 829                         log.debug("Migration1707ArtifactUuidFix  fix service: id {},  name {} ", service.getUniqueId(),
 830                                         service.getName());
 831                         List<ComponentInstance> instances = service.getComponentInstances();
 832                         for (ComponentInstance instance : instances) {
 833                                 fixComponentInstances(service, instance);
 834                         }
 835
 836                 }
 837                 return true;
 838
 839         }
 840
 841         private void fixComponentInstances(Service service, ComponentInstance instance) {
 842                 Map<String, ArtifactDefinition> artifactsMap = instance.getDeploymentArtifacts();
 843                 List<GroupInstance> groupsList = instance.getGroupInstances();
 844                 if (groupsList != null && artifactsMap != null) {
 845                         List<GroupInstance> groupsToDelete = new ArrayList<>();
 846                         for (GroupInstance group : groupsList) {
 847                                 fixGroupInstances(service, artifactsMap, groupsToDelete, group);
 848
 849                         }
 850
 851                         if (!groupsToDelete.isEmpty()) {
 852                                 log.debug("Migration1707ArtifactUuidFix  delete group:  resource id {}, group instance to delete {} ",
 853                                                 service.getUniqueId(), groupsToDelete);
 854                                 groupsList.removeAll(groupsToDelete);
 855
 856                         }
 857
 858                         Optional<ArtifactDefinition> optionalVfModuleArtifact = artifactsMap.values().stream()
 859                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.getType())).findAny();
 860                         ArtifactDefinition vfModuleArtifact;
 861                         if(!optionalVfModuleArtifact.isPresent()){
 862                                 vfModuleArtifact = createVfModuleArtifact(instance, service);
 863                                 artifactsMap.put(vfModuleArtifact.getArtifactLabel(), vfModuleArtifact);
 864                         }
 865                         else {
 866                                 vfModuleArtifact = optionalVfModuleArtifact.get();
 867                         }
 868                         fillVfModuleInstHeatEnvPayload(service, instance, groupsList, vfModuleArtifact);
 869                 }
 870         }
 871
 872         private void fixGroupInstances(Service service, Map<String, ArtifactDefinition> artifactsMap,
 873                                                                    List<GroupInstance> groupsToDelete, GroupInstance group) {
 874                 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 875                         log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", service.getUniqueId(),
 876                                         group.getName());
 877                         if(group.getArtifacts() != null){
 878                                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 879                                 if(group.getGroupInstanceArtifacts() != null){
 880                                         List<String> groupInsArtifacts = new ArrayList<>(group.getGroupInstanceArtifacts());
 881                                         groupArtifactsSet.addAll(groupInsArtifacts);
 882                                 }
 883                                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 884
 885                                 clearGroupInstanceArtifacts(group);
 886
 887                                 for (String artifactId : groupArtifacts) {
 888                                         fixArtifactUndergroupInstances(artifactsMap, group, groupArtifacts, artifactId);
 889                                 }
 890                         }
 891                         if (group.getArtifacts() == null || group.getArtifacts().isEmpty()) {
 892                                 log.debug(
 893                                                 "Migration1707ArtifactUuidFix  fix groupInstance add to delete list:  resource id {} name {} , group name {} ",
 894                                                 service.getUniqueId(), service.getName(), group.getName());
 895                                 groupsToDelete.add(group);
 896                         }
 897                 }
 898         }
 899
 900         private void clearGroupInstanceArtifacts(GroupInstance group) {
 901                 if(group.getArtifacts() != null)
 902                         group.getArtifacts().clear();
 903                 else
 904                         group.setArtifacts(new ArrayList<>());
 905                 if(group.getArtifactsUuid() != null)
 906                         group.getArtifactsUuid().clear();
 907                 else{
 908                         group.setArtifactsUuid(new ArrayList<>());
 909                 }
 910                 if(group.getGroupInstanceArtifacts() != null)
 911                         group.getGroupInstanceArtifacts().clear();
 912                 else{
 913                         group.setGroupInstanceArtifacts(new ArrayList<>());
 914                 }
 915                 if(group.getGroupInstanceArtifactsUuid() != null )
 916                         group.getGroupInstanceArtifactsUuid().clear();
 917                 else
 918                         group.setGroupInstanceArtifactsUuid(new ArrayList<>());
 919         }
 920
 921         private void fixArtifactUndergroupInstances(Map<String, ArtifactDefinition> artifactsMap, GroupInstance group,
 922                                                                                                 List<String> groupArtifacts, String artifactId) {
 923                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 924                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 925                                 group.getName(), artifactId, artifactlabel);
 926                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 927                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 928                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.parse(artifact.getArtifactType());
 929                         String correctArtifactId = artifact.getUniqueId();
 930                         String correctArtifactUUID = artifact.getArtifactUUID();
 931                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 932                                 boolean isAddToGroup = true;
 933                                 if (groupArtifacts.size() == 1) {
 934                                         if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 935                                                 isAddToGroup = false;
 936                                                 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 937                                         }
 938                                 }
 939                                 if (isAddToGroup) {
 940                                         log.debug(
 941                                                         MIGRATION1707_ARTIFACT_UUID_FIX,
 942                                                         group.getName(), correctArtifactId, correctArtifactUUID);
 943                                         group.getArtifacts().add(correctArtifactId);
 944                                         if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 945                                                 group.getArtifactsUuid().add(correctArtifactUUID);
 946                                         }
 947                                 }
 948                         } else {
 949                                 log.debug(
 950                                                 MIGRATION1707_ARTIFACT_UUID_FIX,
 951                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 952                                 Set<String> tmpSet = new HashSet<>(group.getGroupInstanceArtifacts());
 953                                 tmpSet.add(correctArtifactId);
 954                                 group.setGroupInstanceArtifacts(new ArrayList<>(tmpSet));
 955                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 956                                         Set<String> tmpSetUUID = new HashSet<>(group.getGroupInstanceArtifactsUuid());
 957                                         tmpSetUUID.add(correctArtifactUUID);
 958                                         group.setGroupInstanceArtifactsUuid(new ArrayList<>(tmpSetUUID));
 959                                 }
 960                         }
 961                 }
 962         }
 963
 964         private boolean fixVf(List<Resource> vfLst) {
 965                 for (Resource resource : vfLst) {
 966                         log.debug("Migration1707ArtifactUuidFix  fix resource: id {},  name {} ", resource.getUniqueId(),
 967                                         resource.getName());
 968                         Map<String, ArtifactDefinition> artifactsMap = resource.getDeploymentArtifacts();
 969                         List<GroupDefinition> groupsList = resource.getGroups();
 970                         List<GroupDefinition> groupsToDelete = new ArrayList<>();
 971                         if (groupsList != null && artifactsMap != null) {
 972                                 for (GroupDefinition group : groupsList) {
 973                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE) && group.getArtifacts() != null) {
 974                                                 fixVfGroup(resource, artifactsMap, group);
 975                                         }
 976                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)
 977                                                         && (group.getArtifacts() == null || group.getArtifacts().isEmpty())) {
 978                                                 log.debug(
 979                                                                 "Migration1707ArtifactUuidFix  add group to delete list fix resource: id {},  name {} ",
 980                                                                 resource.getUniqueId(), resource.getName(), group.getName());
 981                                                 groupsToDelete.add(group);
 982                                         }
 983                                 }
 984
 985                                 if (!groupsToDelete.isEmpty()) {
 986                                         groupsList.removeAll(groupsToDelete);
 987
 988                                 }
 989                         }
 990
 991                 }
 992
 993                 return true;
 994         }
 995
 996         private void fixVfGroup(Resource resource, Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group) {
 997                 log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", resource.getUniqueId(),
 998                                 group.getName());
 999                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
1000                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
1001                 group.getArtifacts().clear();
1002                 group.getArtifactsUuid().clear();
1003
1004                 for (String artifactId : groupArtifacts) {
1005                         fixArtifactUnderGroup(artifactsMap, group, groupArtifacts, artifactId);
1006                 }
1007         }
1008
1009         private void fixArtifactUnderGroup(Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group,
1010                                                                            List<String> groupArtifacts, String artifactId) {
1011
1012                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
1013                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
1014                                 group.getName(), artifactId, artifactlabel);
1015                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
1016                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
1017                         String correctArtifactId = artifact.getUniqueId();
1018                         String correctArtifactUUID = artifact.getArtifactUUID();
1019                         boolean isAddToGroup = true;
1020                         if (groupArtifacts.size() == 1) {
1021                                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.parse(artifact.getArtifactType());
1022                                 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
1023                                         isAddToGroup = false;
1024                                         artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
1025                                 }
1026                         }
1027                         if (isAddToGroup) {
1028                                 log.debug(
1029                                                 MIGRATION1707_ARTIFACT_UUID_FIX,
1030                                                 group.getName(), correctArtifactId, correctArtifactUUID);
1031                                 group.getArtifacts().add(correctArtifactId);
1032                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
1033                                         group.getArtifactsUuid().add(correctArtifactUUID);
1034                                 }
1035                         }
1036
1037                 }
1038         }
1039
1040         private String findArtifactLabelFromArtifactId(String artifactId) {
1041                 String artifactLabel = "";
1042
1043                 int index = artifactId.lastIndexOf('.');
1044                 if (index > 0 && index + 1 < artifactId.length())
1045                         artifactLabel = artifactId.substring(index + 1);
1046                 return artifactLabel;
1047         }
1048
1049         private void writeModuleResultToFile(Writer writer, org.openecomp.sdc.be.model.Component component,
1050                                                                                  Service service) {
1051                 try {
1052                         // "service name, service id, state, version
1053                         StringBuilder sb = new StringBuilder(component.getName());
1054                         sb.append(",").append(component.getUniqueId()).append(",").append(component.getLifecycleState()).append(",")
1055                                         .append(component.getVersion());
1056                         if (service != null) {
1057                                 sb.append(",").append(service.getName());
1058                         }
1059                         sb.append("\n");
1060                         writer.write(sb.toString());
1061                 } catch (IOException e) {
1062                         log.error(e.getMessage());
1063                 }
1064         }
1065
1066         private void writeModuleResultToFile(Writer writer, List<Component> components) {
1067                 try {
1068                         // "service name, service id, state, version
1069                         for (Component component : components) {
1070                                 StringBuilder sb = new StringBuilder(component.getName());
1071                                 sb.append(",").append(component.getUniqueId()).append(",").append(component.getInvariantUUID())
1072                                                 .append(",").append(component.getLifecycleState()).append(",").append(component.getVersion());
1073
1074                                 sb.append("\n");
1075                                 writer.write(sb.toString());
1076                         }
1077                 } catch (IOException e) {
1078
1079                     log.error(e.getMessage());
1080                 }
1081         }
1082
1083         public boolean doFixTosca(Map<String, List<Component>> nodeToFix, Map<String, List<Component>> vfToFix,
1084                                                           Map<String, List<Component>> serviceToFix) {
1085
1086                 Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
1087                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1088                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1089
1090                 Map<String, List<Component>> vertices = getVerticesToValidate(VertexTypeEnum.NODE_TYPE, hasProps);
1091                 validateTosca(vertices, nodeToFix, "RESOURCE_TOSCA_ARTIFACTS");//
1092
1093                 hasProps.clear();
1094                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1095                 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF);
1096                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1097
1098                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1099                 validateTosca(vertices, vfToFix, "VF_TOSCA_ARTIFACTS");
1100
1101                 hasProps.clear();
1102                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
1103                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1104
1105                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1106                 return validateTosca(vertices, serviceToFix, "SERVICE_TOSCA_ARTIFACTS");
1107         }
1108
1109         public Map<String, List<Component>> getVerticesToValidate(VertexTypeEnum type,
1110                                                                                                                           Map<GraphPropertyEnum, Object> hasProps) {
1111
1112                 Map<String, List<Component>> result = new HashMap<>();
1113                 try {
1114
1115                         Either<List<GraphVertex>, JanusGraphOperationStatus> resultsEither = janusGraphDao
1116           .getByCriteria(type, hasProps);
1117                         if (resultsEither.isRight()) {
1118                                 log.error("getVerticesToValidate failed {} ",resultsEither.right().value());
1119                                 return result;
1120                         }
1121                         log.info("getVerticesToValidate: {}  vertices to scan", resultsEither.left().value().size());
1122                         List<GraphVertex> componentsList = resultsEither.left().value();
1123                         componentsList.forEach(vertex -> {
1124                                 String ivariantUuid = (String) vertex.getMetadataProperty(GraphPropertyEnum.INVARIANT_UUID);
1125                                 if (!result.containsKey(ivariantUuid)) {
1126                                         List<Component> compList = new ArrayList<>();
1127                                         result.put(ivariantUuid, compList);
1128                                 }
1129                                 List<Component> compList = result.get(ivariantUuid);
1130
1131                                 ComponentParametersView filter = new ComponentParametersView(true);
1132                                 filter.setIgnoreArtifacts(false);
1133
1134                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1135                                                 .getToscaElement(vertex.getUniqueId(), filter);
1136                                 if (toscaElement.isRight()) {
1137                                         log.error("getVerticesToValidate: failed to find element {}  staus is {}", vertex.getUniqueId()
1138                                                         ,toscaElement.right().value());
1139                                 } else {
1140                                         compList.add(toscaElement.left().value());
1141                                 }
1142                                 janusGraphDao.commit();
1143
1144                         });
1145
1146                 } catch (Exception e) {
1147                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1148
1149                 } finally {
1150                         janusGraphDao.commit();
1151
1152                 }
1153                 return result;
1154         }
1155
1156         public boolean validateTosca(Map<String, List<Component>> vertices, Map<String, List<Component>> compToFix,
1157                                                                  String name) {
1158                 boolean result = true;
1159                 long time = System.currentTimeMillis();
1160                 String fileName = name + "_" + time + ".csv";
1161                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
1162                         writer.write("name, UUID, invariantUUID, state, version\n");
1163                         for (Map.Entry<String, List<Component>> entry : vertices.entrySet()) {
1164                                 List<Component> compList = entry.getValue();
1165                                 Set<String> artifactEsId = new HashSet<>();
1166                                 for (Component component : compList) {
1167                                         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
1168                                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1169                                                         .filter(a -> artifactEsId.contains(a.getEsId()) && a.getEsId() != null).findAny();
1170                                         if (op.isPresent()) {
1171                                                 result = false;
1172                                                 writeModuleResultToFile(writer, compList);
1173                                                 writer.flush();
1174                                                 break;
1175                                         } else {
1176                                                 artifactEsId.addAll(toscaArtifacts.values().stream().map(ArtifactDefinition::getEsId)
1177                                                                 .collect(Collectors.toList()));
1178                                         }
1179                                 }
1180                                 if (!result) {
1181                                         List<Component> compListfull = new ArrayList<>();
1182                                         for (Component c : compList) {
1183                                                 ComponentParametersView filter = new ComponentParametersView(true);
1184                                                 filter.setIgnoreComponentInstances(false);
1185                                                 filter.setIgnoreArtifacts(false);
1186                                                 filter.setIgnoreGroups(false);
1187
1188                                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1189                                                                 .getToscaElement(c.getUniqueId(), filter);
1190                                                 if (toscaElement.isRight()) {
1191                                                         log.debug("getVerticesToValidate: failed to find element" + c.getUniqueId()
1192                                                                         + " status is" + toscaElement.right().value());
1193                                                 } else {
1194                                                         compListfull.add(toscaElement.left().value());
1195                                                 }
1196                                                 this.janusGraphDao.commit();
1197                                         }
1198
1199                                         compToFix.put(entry.getKey(), compListfull);
1200                                         result = true;
1201                                 }
1202
1203                         }
1204
1205                 } catch (Exception e) {
1206                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1207                         return false;
1208                 } finally {
1209                         janusGraphDao.commit();
1210                 }
1211                 return result;
1212         }
1213
1214         private Either<Component, ToscaError> generateToscaArtifact(Component parent) {
1215                 log.debug("tosca artifact generation");
1216                 try {
1217                         Map<String, ArtifactDefinition> toscaArtifacts = parent.getToscaArtifacts();
1218
1219                         ArtifactDefinition toscaArtifact = null;
1220                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1221                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_TEMPLATE.getType())).findAny();
1222
1223                         if (op.isPresent()) {
1224                                 toscaArtifact = op.get();
1225                         }
1226                         if (toscaArtifact != null) {
1227                                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent artifact name {} id {} esId {}",
1228                                                 toscaArtifact.getArtifactName(), toscaArtifact.getUniqueId(), toscaArtifact.getEsId());
1229
1230                                 Either<ToscaRepresentation, ToscaError> exportComponent = toscaExportUtils.exportComponent(parent);
1231                                 if (exportComponent.isRight()) {
1232                                         log.debug("Failed export tosca yaml for component {} error {}", parent.getUniqueId(),
1233                                                         exportComponent.right().value());
1234
1235                                         return Either.right(exportComponent.right().value());
1236                                 }
1237                                 log.debug("Tosca yaml exported for component {} ", parent.getUniqueId());
1238
1239                                 toscaArtifact.setPayload(exportComponent.left().value().getMainYaml());
1240                                 byte[] decodedPayload = toscaArtifact.getPayloadData();
1241
1242                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1243                                                 toscaArtifact.getArtifactLabel());
1244                                 toscaArtifact.setUniqueId(uniqueId);
1245                                 toscaArtifact.setEsId(toscaArtifact.getUniqueId());
1246
1247                                 toscaArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1248                                 DAOArtifactData artifactData = new DAOArtifactData(toscaArtifact.getEsId(), decodedPayload);
1249                                 artifactCassandraDao.saveArtifact(artifactData);
1250
1251                                 log.debug("Tosca yaml artifact esId  {} ", toscaArtifact.getEsId());
1252                         }
1253                         ArtifactDefinition csarArtifact = null;
1254                         op = toscaArtifacts.values().stream()
1255                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_CSAR.getType())).findAny();
1256
1257                         if (op.isPresent()) {
1258                                 csarArtifact = op.get();
1259                         }
1260
1261                         if (csarArtifact != null) {
1262                                 Either<byte[], ResponseFormat> generated = csarUtils.createCsar(parent, true, true);
1263
1264                                 if (generated.isRight()) {
1265                                         log.debug("Failed to export tosca csar for component {} error {}", parent.getUniqueId(),
1266                                                         generated.right().value());
1267
1268                                         return Either.right(ToscaError.GENERAL_ERROR);
1269                                 }
1270                                 byte[] value = generated.left().value();
1271                                 csarArtifact.setPayload(value);
1272                                 byte[] decodedPayload = csarArtifact.getPayloadData();
1273
1274                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1275                                                 csarArtifact.getArtifactLabel());
1276                                 csarArtifact.setUniqueId(uniqueId);
1277                                 csarArtifact.setEsId(csarArtifact.getUniqueId());
1278
1279                                 csarArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1280                                 DAOArtifactData artifactData = new DAOArtifactData(csarArtifact.getEsId(), decodedPayload);
1281                                 artifactCassandraDao.saveArtifact(artifactData);
1282                                 log.debug("Tosca csar artifact esId  {} ", csarArtifact.getEsId());
1283
1284                         }
1285
1286                 } catch (Exception ex) {
1287                         log.error("Failed to generate tosca atifact component id {} component name {} error {}",
1288                                         parent.getUniqueId(), parent.getName(), ex.getMessage());
1289
1290                         return Either.right(ToscaError.GENERAL_ERROR);
1291                 }
1292
1293                 return Either.left(parent);
1294         }
1295
1296         private ArtifactDefinition createVfModuleArtifact(ComponentInstance currVF, Service service) {
1297
1298                 ArtifactDefinition vfModuleArtifactDefinition = new ArtifactDefinition();
1299
1300                 vfModuleArtifactDefinition.setDescription("Auto-generated VF Modules information artifact");
1301                 vfModuleArtifactDefinition.setArtifactDisplayName("Vf Modules Metadata");
1302                 vfModuleArtifactDefinition.setArtifactType(ArtifactTypeEnum.VF_MODULES_METADATA.getType());
1303                 vfModuleArtifactDefinition.setArtifactGroupType(ArtifactGroupTypeEnum.DEPLOYMENT);
1304                 vfModuleArtifactDefinition.setArtifactLabel("vfModulesMetadata");
1305                 vfModuleArtifactDefinition.setTimeout(0);
1306                 vfModuleArtifactDefinition.setArtifactName(currVF.getNormalizedName() + "_modules.json");
1307
1308                 return vfModuleArtifactDefinition;
1309         }
1310
1311
1312         private void fillVfModuleInstHeatEnvPayload(Component parent, ComponentInstance instance, List<GroupInstance> groupsForCurrVF,
1313                                                                                                 ArtifactDefinition vfModuleArtifact) {
1314                 log.debug("generate new vf module for component. name  {}, id {}, Version {}", instance.getName(), instance.getUniqueId());
1315
1316                 String uniqueId = UniqueIdBuilder.buildInstanceArtifactUniqueId(parent.getUniqueId(), instance.getUniqueId(), vfModuleArtifact.getArtifactLabel());
1317
1318                 vfModuleArtifact.setUniqueId(uniqueId);
1319                 vfModuleArtifact.setEsId(vfModuleArtifact.getUniqueId());
1320
1321                 List<VfModuleArtifactPayload> vfModulePayloadForCurrVF = new ArrayList<>();
1322                 if (groupsForCurrVF != null) {
1323                         for (GroupInstance groupInstance : groupsForCurrVF) {
1324                                 VfModuleArtifactPayload modulePayload = new VfModuleArtifactPayload(groupInstance);
1325                                 vfModulePayloadForCurrVF.add(modulePayload);
1326                         }
1327                         Collections.sort(vfModulePayloadForCurrVF,
1328                                         (art1, art2) -> VfModuleArtifactPayload.compareByGroupName(art1, art2));
1329
1330                         final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1331
1332                         String vfModulePayloadString = gson.toJson(vfModulePayloadForCurrVF);
1333                         log.debug("vfModulePayloadString {}", vfModulePayloadString);
1334                         if (vfModulePayloadString != null) {
1335                                 String newCheckSum = GeneralUtility
1336                                                 .calculateMD5Base64EncodedByByteArray(vfModulePayloadString.getBytes());
1337                                 vfModuleArtifact.setArtifactChecksum(newCheckSum);
1338
1339                                 DAOArtifactData artifactData = new DAOArtifactData(vfModuleArtifact.getEsId(),
1340                                                 vfModulePayloadString.getBytes());
1341                                 artifactCassandraDao.saveArtifact(artifactData);
1342
1343                         }
1344
1345                 }
1346
1347         }
1348
1349         private Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> parseVFModuleJson(ArtifactDefinition vfModuleArtifact) {
1350                 log.info("Try to get vfModule json from cassandra {}", vfModuleArtifact.getEsId());
1351                 Either<DAOArtifactData, CassandraOperationStatus> vfModuleData = artifactCassandraDao.getArtifact(vfModuleArtifact.getEsId());
1352
1353                 if (vfModuleData.isRight()) {
1354                         CassandraOperationStatus resourceUploadStatus = vfModuleData.right().value();
1355                         StorageOperationStatus storageResponse = DaoStatusConverter.convertCassandraStatusToStorageStatus(resourceUploadStatus);
1356                         log.error("failed to fetch vfModule json {} from cassandra. Status is {}", vfModuleArtifact.getEsId(), storageResponse);
1357                         return Either.right(storageResponse);
1358
1359                 }
1360
1361                 DAOArtifactData DAOArtifactData = vfModuleData.left().value();
1362                 String gsonData = new String( DAOArtifactData.getDataAsArray());
1363                 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1364                 JsonArray jsonElement = new JsonArray();
1365                 jsonElement = gson.fromJson(gsonData, jsonElement.getClass());
1366                 List<VfModuleArtifactPayloadEx> vfModules = new ArrayList<>();
1367                 jsonElement.forEach(je ->{
1368                         VfModuleArtifactPayloadEx vfModule = ComponentsUtils.parseJsonToObject(je.toString(), VfModuleArtifactPayloadEx.class);
1369                         vfModules.add(vfModule);
1370                 });
1371
1372                 log.debug  ("parse vf module finish {}", gsonData);
1373                 return Either.left(vfModules);
1374
1375         }
1376 }
1377
1378