asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/ArtifactUuidFix.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23 import com.google.gson.Gson;
  24 import com.google.gson.GsonBuilder;
  25 import com.google.gson.JsonArray;
  26 import fj.data.Either;
  27 import org.openecomp.sdc.asdctool.impl.validator.utils.VfModuleArtifactPayloadEx;
  28 import org.openecomp.sdc.be.components.distribution.engine.VfModuleArtifactPayload;
  29 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  30 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  31 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
  32 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
  33 import org.openecomp.sdc.be.dao.jsongraph.JanusGraphDao;
  34 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
  35 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
  36 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
  37 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
  38 import org.openecomp.sdc.be.datatypes.elements.GroupDataDefinition;
  39 import org.openecomp.sdc.be.datatypes.elements.MapArtifactDataDefinition;
  40 import org.openecomp.sdc.be.datatypes.elements.MapGroupsDataDefinition;
  41 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
  42 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
  43 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
  44 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  45 import org.openecomp.sdc.be.datatypes.tosca.ToscaDataDefinition;
  46 import org.openecomp.sdc.be.impl.ComponentsUtils;
  47 import org.openecomp.sdc.be.model.*;
  48 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.TopologyTemplate;
  49 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.ToscaElement;
  50 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
  51 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
  52 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  53 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
  54 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
  55 import org.openecomp.sdc.be.resources.data.ESArtifactData;
  56 import org.openecomp.sdc.be.tosca.CsarUtils;
  57 import org.openecomp.sdc.be.tosca.ToscaError;
  58 import org.openecomp.sdc.be.tosca.ToscaExportHandler;
  59 import org.openecomp.sdc.be.tosca.ToscaRepresentation;
  60 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
  61 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
  62 import org.openecomp.sdc.common.api.Constants;
  63 import org.openecomp.sdc.common.log.wrappers.Logger;
  64 import org.openecomp.sdc.common.util.GeneralUtility;
  65 import org.openecomp.sdc.exception.ResponseFormat;
  66 import org.springframework.beans.factory.annotation.Autowired;
  67
  68 import java.io.BufferedWriter;
  69 import java.io.FileOutputStream;
  70 import java.io.IOException;
  71 import java.io.OutputStreamWriter;
  72 import java.io.Writer;
  73 import java.util.ArrayList;
  74 import java.util.Collections;
  75 import java.util.HashMap;
  76 import java.util.HashSet;
  77 import java.util.List;
  78 import java.util.Map;
  79 import java.util.Optional;
  80 import java.util.Set;
  81 import java.util.EnumMap;
  82 import java.util.stream.Collectors;
  83
  84 @org.springframework.stereotype.Component("artifactUuidFix")
  85 public class ArtifactUuidFix {
  86
  87         private static final String MIGRATION1707_ARTIFACT_UUID_FIX = "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ";
  88
  89     private static final String FAILED_TO_FETCH_VF_RESOURCES = "Failed to fetch vf resources ";
  90
  91     private static final String UTF8 = "utf-8";
  92
  93         @Autowired
  94         private JanusGraphDao janusGraphDao;
  95
  96         @Autowired
  97         private ToscaOperationFacade toscaOperationFacade;
  98         @Autowired
  99         private ToscaExportHandler toscaExportUtils;
 100         @Autowired
 101         private ArtifactCassandraDao artifactCassandraDao;
 102
 103
 104         @Autowired
 105         private CsarUtils csarUtils;
 106
 107         private static Logger log = Logger.getLogger(ArtifactUuidFix.class.getName());
 108
 109         public boolean doFix(String fixComponent, String runMode) {
 110                 List<Resource> vfLst = new ArrayList<>();
 111                 List<Service> serviceList = new ArrayList<>();
 112                 Map<String, List<Component>> nodeToFixTosca = new HashMap<>();
 113                 Map<String, List<Component>> vfToFixTosca = new HashMap<>();
 114                 Map<String, List<Component>> serviceToFixTosca = new HashMap<>();
 115
 116                 long time = System.currentTimeMillis();
 117
 118                 doFixTosca(nodeToFixTosca, vfToFixTosca, serviceToFixTosca);
 119
 120                 if ("vf_only".equals(fixComponent)) {
 121                         if (!fetchFaultVf(fixComponent, vfLst, time)) {
 122                                 return false;
 123                         }
 124                 } else {
 125                         if (!fetchServices(fixComponent, serviceList, time)) {
 126                                 return false;
 127                         }
 128                 }
 129                 if ("service_vf".equals(runMode) || "fix".equals(runMode)) {
 130                         log.info("Mode {}. Find problem VFs", runMode);
 131                         if (!fetchVf(serviceList, vfLst, time)) {
 132                                 log.info("Mode {}. Find problem VFs finished with failure", runMode);
 133                                 return false;
 134                         }
 135                         log.info("Mode {}. Find problem VFs finished with success", runMode);
 136                 }
 137                 if ("fix".equals(runMode) || "fix_only_services".equals(runMode)) {
 138                         log.info("Mode {}. Start fix", runMode);
 139                         if (!fix(vfLst, serviceList, nodeToFixTosca, vfToFixTosca, serviceToFixTosca)) {
 140                                 log.info("Mode {}. Fix finished with failure", runMode);
 141                                 return false;
 142                         }
 143                         log.info("Mode {}. Fix finished with success", runMode);
 144                 }
 145
 146                 return true;
 147         }
 148
 149         private boolean fetchFaultVf(String fixComponent, List<Resource> vfLst, long time) {
 150                 log.info("Find fault VF ");
 151                 String fileName = "fault_" + time + ".csv";
 152                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 153                         writer.write("vf name, vf id, state, version\n");
 154
 155                         Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
 156                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
 157                         hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF.name());
 158
 159                         Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
 160                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 161                         log.info("Try to fetch resources with properties {} and not {}", hasProps, hasNotProps);
 162
 163                         Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
 164                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 165                         if (servicesByCriteria.isRight()) {
 166                                 log.info("Failed to fetch resources {}", servicesByCriteria.right().value());
 167                                 return false;
 168                         }
 169                         List<GraphVertex> resources = servicesByCriteria.left().value();
 170                         for (GraphVertex gv : resources) {
 171                                 ComponentParametersView filter = new ComponentParametersView(true);
 172                                 filter.setIgnoreComponentInstances(false);
 173                                 filter.setIgnoreArtifacts(false);
 174                                 filter.setIgnoreGroups(false);
 175
 176                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 177                                                 .getToscaElement(gv.getUniqueId());
 178                                 if (toscaElement.isRight()) {
 179                                         log.info("Failed to fetch resources {} {}", gv.getUniqueId(), toscaElement.right().value());
 180                                         continue;
 181                                 }
 182
 183                                 Resource resource = toscaElement.left().value();
 184                                 String resourceName = resource.getName();
 185                                 Map<String, ArtifactDefinition> deploymentArtifacts = resource.getDeploymentArtifacts();
 186                                 List<GroupDefinition> groups = resource.getGroups();
 187                                 if (groups == null || groups.isEmpty()) {
 188                                         log.info("No groups for resource {} id {} ", resourceName, gv.getUniqueId());
 189                                         continue;
 190                                 }
 191                                 boolean isProblematic = false;
 192                                 for (GroupDefinition gr : groups) {
 193                                         if (gr.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 194                                                 if (isProblematicGroup(gr, resourceName, deploymentArtifacts)) {
 195                                                         isProblematic = true;
 196                                                         break;
 197                                                 }
 198                                         }
 199                                 }
 200                                 if (isProblematic) {
 201                                         vfLst.add(resource);
 202                                         writeModuleResultToFile(writer, resource, null);
 203                                         writer.flush();
 204                                 }
 205                                 janusGraphDao.commit();
 206                         }
 207
 208                 } catch (Exception e) {
 209                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
 210                         return false;
 211                 } finally {
 212                         janusGraphDao.commit();
 213                 }
 214                 return true;
 215         }
 216
 217         private boolean fetchVf(List<Service> serviceList, List<Resource> vfLst, long time) {
 218                 log.info("Find problem VF ");
 219                 if (serviceList.isEmpty()) {
 220                         log.info("No services as input");
 221                         return true;
 222                 }
 223                 String fileName = "problemVf_" + time + ".csv";
 224                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 225                         writer.write("vf name, vf id, state, version, example service name\n");
 226                         Set<String> vfIds = new HashSet<>();
 227                         for (Service service : serviceList) {
 228                                 List<ComponentInstance> componentInstances = service.getComponentInstances().stream()
 229                                                 .filter(ci -> ci.getOriginType().equals(OriginTypeEnum.VF)).collect(Collectors.toList());
 230                                 for (ComponentInstance ci : componentInstances) {
 231                                         if (!vfIds.contains(ci.getComponentUid())) {
 232                                                 vfIds.add(ci.getComponentUid());
 233                                                 ComponentParametersView filter = new ComponentParametersView(true);
 234                                                 filter.setIgnoreComponentInstances(false);
 235                                                 filter.setIgnoreArtifacts(false);
 236                                                 filter.setIgnoreGroups(false);
 237                                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 238                                                                 .getToscaElement(ci.getComponentUid(), filter);
 239                                                 if (toscaElement.isRight()) {
 240                                                         log.info("Failed to fetch resource {} {}", ci.getComponentUid(),
 241                                                                         toscaElement.right().value());
 242                                                         continue;
 243                                                 }
 244                                                 Resource resource = toscaElement.left().value();
 245                                                 if (resource.getResourceType().equals(ResourceTypeEnum.VF)) {
 246                                                         vfLst.add(resource);
 247                                                         writeModuleResultToFile(writer, resource, service);
 248                                                         writer.flush();
 249
 250                                                 }
 251                                                 janusGraphDao.commit();
 252                                         }
 253                                 }
 254                         }
 255                         log.info("output file with list of Vf : {}", fileName);
 256                 } catch (Exception e) {
 257                         log.info("Failed to fetch services ", e);
 258                         return false;
 259                 } finally {
 260                         janusGraphDao.commit();
 261                 }
 262                 return true;
 263         }
 264
 265         private boolean fetchServices(String fixServices, List<Service> serviceList, long time) {
 266                 log.info("Find problem Services {}", fixServices);
 267                 String fileName = "problemService_" + time + ".csv";
 268                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 269                         writer.write("service name, service id, state, version\n");
 270
 271                         Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
 272                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
 273                         if ("distributed_only".equals(fixServices)) {
 274                                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
 275                                 hasProps.put(GraphPropertyEnum.DISTRIBUTION_STATUS, DistributionStatusEnum.DISTRIBUTED.name());
 276                         }
 277
 278                         Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
 279                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 280                         log.info("Try to fetch services with properties {} and not {}", hasProps, hasNotProps);
 281
 282                         Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
 283                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 284                         if (servicesByCriteria.isRight()) {
 285                                 log.info("Failed to fetch services {}", servicesByCriteria.right().value());
 286                                 return false;
 287                         }
 288                         List<GraphVertex> services = servicesByCriteria.left().value();
 289                         for (GraphVertex gv : services) {
 290                                 ComponentParametersView filter = new ComponentParametersView(true);
 291                                 filter.setIgnoreComponentInstances(false);
 292                                 filter.setIgnoreArtifacts(false);
 293                                 filter.setIgnoreGroups(false);
 294
 295                                 Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade
 296                                                 .getToscaElement(gv.getUniqueId(), filter);
 297                                 if (toscaElement.isRight()) {
 298                                         log.info("Failed to fetch service {} {}", gv.getUniqueId(), toscaElement.right().value());
 299                                         continue;
 300                                 }
 301                                 Service service = toscaElement.left().value();
 302
 303                                 String serviceName = (String) gv.getMetadataProperty(GraphPropertyEnum.NAME);
 304
 305                                 boolean isProblematic = isProblematicService(service, serviceName);
 306                                 if (isProblematic) {
 307                                         serviceList.add(service);
 308                                         writeModuleResultToFile(writer, service, null);
 309                                         writer.flush();
 310
 311                                 }
 312
 313                                 janusGraphDao.commit();
 314                         }
 315                         log.info("output file with list of services : {}", fileName);
 316                 } catch (Exception e) {
 317                         log.info("Failed to fetch services ", e);
 318                         return false;
 319                 } finally {
 320                         janusGraphDao.commit();
 321                 }
 322                 return true;
 323         }
 324
 325         private boolean isProblematicService( Service service, String serviceName) {
 326
 327                 List<ComponentInstance> componentInstances = service.getComponentInstances();
 328
 329                 if (componentInstances == null) {
 330                         log.info("No instances for service {} ", service.getUniqueId());
 331                         return false;
 332                 }
 333                 boolean isCheckVFModules = true;
 334                 if(service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN ||
 335                                 service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT){
 336                         isCheckVFModules = false;
 337                 }
 338                 for (ComponentInstance ci : componentInstances) {
 339                         Map<String, ArtifactDefinition> deploymentArtifacts = ci.getDeploymentArtifacts();
 340                         List<GroupInstance> groupInstances = ci.getGroupInstances();
 341                         if (groupInstances == null || groupInstances.isEmpty()) {
 342                                 log.info("No instance groups for instance {} in service {} id {} ", ci.getName(), serviceName,
 343                                                 service.getUniqueId());
 344                                 continue;
 345                         }
 346                         List<VfModuleArtifactPayloadEx> vfModules = null;
 347                         if(isCheckVFModules){
 348                                 Optional<ArtifactDefinition> optionalVfModuleArtifact = deploymentArtifacts.values().stream()
 349                                                 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 350
 351                                  if(!optionalVfModuleArtifact.isPresent())
 352                                          return true;
 353
 354                                  ArtifactDefinition vfModuleArtifact = optionalVfModuleArtifact.get();
 355                                  Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> vfModulesEither = parseVFModuleJson(vfModuleArtifact);
 356                                  if(vfModulesEither.isRight()){
 357                                          log.error("Failed to parse vfModule for service {} status is {}", service.getUniqueId(), vfModulesEither.right().value());
 358                                          return true;
 359                                  }
 360                                  vfModules = vfModulesEither.left().value();
 361                                  if(vfModules == null || vfModules.isEmpty() ){
 362                                          log.info("vfModules empty for service {}", service.getUniqueId());
 363                                          return true;
 364                                  }
 365                         }
 366
 367                         for (GroupInstance gi : groupInstances) {
 368                                 if (gi.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 369                                         VfModuleArtifactPayloadEx vfModule= null;
 370                                         if(isCheckVFModules && vfModules != null){
 371                                                 Optional<VfModuleArtifactPayloadEx> op = vfModules.stream().filter(vf -> vf.getVfModuleModelName().equals(gi.getGroupName())).findAny();
 372                                                 if(!op.isPresent()){
 373                                                          log.error("Failed to find vfModule for group {}", gi.getGroupName());
 374                                                         return true;
 375                                                 }
 376                                                 vfModule = op.get();
 377                                         }
 378                                         if (isProblematicGroupInstance(gi, ci.getName(), serviceName, deploymentArtifacts, vfModule)) {
 379                                                 return true;
 380                                         }
 381                                 }
 382                         }
 383
 384                 }
 385                 return false;
 386         }
 387
 388         private boolean isProblematicGroup(GroupDefinition gr, String resourceName,
 389                         Map<String, ArtifactDefinition> deploymentArtifacts) {
 390                 List<String> artifacts = gr.getArtifacts();
 391                 List<String> artifactsUuid = gr.getArtifactsUuid();
 392                 Set<String> artifactsSet = new HashSet<>();
 393
 394                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 395                         log.info("No groups in resource {} ", resourceName);
 396                         return true;
 397                 }
 398                 artifactsSet.addAll(artifacts);
 399                 if (artifactsSet.size() < artifacts.size()) {
 400                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", gr.getName(), resourceName);
 401                         return true;
 402                 }
 403
 404                 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
 405                         log.info(" artifacts.size() < artifactsUuid.size() group {} in resource {} ", gr.getName(), resourceName);
 406                         return true;
 407                 }
 408                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 409                         log.info(
 410                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() group {} in resource {} ",
 411                                         gr.getName(), resourceName);
 412                         return true;
 413                 }
 414                 if (artifactsUuid != null && artifactsUuid.contains(null)) {
 415                         log.info(" artifactsUuid.contains(null) group {} in resource {} ", gr.getName(), resourceName);
 416                         return true;
 417                 }
 418
 419                 for (String artifactId : artifacts) {
 420                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 421                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 422                         if (artifactDefinition == null) {
 423                                 log.info(" artifactDefinition == null label {} group {} in resource {} ", artifactlabel, gr.getName(),
 424                                                 resourceName);
 425                                 return true;
 426                         }
 427                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 428                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 429                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 430                                         log.info(
 431                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} group {} in resource {} ",
 432                                                         artifactlabel, artifactId, gr.getName(), resourceName);
 433                                         return true;
 434                                 }
 435                                 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 436                                         log.info(
 437                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} group {} in resource {} ",
 438                                                         artifactlabel, gr.getName(), resourceName);
 439                                         return true;
 440                                 }
 441                         }
 442                 }
 443                 for (String artifactUUID : artifactsUuid) {
 444                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 445                         if (label != null && !label.isEmpty() && !label.equals("")) {
 446                                 return true;
 447                         }
 448                 }
 449
 450                 return false;
 451         }
 452
 453         private boolean isProblematicGroupInstance(GroupInstance gi, String instName, String servicename,
 454                         Map<String, ArtifactDefinition> deploymentArtifacts, VfModuleArtifactPayloadEx vfModule) {
 455                 List<String> artifacts = gi.getArtifacts();
 456                 List<String> artifactsUuid = gi.getArtifactsUuid();
 457                 List<String> instArtifactsUuid = gi.getGroupInstanceArtifactsUuid();
 458                 List<String> instArtifactsId = gi.getGroupInstanceArtifacts();
 459                 Set<String> instArtifatIdSet = new HashSet<>();
 460                 Set<String> artifactsSet = new HashSet<>();
 461
 462                 log.info("check group {} for instance {} ", gi.getGroupName(), instName);
 463                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 464                         log.info("No instance groups for instance {} in service {} ", instName, servicename);
 465                         return true;
 466                 }
 467                 artifactsSet.addAll(artifacts);
 468                 if (artifactsSet.size() < artifacts.size()) {
 469                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", instName, servicename);
 470                         return true;
 471                 }
 472
 473                 if (instArtifactsId != null && !instArtifactsId.isEmpty()) {
 474                         instArtifatIdSet.addAll(instArtifactsId);
 475                 }
 476
 477                 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
 478                         log.info(" artifacts.size() < artifactsUuid.size() inst {} in service {} ", instName, servicename);
 479                         return true;
 480                 }
 481                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 482                         log.info(
 483                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() inst {} in service {} ",
 484                                         instName, servicename);
 485                         return true;
 486                 }
 487                 if (artifactsUuid != null && artifactsUuid.contains(null)) {
 488                         log.info(" artifactsUuid.contains(null) inst {} in service {} ", instName, servicename);
 489                         return true;
 490                 }
 491                 if (instArtifactsId != null && instArtifatIdSet.size() < instArtifactsId.size()) {
 492                         log.info(" instArtifatIdSet.size() < instArtifactsId.size() inst {} in service {} ", instName, servicename);
 493                         return true;
 494                 }
 495
 496                 if ((instArtifactsId != null && instArtifactsUuid != null)
 497                                 && instArtifactsId.size() != instArtifactsUuid.size()) {
 498                         log.info(" instArtifactsId.size() != instArtifactsUuid.size() inst {} in service {} ", instName,
 499                                         servicename);
 500                         return true;
 501                 }
 502
 503                 for (String artifactId : artifacts) {
 504                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 505                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 506                         if (artifactDefinition == null) {
 507                                 log.info(" artifactDefinition == null label {} inst {} in service {} ", artifactlabel, instName,
 508                                                 servicename);
 509                                 return true;
 510                         }
 511                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 512                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 513                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 514                                         log.info(
 515                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} inst {} in service {} ",
 516                                                         artifactlabel, artifactId, instName, servicename);
 517                                         return true;
 518                                 }
 519                                 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 520                                         log.info(
 521                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 522                                                         artifactlabel, instName, servicename);
 523                                         return true;
 524                                 }
 525                         } else {
 526                                 if (instArtifactsUuid == null || instArtifactsUuid.isEmpty()) {
 527                                         log.info(" instArtifactsUuid empty. label {} inst {} in service {} ", artifactlabel, instName,
 528                                                         servicename);
 529                                         return true;
 530                                 }
 531                                 if (!instArtifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 532                                         log.info(
 533                                                         " instArtifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 534                                                         artifactlabel, instName, servicename);
 535                                         return true;
 536                                 }
 537                         }
 538                 }
 539                 for (String artifactUUID : artifactsUuid) {
 540                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 541                         if (label != null && !label.isEmpty() && !"".equals(label)) {
 542                                 return true;
 543                         }
 544                 }
 545                 if(vfModule != null && artifactsUuid != null){
 546                         return isProblematicVFModule(vfModule, artifactsUuid, instArtifactsUuid);
 547                 }
 548
 549                 return false;
 550         }
 551
 552         private boolean isProblematicVFModule(VfModuleArtifactPayloadEx vfModule, List<String> artifactsUuid,
 553                         List<String> instArtifactsUuid) {
 554                 log.info(" isProblematicVFModule  {}  ", vfModule.getVfModuleModelName());
 555                 List<String> vfModuleArtifacts = vfModule.getArtifacts();
 556                 List<String> allArtifacts = new ArrayList<>();
 557                 allArtifacts.addAll(artifactsUuid);
 558                 if(instArtifactsUuid != null)
 559                         allArtifacts.addAll(instArtifactsUuid);
 560                 if((vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()){
 561                         log.error(" vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()");
 562                         return true;
 563                 }
 564                 if(vfModuleArtifacts!= null){
 565                         if( vfModuleArtifacts.size() != allArtifacts.size()){
 566                                 log.error(" vfModuleArtifacts.size() != allArtifacts.size()");
 567                                 return true;
 568                         }
 569                         for(String vfModuleArtifact: vfModuleArtifacts){
 570                                 Optional<String> op = allArtifacts.stream().filter(a -> a.equals(vfModuleArtifact)).findAny();
 571                                 if(!op.isPresent()){
 572                                         log.error("failed to find artifact {} in group artifacts {}", vfModuleArtifact,  allArtifacts);
 573                                         return true;
 574                                 }
 575                         }
 576                 }
 577                 return false;
 578         }
 579
 580
 581
 582         private boolean fix(List<Resource> vfLst, List<Service> serviceList, Map<String, List<Component>> nodesToFixTosca,
 583                         Map<String, List<Component>> vfToFixTosca, Map<String, List<Component>> servicesToFixTosca) {
 584                 boolean res = true;
 585                 log.info(" Fix started ***** ");
 586                 if (vfLst != null && !vfLst.isEmpty()) {
 587                         res = fixVf(vfLst);
 588
 589                 }
 590
 591                 if (res && serviceList != null && !serviceList.isEmpty()) {
 592                         res = fixServices(serviceList);
 593
 594                 }
 595
 596                 Set<String> fixedIds = new HashSet<>();
 597
 598                 long time = System.currentTimeMillis();
 599                 String fileName = "FailedGenerateTosca" + "_" + time + ".csv";
 600                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 601                         writer.write("componentType, name, version, UID, UUID, invariantUUID, state\n");
 602                         List<Component> failedList = new ArrayList<>();
 603
 604                         if (res && nodesToFixTosca != null && !nodesToFixTosca.isEmpty()) {
 605
 606                                 generateAndSaveToscaArtifacts(nodesToFixTosca, fixedIds, null, failedList);
 607
 608                         }
 609                         if (vfToFixTosca != null && !vfToFixTosca.isEmpty()) {
 610
 611                                 generateAndSaveToscaArtifacts(vfToFixTosca, fixedIds, vfLst, failedList);
 612
 613                         }
 614
 615                         for (Component component : vfLst) {
 616                                 res = generateToscaPerComponent(fixedIds, component);
 617                                 if (res) {
 618                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 619                                         Map<String, GroupDataDefinition> groups = topologyTemplate.getGroups();
 620                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.GROUPS, EdgeLabelEnum.GROUPS, groups);
 621                                         if (res) {
 622                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getDeploymentArtifacts();
 623                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.DEPLOYMENT_ARTIFACTS,
 624                                                                 EdgeLabelEnum.DEPLOYMENT_ARTIFACTS, arifacts);
 625                                         }
 626                                         if (res) {
 627                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 628                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 629                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 630                                         }
 631                                         janusGraphDao.commit();
 632                                 } else {
 633                                         failedList.add(component);
 634                                 }
 635                         }
 636
 637                         if (servicesToFixTosca != null && !servicesToFixTosca.isEmpty()) {
 638                                 generateAndSaveToscaArtifacts(servicesToFixTosca, fixedIds, serviceList, failedList);
 639
 640                         }
 641
 642
 643                         for (Component component : serviceList) {
 644                                 res = generateToscaPerComponent(fixedIds, component);
 645                                 if (res) {
 646                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 647                                         Map<String, MapGroupsDataDefinition> groups = topologyTemplate.getInstGroups();
 648                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_GROUPS, EdgeLabelEnum.INST_GROUPS,
 649                                                         groups);
 650
 651                                         if (res) {
 652                                                 Map<String, MapArtifactDataDefinition> artifacts = topologyTemplate
 653                                                                 .getInstDeploymentArtifacts();
 654                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_DEPLOYMENT_ARTIFACTS,
 655                                                                 EdgeLabelEnum.INST_DEPLOYMENT_ARTIFACTS, artifacts);
 656                                         }
 657                                         if (res) {
 658                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 659                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 660                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 661                                         }
 662                                         janusGraphDao.commit();
 663                                 } else {
 664                                         failedList.add(component);
 665                                 }
 666
 667                         }
 668                         if (!failedList.isEmpty()) {
 669                                 for (Component component : failedList) {
 670                                         StringBuilder sb = new StringBuilder(component.getComponentType().getValue());
 671                                         sb.append(",").append(component.getName()).append(",").append(component.getVersion()).append(",")
 672                                                         .append(component.getUniqueId()).append(",").append(component.getUUID()).append(",")
 673                                                         .append(component.getInvariantUUID()).append(",").append(component.getLifecycleState());
 674
 675                                         sb.append("\n");
 676                                         writer.write(sb.toString());
 677                                 }
 678                                 writer.flush();
 679                         }
 680                 } catch (IOException e) {
 681                     log.error(e.getMessage());
 682                 } finally {
 683                         janusGraphDao.commit();
 684                 }
 685                 log.info(" Fix finished with res {} ***** ", res);
 686                 return res;
 687         }
 688
 689         private boolean generateAndSaveToscaArtifacts(Map<String, List<Component>> nodesToFixTosca, Set<String> fixedIds,
 690                         List<? extends Component> componentsWithFailedGroups, List<Component> failedList) {
 691                 boolean res = true;
 692                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts started ");
 693                 for (Map.Entry<String, List<Component>> entry : nodesToFixTosca.entrySet()) {
 694
 695                         List<Component> component = entry.getValue();
 696                         for (Component c : component) {
 697                                 log.debug("Migration1707ArtifactUuidFix  fix tosca on component : id {},  name {} ", c.getUniqueId(),
 698                                                 c.getName());
 699                                 if (componentsWithFailedGroups != null) {
 700                                         Optional<Component> op = (Optional<Component>) componentsWithFailedGroups.stream()
 701                                                         .filter(cg -> cg.getUniqueId().equals(c.getUniqueId())).findAny();
 702                                         if (!op.isPresent())
 703                                                 res = generateToscaPerComponent(fixedIds, c);
 704                                 } else
 705                                         res = generateToscaPerComponent(fixedIds, c);
 706                                 if (res) {
 707                                         ToscaElement topologyTemplate = ModelConverter.convertToToscaElement(c);
 708                                         Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 709                                         res = fixDataOnGraph(c.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS, EdgeLabelEnum.TOSCA_ARTIFACTS,
 710                                                         arifacts);
 711                                         janusGraphDao.commit();
 712                                 } else {
 713                                         failedList.add(c);
 714                                 }
 715
 716                         }
 717                 }
 718                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts finished with res {} ", res);
 719                 return res;
 720         }
 721
 722         private boolean generateToscaPerComponent(Set<String> fixedIds, Component c) {
 723                 boolean res = true;
 724                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent started component name {} id {}",
 725                                 c.getName(), c.getUniqueId());
 726                 try {
 727                         Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
 728                                         .getToscaFullElement(c.getUniqueId());
 729                         if (toscaElement.isRight()) {
 730                                 log.info("Failed to fetch resources {} {}", c.getUniqueId(), toscaElement.right().value());
 731                                 return false;
 732                         }
 733                         Component toscaElementFull = toscaElement.left().value();
 734                         toscaElementFull.setGroups(c.getGroups());
 735                         List<ComponentInstance> ciListFull = toscaElementFull.getComponentInstances();
 736                         List<ComponentInstance> ciList = c.getComponentInstances();
 737                         if (ciListFull != null && !ciListFull.isEmpty()) {
 738                                 ciListFull.forEach(ciFull -> {
 739                                         ComponentInstance compInst = ciList.stream()
 740                                                         .filter(ci -> ci.getUniqueId().equals(ciFull.getUniqueId())).findAny().get();
 741                                         ciFull.setGroupInstances(compInst.getGroupInstances());
 742                                 });
 743                         }
 744
 745                         Either<Component, ToscaError> either = generateToscaArtifact(toscaElementFull);
 746
 747                         if (either.isRight()) {
 748                                 log.error("Couldn't generate and save tosca template component  unique id {}, name {} error: {}",
 749                                                 toscaElementFull.getUniqueId(), toscaElementFull.getName(), either.right().value());
 750                                 res = false;
 751
 752                         }
 753
 754                         if (res) {
 755                                 c.setToscaArtifacts(either.left().value().getToscaArtifacts());
 756                                 fixedIds.add(toscaElementFull.getUniqueId());
 757                         }
 758                 } finally {
 759                         if (res)
 760                                 janusGraphDao.commit();
 761                         else
 762                                 janusGraphDao.rollback();
 763                 }
 764                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent finished  component name {} id {} res {}",
 765                                 c.getName(), c.getUniqueId(), res);
 766                 return res;
 767         }
 768
 769         private <T extends ToscaDataDefinition> boolean fixDataOnGraph(String componentId, VertexTypeEnum vertexTypeEnum,
 770                         EdgeLabelEnum edgeLabelEnum, Map<String, T> groups) {
 771                 log.debug("amount groups to update: VertexTypeEnum {} EdgeLabelEnum {} data size {}", vertexTypeEnum.getName(),
 772                                 edgeLabelEnum, groups.size());
 773                 boolean res = true;
 774                 Either<GraphVertex, JanusGraphOperationStatus> getResponse = janusGraphDao.getVertexById(componentId,
 775                                 JsonParseFlagEnum.NoParse);
 776                 if (getResponse.isRight()) {
 777                         log.debug("Couldn't fetch component  unique id {}, error: {}", componentId, getResponse.right().value());
 778                         res = false;
 779
 780                 }
 781                 if (res) {
 782                         GraphVertex componentVertex = getResponse.left().value();
 783
 784                         GraphVertex toscaDataVertex = null;
 785                         Either<GraphVertex, JanusGraphOperationStatus> groupVertexEither = janusGraphDao.getChildVertex(componentVertex,
 786                                         edgeLabelEnum, JsonParseFlagEnum.ParseJson);
 787                         if (groupVertexEither.isRight() && groupVertexEither.right().value() == JanusGraphOperationStatus.NOT_FOUND) {
 788                                 log.debug("no child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum, componentId,
 789                                                 groupVertexEither.right().value());
 790                                 return true;
 791                         }
 792                         if (groupVertexEither.isRight()) {
 793                                 res = false;
 794                                 log.debug("failed to get child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum,
 795                                                 componentId, groupVertexEither.right().value());
 796                         }
 797                         if (res) {
 798                                 toscaDataVertex = groupVertexEither.left().value();
 799                                 toscaDataVertex.setJson(groups);
 800                                 Either<GraphVertex, JanusGraphOperationStatus> updatevertexEither = janusGraphDao.updateVertex(toscaDataVertex);
 801                                 if (updatevertexEither.isRight()) {
 802                                         log.debug("failed to update vertex for component  unique id {}, error: {}", componentId,
 803                                                         updatevertexEither.right().value());
 804                                         janusGraphDao.rollback();
 805                                         return false;
 806                                 }
 807                         }
 808                 }
 809                 log.debug("Fix data on graph finished: VertexTypeEnum {} EdgeLabelEnum {} res {}", vertexTypeEnum.getName(),
 810                                 res);
 811                 return res;
 812         }
 813
 814         private boolean fixServices(List<Service> serviceList) {
 815                 for (Service service : serviceList) {
 816                         log.debug("Migration1707ArtifactUuidFix  fix service: id {},  name {} ", service.getUniqueId(),
 817                                         service.getName());
 818                         List<ComponentInstance> instances = service.getComponentInstances();
 819                         for (ComponentInstance instance : instances) {
 820                                 fixComponentInstances(service, instance);
 821                         }
 822
 823                 }
 824                 return true;
 825
 826         }
 827
 828         private void fixComponentInstances(Service service, ComponentInstance instance) {
 829                 Map<String, ArtifactDefinition> artifactsMap = instance.getDeploymentArtifacts();
 830                 List<GroupInstance> groupsList = instance.getGroupInstances();
 831                 if (groupsList != null && artifactsMap != null) {
 832                         List<GroupInstance> groupsToDelete = new ArrayList<>();
 833                         for (GroupInstance group : groupsList) {
 834                                 fixGroupInstances(service, artifactsMap, groupsToDelete, group);
 835
 836                         }
 837
 838                         if (!groupsToDelete.isEmpty()) {
 839                                 log.debug("Migration1707ArtifactUuidFix  delete group:  resource id {}, group instance to delete {} ",
 840                                                 service.getUniqueId(), groupsToDelete);
 841                                 groupsList.removeAll(groupsToDelete);
 842
 843                         }
 844
 845                         Optional<ArtifactDefinition> optionalVfModuleArtifact = artifactsMap.values().stream()
 846                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 847                         ArtifactDefinition vfModuleArtifact;
 848                         if(!optionalVfModuleArtifact.isPresent()){
 849                                 vfModuleArtifact = createVfModuleArtifact(instance, service);
 850                                 artifactsMap.put(vfModuleArtifact.getArtifactLabel(), vfModuleArtifact);
 851                         }
 852                         else {
 853                                 vfModuleArtifact = optionalVfModuleArtifact.get();
 854                         }
 855                         fillVfModuleInstHeatEnvPayload(service, instance, groupsList, vfModuleArtifact);
 856                 }
 857         }
 858
 859         private void fixGroupInstances(Service service, Map<String, ArtifactDefinition> artifactsMap,
 860                         List<GroupInstance> groupsToDelete, GroupInstance group) {
 861                 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 862                         log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", service.getUniqueId(),
 863                                         group.getName());
 864                         if(group.getArtifacts() != null){
 865                                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 866                                 if(group.getGroupInstanceArtifacts() != null){
 867                                         List<String> groupInsArtifacts = new ArrayList<>(group.getGroupInstanceArtifacts());
 868                                         groupArtifactsSet.addAll(groupInsArtifacts);
 869                                 }
 870                                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 871
 872                                 clearGroupInstanceArtifacts(group);
 873
 874                                 for (String artifactId : groupArtifacts) {
 875                                         fixArtifactUndergroupInstances(artifactsMap, group, groupArtifacts, artifactId);
 876                                 }
 877                         }
 878                         if (group.getArtifacts() == null || group.getArtifacts().isEmpty()) {
 879                                 log.debug(
 880                                                 "Migration1707ArtifactUuidFix  fix groupInstance add to delete list:  resource id {} name {} , group name {} ",
 881                                                 service.getUniqueId(), service.getName(), group.getName());
 882                                 groupsToDelete.add(group);
 883                         }
 884                 }
 885         }
 886
 887         private void clearGroupInstanceArtifacts(GroupInstance group) {
 888                 if(group.getArtifacts() != null)
 889                         group.getArtifacts().clear();
 890                 else
 891                         group.setArtifacts(new ArrayList<>());
 892                 if(group.getArtifactsUuid() != null)
 893                         group.getArtifactsUuid().clear();
 894                 else{
 895                         group.setArtifactsUuid(new ArrayList<>());
 896                 }
 897                 if(group.getGroupInstanceArtifacts() != null)
 898                         group.getGroupInstanceArtifacts().clear();
 899                 else{
 900                         group.setGroupInstanceArtifacts(new ArrayList<>());
 901                 }
 902                 if(group.getGroupInstanceArtifactsUuid() != null )
 903                         group.getGroupInstanceArtifactsUuid().clear();
 904                 else
 905                         group.setGroupInstanceArtifactsUuid(new ArrayList<>());
 906         }
 907
 908         private void fixArtifactUndergroupInstances(Map<String, ArtifactDefinition> artifactsMap, GroupInstance group,
 909                         List<String> groupArtifacts, String artifactId) {
 910                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 911                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 912                                 group.getName(), artifactId, artifactlabel);
 913                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 914                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 915                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
 916                         String correctArtifactId = artifact.getUniqueId();
 917                         String correctArtifactUUID = artifact.getArtifactUUID();
 918                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 919                                 boolean isAddToGroup = true;
 920                                 if (groupArtifacts.size() == 1) {
 921                                         if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 922                                                 isAddToGroup = false;
 923                                                 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 924                                         }
 925                                 }
 926                                 if (isAddToGroup) {
 927                                         log.debug(
 928                                                         MIGRATION1707_ARTIFACT_UUID_FIX,
 929                                                         group.getName(), correctArtifactId, correctArtifactUUID);
 930                                         group.getArtifacts().add(correctArtifactId);
 931                                         if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 932                                                 group.getArtifactsUuid().add(correctArtifactUUID);
 933                                         }
 934                                 }
 935                         } else {
 936                                 log.debug(
 937                                                 MIGRATION1707_ARTIFACT_UUID_FIX,
 938                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 939                                 Set<String> tmpSet = new HashSet<>(group.getGroupInstanceArtifacts());
 940                                 tmpSet.add(correctArtifactId);
 941                                 group.setGroupInstanceArtifacts(new ArrayList<>(tmpSet));
 942                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 943                                         Set<String> tmpSetUUID = new HashSet<>(group.getGroupInstanceArtifactsUuid());
 944                                         tmpSetUUID.add(correctArtifactUUID);
 945                                         group.setGroupInstanceArtifactsUuid(new ArrayList<>(tmpSetUUID));
 946                                 }
 947                         }
 948                 }
 949         }
 950
 951         private boolean fixVf(List<Resource> vfLst) {
 952                 for (Resource resource : vfLst) {
 953                         log.debug("Migration1707ArtifactUuidFix  fix resource: id {},  name {} ", resource.getUniqueId(),
 954                                         resource.getName());
 955                         Map<String, ArtifactDefinition> artifactsMap = resource.getDeploymentArtifacts();
 956                         List<GroupDefinition> groupsList = resource.getGroups();
 957                         List<GroupDefinition> groupsToDelete = new ArrayList<>();
 958                         if (groupsList != null && artifactsMap != null) {
 959                                 for (GroupDefinition group : groupsList) {
 960                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE) && group.getArtifacts() != null) {
 961                                                 fixVfGroup(resource, artifactsMap, group);
 962                                         }
 963                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)
 964                                                         && (group.getArtifacts() == null || group.getArtifacts().isEmpty())) {
 965                                                 log.debug(
 966                                                                 "Migration1707ArtifactUuidFix  add group to delete list fix resource: id {},  name {} ",
 967                                                                 resource.getUniqueId(), resource.getName(), group.getName());
 968                                                 groupsToDelete.add(group);
 969                                         }
 970                                 }
 971
 972                                 if (!groupsToDelete.isEmpty()) {
 973                                         groupsList.removeAll(groupsToDelete);
 974
 975                                 }
 976                         }
 977
 978                 }
 979
 980                 return true;
 981         }
 982
 983         private void fixVfGroup(Resource resource, Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group) {
 984                 log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", resource.getUniqueId(),
 985                                 group.getName());
 986                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 987                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 988                 group.getArtifacts().clear();
 989                 group.getArtifactsUuid().clear();
 990
 991                 for (String artifactId : groupArtifacts) {
 992                         fixArtifactUnderGroup(artifactsMap, group, groupArtifacts, artifactId);
 993                 }
 994         }
 995
 996         private void fixArtifactUnderGroup(Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group,
 997                         List<String> groupArtifacts, String artifactId) {
 998
 999                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
1000                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
1001                                 group.getName(), artifactId, artifactlabel);
1002                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
1003                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
1004                         String correctArtifactId = artifact.getUniqueId();
1005                         String correctArtifactUUID = artifact.getArtifactUUID();
1006                         boolean isAddToGroup = true;
1007                         if (groupArtifacts.size() == 1) {
1008                                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
1009                                 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
1010                                         isAddToGroup = false;
1011                                         artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
1012                                 }
1013                         }
1014                         if (isAddToGroup) {
1015                                 log.debug(
1016                                                 MIGRATION1707_ARTIFACT_UUID_FIX,
1017                                                 group.getName(), correctArtifactId, correctArtifactUUID);
1018                                 group.getArtifacts().add(correctArtifactId);
1019                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
1020                                         group.getArtifactsUuid().add(correctArtifactUUID);
1021                                 }
1022                         }
1023
1024                 }
1025         }
1026
1027         private String findArtifactLabelFromArtifactId(String artifactId) {
1028                 String artifactLabel = "";
1029
1030                 int index = artifactId.lastIndexOf('.');
1031                 if (index > 0 && index + 1 < artifactId.length())
1032                         artifactLabel = artifactId.substring(index + 1);
1033                 return artifactLabel;
1034         }
1035
1036         private void writeModuleResultToFile(Writer writer, org.openecomp.sdc.be.model.Component component,
1037                         Service service) {
1038                 try {
1039                         // "service name, service id, state, version
1040                         StringBuilder sb = new StringBuilder(component.getName());
1041                         sb.append(",").append(component.getUniqueId()).append(",").append(component.getLifecycleState()).append(",")
1042                                         .append(component.getVersion());
1043                         if (service != null) {
1044                                 sb.append(",").append(service.getName());
1045                         }
1046                         sb.append("\n");
1047                         writer.write(sb.toString());
1048                 } catch (IOException e) {
1049                         log.error(e.getMessage());
1050                 }
1051         }
1052
1053         private void writeModuleResultToFile(Writer writer, List<Component> components) {
1054                 try {
1055                         // "service name, service id, state, version
1056                         for (Component component : components) {
1057                                 StringBuilder sb = new StringBuilder(component.getName());
1058                                 sb.append(",").append(component.getUniqueId()).append(",").append(component.getInvariantUUID())
1059                                                 .append(",").append(component.getLifecycleState()).append(",").append(component.getVersion());
1060
1061                                 sb.append("\n");
1062                                 writer.write(sb.toString());
1063                         }
1064                 } catch (IOException e) {
1065
1066                     log.error(e.getMessage());
1067                 }
1068         }
1069
1070         public boolean doFixTosca(Map<String, List<Component>> nodeToFix, Map<String, List<Component>> vfToFix,
1071                         Map<String, List<Component>> serviceToFix) {
1072
1073                 Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
1074                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1075                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1076
1077                 Map<String, List<Component>> vertices = getVerticesToValidate(VertexTypeEnum.NODE_TYPE, hasProps);
1078                 boolean result = validateTosca(vertices, nodeToFix, "RESOURCE_TOSCA_ARTIFACTS");//
1079
1080                 hasProps.clear();
1081                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1082                 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF);
1083                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1084
1085                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1086                 result = validateTosca(vertices, vfToFix, "VF_TOSCA_ARTIFACTS");
1087
1088                 hasProps.clear();
1089                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
1090                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1091
1092                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1093                 result = validateTosca(vertices, serviceToFix, "SERVICE_TOSCA_ARTIFACTS");
1094
1095                 return result;
1096         }
1097
1098         public Map<String, List<Component>> getVerticesToValidate(VertexTypeEnum type,
1099                         Map<GraphPropertyEnum, Object> hasProps) {
1100
1101                 Map<String, List<Component>> result = new HashMap<>();
1102                 try {
1103
1104                         Either<List<GraphVertex>, JanusGraphOperationStatus> resultsEither = janusGraphDao
1105           .getByCriteria(type, hasProps);
1106                         if (resultsEither.isRight()) {
1107                                 log.error("getVerticesToValidate failed {} ",resultsEither.right().value());
1108                                 return result;
1109                         }
1110                         log.info("getVerticesToValidate: {}  vertices to scan", resultsEither.left().value().size());
1111                         List<GraphVertex> componentsList = resultsEither.left().value();
1112                         componentsList.forEach(vertex -> {
1113                                 String ivariantUuid = (String) vertex.getMetadataProperty(GraphPropertyEnum.INVARIANT_UUID);
1114                                 if (!result.containsKey(ivariantUuid)) {
1115                                         List<Component> compList = new ArrayList<>();
1116                                         result.put(ivariantUuid, compList);
1117                                 }
1118                                 List<Component> compList = result.get(ivariantUuid);
1119
1120                                 ComponentParametersView filter = new ComponentParametersView(true);
1121                                 filter.setIgnoreArtifacts(false);
1122
1123                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1124                                                 .getToscaElement(vertex.getUniqueId(), filter);
1125                                 if (toscaElement.isRight()) {
1126                                         log.error("getVerticesToValidate: failed to find element {}  staus is {}", vertex.getUniqueId()
1127                                                         ,toscaElement.right().value());
1128                                 } else {
1129                                         compList.add(toscaElement.left().value());
1130                                 }
1131                                 janusGraphDao.commit();
1132
1133                         });
1134
1135                 } catch (Exception e) {
1136                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1137
1138                 } finally {
1139                         janusGraphDao.commit();
1140
1141                 }
1142                 return result;
1143         }
1144
1145         public boolean validateTosca(Map<String, List<Component>> vertices, Map<String, List<Component>> compToFix,
1146                         String name) {
1147                 boolean result = true;
1148                 long time = System.currentTimeMillis();
1149                 String fileName = name + "_" + time + ".csv";
1150                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
1151                         writer.write("name, UUID, invariantUUID, state, version\n");
1152                         for (Map.Entry<String, List<Component>> entry : vertices.entrySet()) {
1153                                 List<Component> compList = entry.getValue();
1154                                 Set<String> artifactEsId = new HashSet<>();
1155                                 for (Component component : compList) {
1156                                         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
1157                                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1158                                                         .filter(a -> artifactEsId.contains(a.getEsId()) && a.getEsId() != null).findAny();
1159                                         if (op.isPresent()) {
1160                                                 result = false;
1161                                                 writeModuleResultToFile(writer, compList);
1162                                                 writer.flush();
1163                                                 break;
1164                                         } else {
1165                                                 artifactEsId.addAll(toscaArtifacts.values().stream().map(ArtifactDefinition::getEsId)
1166                                                                 .collect(Collectors.toList()));
1167                                         }
1168                                 }
1169                                 if (!result) {
1170                                         List<Component> compListfull = new ArrayList<>();
1171                                         for (Component c : compList) {
1172                                                 ComponentParametersView filter = new ComponentParametersView(true);
1173                                                 filter.setIgnoreComponentInstances(false);
1174                                                 filter.setIgnoreArtifacts(false);
1175                                                 filter.setIgnoreGroups(false);
1176
1177                                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1178                                                                 .getToscaElement(c.getUniqueId(), filter);
1179                                                 if (toscaElement.isRight()) {
1180                                                         log.debug("getVerticesToValidate: failed to find element" + c.getUniqueId()
1181                                                                         + " status is" + toscaElement.right().value());
1182                                                 } else {
1183                                                         compListfull.add(toscaElement.left().value());
1184                                                 }
1185                                                 this.janusGraphDao.commit();
1186                                         }
1187
1188                                         compToFix.put(entry.getKey(), compListfull);
1189                                         result = true;
1190                                 }
1191
1192                         }
1193
1194                 } catch (Exception e) {
1195                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1196                         return false;
1197                 } finally {
1198                         janusGraphDao.commit();
1199                 }
1200                 return result;
1201         }
1202
1203         private Either<Component, ToscaError> generateToscaArtifact(Component parent) {
1204                 log.debug("tosca artifact generation");
1205                 try {
1206                         Map<String, ArtifactDefinition> toscaArtifacts = parent.getToscaArtifacts();
1207
1208                         ArtifactDefinition toscaArtifact = null;
1209                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1210                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_TEMPLATE.getType())).findAny();
1211
1212                         if (op.isPresent()) {
1213                                 toscaArtifact = op.get();
1214                         }
1215                         if (toscaArtifact != null) {
1216                                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent artifact name {} id {} esId {}",
1217                                                 toscaArtifact.getArtifactName(), toscaArtifact.getUniqueId(), toscaArtifact.getEsId());
1218
1219                                 Either<ToscaRepresentation, ToscaError> exportComponent = toscaExportUtils.exportComponent(parent);
1220                                 if (exportComponent.isRight()) {
1221                                         log.debug("Failed export tosca yaml for component {} error {}", parent.getUniqueId(),
1222                                                         exportComponent.right().value());
1223
1224                                         return Either.right(exportComponent.right().value());
1225                                 }
1226                                 log.debug("Tosca yaml exported for component {} ", parent.getUniqueId());
1227                                 String payload = exportComponent.left().value().getMainYaml();
1228
1229                                 toscaArtifact.setPayloadData(payload);
1230                                 byte[] decodedPayload = toscaArtifact.getPayloadData();
1231
1232                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1233                                                 toscaArtifact.getArtifactLabel());
1234                                 toscaArtifact.setUniqueId(uniqueId);
1235                                 toscaArtifact.setEsId(toscaArtifact.getUniqueId());
1236
1237                                 toscaArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1238                                 ESArtifactData artifactData = new ESArtifactData(toscaArtifact.getEsId(), decodedPayload);
1239                                 artifactCassandraDao.saveArtifact(artifactData);
1240
1241                                 log.debug("Tosca yaml artifact esId  {} ", toscaArtifact.getEsId());
1242                         }
1243                         ArtifactDefinition csarArtifact = null;
1244                         op = toscaArtifacts.values().stream()
1245                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_CSAR.getType())).findAny();
1246
1247                         if (op.isPresent()) {
1248                                 csarArtifact = op.get();
1249                         }
1250
1251                         if (csarArtifact != null) {
1252                                 Either<byte[], ResponseFormat> generated = csarUtils.createCsar(parent, true, true);
1253
1254                                 if (generated.isRight()) {
1255                                         log.debug("Failed to export tosca csar for component {} error {}", parent.getUniqueId(),
1256                                                         generated.right().value());
1257
1258                                         return Either.right(ToscaError.GENERAL_ERROR);
1259                                 }
1260                                 byte[] value = generated.left().value();
1261                                 csarArtifact.setPayload(value);
1262                                 byte[] decodedPayload = csarArtifact.getPayloadData();
1263
1264                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1265                                                 csarArtifact.getArtifactLabel());
1266                                 csarArtifact.setUniqueId(uniqueId);
1267                                 csarArtifact.setEsId(csarArtifact.getUniqueId());
1268
1269                                 csarArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1270                                 ESArtifactData artifactData = new ESArtifactData(csarArtifact.getEsId(), decodedPayload);
1271                                 artifactCassandraDao.saveArtifact(artifactData);
1272                                 log.debug("Tosca csar artifact esId  {} ", csarArtifact.getEsId());
1273
1274                         }
1275
1276                 } catch (Exception ex) {
1277                         log.error("Failed to generate tosca atifact component id {} component name {} error {}",
1278                                         parent.getUniqueId(), parent.getName(), ex.getMessage());
1279
1280                         return Either.right(ToscaError.GENERAL_ERROR);
1281                 }
1282
1283                 return Either.left(parent);
1284         }
1285
1286            private ArtifactDefinition createVfModuleArtifact(ComponentInstance currVF, Service service) {
1287
1288                 ArtifactDefinition vfModuleArtifactDefinition = new ArtifactDefinition();
1289
1290                 vfModuleArtifactDefinition.setDescription("Auto-generated VF Modules information artifact");
1291                 vfModuleArtifactDefinition.setArtifactDisplayName("Vf Modules Metadata");
1292                 vfModuleArtifactDefinition.setArtifactType(ArtifactTypeEnum.VF_MODULES_METADATA.getType());
1293                 vfModuleArtifactDefinition.setArtifactGroupType(ArtifactGroupTypeEnum.DEPLOYMENT);
1294                 vfModuleArtifactDefinition.setArtifactLabel("vfModulesMetadata");
1295                 vfModuleArtifactDefinition.setTimeout(0);
1296                 vfModuleArtifactDefinition.setArtifactName(currVF.getNormalizedName() + "_modules.json");
1297
1298                return vfModuleArtifactDefinition;
1299             }
1300
1301
1302         private void fillVfModuleInstHeatEnvPayload(Component parent, ComponentInstance instance, List<GroupInstance> groupsForCurrVF,
1303                         ArtifactDefinition vfModuleArtifact) {
1304                 log.debug("generate new vf module for component. name  {}, id {}, Version {}", instance.getName(), instance.getUniqueId());
1305
1306                 String uniqueId = UniqueIdBuilder.buildInstanceArtifactUniqueId(parent.getUniqueId(), instance.getUniqueId(), vfModuleArtifact.getArtifactLabel());
1307
1308                 vfModuleArtifact.setUniqueId(uniqueId);
1309                 vfModuleArtifact.setEsId(vfModuleArtifact.getUniqueId());
1310
1311                 List<VfModuleArtifactPayload> vfModulePayloadForCurrVF = new ArrayList<>();
1312                 if (groupsForCurrVF != null) {
1313                         for (GroupInstance groupInstance : groupsForCurrVF) {
1314                                 VfModuleArtifactPayload modulePayload = new VfModuleArtifactPayload(groupInstance);
1315                                 vfModulePayloadForCurrVF.add(modulePayload);
1316                         }
1317                         Collections.sort(vfModulePayloadForCurrVF,
1318                                         (art1, art2) -> VfModuleArtifactPayload.compareByGroupName(art1, art2));
1319
1320                         final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1321
1322                         String vfModulePayloadString = gson.toJson(vfModulePayloadForCurrVF);
1323                         log.debug("vfModulePayloadString {}", vfModulePayloadString);
1324                         if (vfModulePayloadString != null) {
1325                                 String newCheckSum = GeneralUtility
1326                                                 .calculateMD5Base64EncodedByByteArray(vfModulePayloadString.getBytes());
1327                                 vfModuleArtifact.setArtifactChecksum(newCheckSum);
1328
1329                                 ESArtifactData artifactData = new ESArtifactData(vfModuleArtifact.getEsId(),
1330                                                 vfModulePayloadString.getBytes());
1331                                 artifactCassandraDao.saveArtifact(artifactData);
1332
1333                         }
1334
1335                 }
1336
1337         }
1338
1339         private Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> parseVFModuleJson(ArtifactDefinition vfModuleArtifact) {
1340                 log.info("Try to get vfModule json from cassandra {}", vfModuleArtifact.getEsId());
1341                 Either<ESArtifactData, CassandraOperationStatus> vfModuleData = artifactCassandraDao.getArtifact(vfModuleArtifact.getEsId());
1342
1343                 if (vfModuleData.isRight()) {
1344                         CassandraOperationStatus resourceUploadStatus = vfModuleData.right().value();
1345                         StorageOperationStatus storageResponse = DaoStatusConverter.convertCassandraStatusToStorageStatus(resourceUploadStatus);
1346                         log.error("failed to fetch vfModule json {} from cassandra. Status is {}", vfModuleArtifact.getEsId(), storageResponse);
1347                         return Either.right(storageResponse);
1348
1349                 }
1350
1351                 ESArtifactData esArtifactData = vfModuleData.left().value();
1352                 String gsonData = new String( esArtifactData.getDataAsArray());
1353                 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1354                 JsonArray jsonElement = new JsonArray();
1355                 jsonElement = gson.fromJson(gsonData, jsonElement.getClass());
1356                 List<VfModuleArtifactPayloadEx> vfModules = new ArrayList<>();
1357                 jsonElement.forEach(je ->{
1358                         VfModuleArtifactPayloadEx vfModule = ComponentsUtils.parseJsonToObject(je.toString(), VfModuleArtifactPayloadEx.class);
1359                         vfModules.add(vfModule);
1360                 });
1361
1362                 log.debug  ("parse vf module finish {}", gsonData);
1363                 return Either.left(vfModules);
1364
1365         }
1366 }
1367
1368