asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/ArtifactUuidFix.java

   1 package org.openecomp.sdc.asdctool.impl;
   2
   3 import com.google.gson.Gson;
   4 import com.google.gson.GsonBuilder;
   5 import com.google.gson.JsonArray;
   6 import fj.data.Either;
   7 import org.openecomp.sdc.asdctool.impl.validator.utils.VfModuleArtifactPayloadEx;
   8 import org.openecomp.sdc.be.components.distribution.engine.VfModuleArtifactPayload;
   9 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  10 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  11 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
  12 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
  13 import org.openecomp.sdc.be.dao.jsongraph.JanusGraphDao;
  14 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
  15 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
  16 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
  17 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
  18 import org.openecomp.sdc.be.datatypes.elements.GroupDataDefinition;
  19 import org.openecomp.sdc.be.datatypes.elements.MapArtifactDataDefinition;
  20 import org.openecomp.sdc.be.datatypes.elements.MapGroupsDataDefinition;
  21 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
  22 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
  23 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
  24 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  25 import org.openecomp.sdc.be.datatypes.tosca.ToscaDataDefinition;
  26 import org.openecomp.sdc.be.impl.ComponentsUtils;
  27 import org.openecomp.sdc.be.model.*;
  28 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.TopologyTemplate;
  29 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.ToscaElement;
  30 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
  31 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
  32 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  33 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
  34 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
  35 import org.openecomp.sdc.be.resources.data.ESArtifactData;
  36 import org.openecomp.sdc.be.tosca.CsarUtils;
  37 import org.openecomp.sdc.be.tosca.ToscaError;
  38 import org.openecomp.sdc.be.tosca.ToscaExportHandler;
  39 import org.openecomp.sdc.be.tosca.ToscaRepresentation;
  40 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
  41 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
  42 import org.openecomp.sdc.common.api.Constants;
  43 import org.openecomp.sdc.common.log.wrappers.Logger;
  44 import org.openecomp.sdc.common.util.GeneralUtility;
  45 import org.openecomp.sdc.exception.ResponseFormat;
  46 import org.springframework.beans.factory.annotation.Autowired;
  47
  48 import java.io.BufferedWriter;
  49 import java.io.FileOutputStream;
  50 import java.io.IOException;
  51 import java.io.OutputStreamWriter;
  52 import java.io.Writer;
  53 import java.util.ArrayList;
  54 import java.util.Collections;
  55 import java.util.HashMap;
  56 import java.util.HashSet;
  57 import java.util.List;
  58 import java.util.Map;
  59 import java.util.Optional;
  60 import java.util.Set;
  61 import java.util.EnumMap;
  62 import java.util.stream.Collectors;
  63
  64 @org.springframework.stereotype.Component("artifactUuidFix")
  65 public class ArtifactUuidFix {
  66
  67         private static final String MIGRATION1707_ARTIFACT_UUID_FIX = "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ";
  68
  69     private static final String FAILED_TO_FETCH_VF_RESOURCES = "Failed to fetch vf resources ";
  70
  71     private static final String UTF8 = "utf-8";
  72
  73         @Autowired
  74         private JanusGraphDao janusGraphDao;
  75
  76         @Autowired
  77         private ToscaOperationFacade toscaOperationFacade;
  78         @Autowired
  79         private ToscaExportHandler toscaExportUtils;
  80         @Autowired
  81         private ArtifactCassandraDao artifactCassandraDao;
  82
  83
  84         @Autowired
  85         private CsarUtils csarUtils;
  86
  87         private static Logger log = Logger.getLogger(ArtifactUuidFix.class.getName());
  88
  89         public boolean doFix(String fixComponent, String runMode) {
  90                 List<Resource> vfLst = new ArrayList<>();
  91                 List<Service> serviceList = new ArrayList<>();
  92                 Map<String, List<Component>> nodeToFixTosca = new HashMap<>();
  93                 Map<String, List<Component>> vfToFixTosca = new HashMap<>();
  94                 Map<String, List<Component>> serviceToFixTosca = new HashMap<>();
  95
  96                 long time = System.currentTimeMillis();
  97
  98                 doFixTosca(nodeToFixTosca, vfToFixTosca, serviceToFixTosca);
  99
 100                 if ("vf_only".equals(fixComponent)) {
 101                         if (!fetchFaultVf(fixComponent, vfLst, time)) {
 102                                 return false;
 103                         }
 104                 } else {
 105                         if (!fetchServices(fixComponent, serviceList, time)) {
 106                                 return false;
 107                         }
 108                 }
 109                 if ("service_vf".equals(runMode) || "fix".equals(runMode)) {
 110                         log.info("Mode {}. Find problem VFs", runMode);
 111                         if (!fetchVf(serviceList, vfLst, time)) {
 112                                 log.info("Mode {}. Find problem VFs finished with failure", runMode);
 113                                 return false;
 114                         }
 115                         log.info("Mode {}. Find problem VFs finished with success", runMode);
 116                 }
 117                 if ("fix".equals(runMode) || "fix_only_services".equals(runMode)) {
 118                         log.info("Mode {}. Start fix", runMode);
 119                         if (!fix(vfLst, serviceList, nodeToFixTosca, vfToFixTosca, serviceToFixTosca)) {
 120                                 log.info("Mode {}. Fix finished with failure", runMode);
 121                                 return false;
 122                         }
 123                         log.info("Mode {}. Fix finished with success", runMode);
 124                 }
 125
 126                 return true;
 127         }
 128
 129         private boolean fetchFaultVf(String fixComponent, List<Resource> vfLst, long time) {
 130                 log.info("Find fault VF ");
 131                 String fileName = "fault_" + time + ".csv";
 132                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 133                         writer.write("vf name, vf id, state, version\n");
 134
 135                         Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
 136                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
 137                         hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF.name());
 138
 139                         Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
 140                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 141                         log.info("Try to fetch resources with properties {} and not {}", hasProps, hasNotProps);
 142
 143                         Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
 144                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 145                         if (servicesByCriteria.isRight()) {
 146                                 log.info("Failed to fetch resources {}", servicesByCriteria.right().value());
 147                                 return false;
 148                         }
 149                         List<GraphVertex> resources = servicesByCriteria.left().value();
 150                         for (GraphVertex gv : resources) {
 151                                 ComponentParametersView filter = new ComponentParametersView(true);
 152                                 filter.setIgnoreComponentInstances(false);
 153                                 filter.setIgnoreArtifacts(false);
 154                                 filter.setIgnoreGroups(false);
 155
 156                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 157                                                 .getToscaElement(gv.getUniqueId());
 158                                 if (toscaElement.isRight()) {
 159                                         log.info("Failed to fetch resources {} {}", gv.getUniqueId(), toscaElement.right().value());
 160                                         continue;
 161                                 }
 162
 163                                 Resource resource = toscaElement.left().value();
 164                                 String resourceName = resource.getName();
 165                                 Map<String, ArtifactDefinition> deploymentArtifacts = resource.getDeploymentArtifacts();
 166                                 List<GroupDefinition> groups = resource.getGroups();
 167                                 if (groups == null || groups.isEmpty()) {
 168                                         log.info("No groups for resource {} id {} ", resourceName, gv.getUniqueId());
 169                                         continue;
 170                                 }
 171                                 boolean isProblematic = false;
 172                                 for (GroupDefinition gr : groups) {
 173                                         if (gr.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 174                                                 if (isProblematicGroup(gr, resourceName, deploymentArtifacts)) {
 175                                                         isProblematic = true;
 176                                                         break;
 177                                                 }
 178                                         }
 179                                 }
 180                                 if (isProblematic) {
 181                                         vfLst.add(resource);
 182                                         writeModuleResultToFile(writer, resource, null);
 183                                         writer.flush();
 184                                 }
 185                                 janusGraphDao.commit();
 186                         }
 187
 188                 } catch (Exception e) {
 189                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
 190                         return false;
 191                 } finally {
 192                         janusGraphDao.commit();
 193                 }
 194                 return true;
 195         }
 196
 197         private boolean fetchVf(List<Service> serviceList, List<Resource> vfLst, long time) {
 198                 log.info("Find problem VF ");
 199                 if (serviceList.isEmpty()) {
 200                         log.info("No services as input");
 201                         return true;
 202                 }
 203                 String fileName = "problemVf_" + time + ".csv";
 204                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 205                         writer.write("vf name, vf id, state, version, example service name\n");
 206                         Set<String> vfIds = new HashSet<>();
 207                         for (Service service : serviceList) {
 208                                 List<ComponentInstance> componentInstances = service.getComponentInstances().stream()
 209                                                 .filter(ci -> ci.getOriginType().equals(OriginTypeEnum.VF)).collect(Collectors.toList());
 210                                 for (ComponentInstance ci : componentInstances) {
 211                                         if (!vfIds.contains(ci.getComponentUid())) {
 212                                                 vfIds.add(ci.getComponentUid());
 213                                                 ComponentParametersView filter = new ComponentParametersView(true);
 214                                                 filter.setIgnoreComponentInstances(false);
 215                                                 filter.setIgnoreArtifacts(false);
 216                                                 filter.setIgnoreGroups(false);
 217                                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 218                                                                 .getToscaElement(ci.getComponentUid(), filter);
 219                                                 if (toscaElement.isRight()) {
 220                                                         log.info("Failed to fetch resource {} {}", ci.getComponentUid(),
 221                                                                         toscaElement.right().value());
 222                                                         continue;
 223                                                 }
 224                                                 Resource resource = toscaElement.left().value();
 225                                                 if (resource.getResourceType().equals(ResourceTypeEnum.VF)) {
 226                                                         vfLst.add(resource);
 227                                                         writeModuleResultToFile(writer, resource, service);
 228                                                         writer.flush();
 229
 230                                                 }
 231                                                 janusGraphDao.commit();
 232                                         }
 233                                 }
 234                         }
 235                         log.info("output file with list of Vf : {}", fileName);
 236                 } catch (Exception e) {
 237                         log.info("Failed to fetch services ", e);
 238                         return false;
 239                 } finally {
 240                         janusGraphDao.commit();
 241                 }
 242                 return true;
 243         }
 244
 245         private boolean fetchServices(String fixServices, List<Service> serviceList, long time) {
 246                 log.info("Find problem Services {}", fixServices);
 247                 String fileName = "problemService_" + time + ".csv";
 248                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 249                         writer.write("service name, service id, state, version\n");
 250
 251                         Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
 252                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
 253                         if ("distributed_only".equals(fixServices)) {
 254                                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
 255                                 hasProps.put(GraphPropertyEnum.DISTRIBUTION_STATUS, DistributionStatusEnum.DISTRIBUTED.name());
 256                         }
 257
 258                         Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
 259                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 260                         log.info("Try to fetch services with properties {} and not {}", hasProps, hasNotProps);
 261
 262                         Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
 263                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 264                         if (servicesByCriteria.isRight()) {
 265                                 log.info("Failed to fetch services {}", servicesByCriteria.right().value());
 266                                 return false;
 267                         }
 268                         List<GraphVertex> services = servicesByCriteria.left().value();
 269                         for (GraphVertex gv : services) {
 270                                 ComponentParametersView filter = new ComponentParametersView(true);
 271                                 filter.setIgnoreComponentInstances(false);
 272                                 filter.setIgnoreArtifacts(false);
 273                                 filter.setIgnoreGroups(false);
 274
 275                                 Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade
 276                                                 .getToscaElement(gv.getUniqueId(), filter);
 277                                 if (toscaElement.isRight()) {
 278                                         log.info("Failed to fetch service {} {}", gv.getUniqueId(), toscaElement.right().value());
 279                                         continue;
 280                                 }
 281                                 Service service = toscaElement.left().value();
 282
 283                                 String serviceName = (String) gv.getMetadataProperty(GraphPropertyEnum.NAME);
 284
 285                                 boolean isProblematic = isProblematicService(service, serviceName);
 286                                 if (isProblematic) {
 287                                         serviceList.add(service);
 288                                         writeModuleResultToFile(writer, service, null);
 289                                         writer.flush();
 290
 291                                 }
 292
 293                                 janusGraphDao.commit();
 294                         }
 295                         log.info("output file with list of services : {}", fileName);
 296                 } catch (Exception e) {
 297                         log.info("Failed to fetch services ", e);
 298                         return false;
 299                 } finally {
 300                         janusGraphDao.commit();
 301                 }
 302                 return true;
 303         }
 304
 305         private boolean isProblematicService( Service service, String serviceName) {
 306
 307                 List<ComponentInstance> componentInstances = service.getComponentInstances();
 308
 309                 if (componentInstances == null) {
 310                         log.info("No instances for service {} ", service.getUniqueId());
 311                         return false;
 312                 }
 313                 boolean isCheckVFModules = true;
 314                 if(service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN ||
 315                                 service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT){
 316                         isCheckVFModules = false;
 317                 }
 318                 for (ComponentInstance ci : componentInstances) {
 319                         Map<String, ArtifactDefinition> deploymentArtifacts = ci.getDeploymentArtifacts();
 320                         List<GroupInstance> groupInstances = ci.getGroupInstances();
 321                         if (groupInstances == null || groupInstances.isEmpty()) {
 322                                 log.info("No instance groups for instance {} in service {} id {} ", ci.getName(), serviceName,
 323                                                 service.getUniqueId());
 324                                 continue;
 325                         }
 326                         List<VfModuleArtifactPayloadEx> vfModules = null;
 327                         if(isCheckVFModules){
 328                                 Optional<ArtifactDefinition> optionalVfModuleArtifact = deploymentArtifacts.values().stream()
 329                                                 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 330
 331                                  if(!optionalVfModuleArtifact.isPresent())
 332                                          return true;
 333
 334                                  ArtifactDefinition vfModuleArtifact = optionalVfModuleArtifact.get();
 335                                  Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> vfModulesEither = parseVFModuleJson(vfModuleArtifact);
 336                                  if(vfModulesEither.isRight()){
 337                                          log.error("Failed to parse vfModule for service {} status is {}", service.getUniqueId(), vfModulesEither.right().value());
 338                                          return true;
 339                                  }
 340                                  vfModules = vfModulesEither.left().value();
 341                                  if(vfModules == null || vfModules.isEmpty() ){
 342                                          log.info("vfModules empty for service {}", service.getUniqueId());
 343                                          return true;
 344                                  }
 345                         }
 346
 347                         for (GroupInstance gi : groupInstances) {
 348                                 if (gi.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 349                                         VfModuleArtifactPayloadEx vfModule= null;
 350                                         if(isCheckVFModules && vfModules != null){
 351                                                 Optional<VfModuleArtifactPayloadEx> op = vfModules.stream().filter(vf -> vf.getVfModuleModelName().equals(gi.getGroupName())).findAny();
 352                                                 if(!op.isPresent()){
 353                                                          log.error("Failed to find vfModule for group {}", gi.getGroupName());
 354                                                         return true;
 355                                                 }
 356                                                 vfModule = op.get();
 357                                         }
 358                                         if (isProblematicGroupInstance(gi, ci.getName(), serviceName, deploymentArtifacts, vfModule)) {
 359                                                 return true;
 360                                         }
 361                                 }
 362                         }
 363
 364                 }
 365                 return false;
 366         }
 367
 368         private boolean isProblematicGroup(GroupDefinition gr, String resourceName,
 369                         Map<String, ArtifactDefinition> deploymentArtifacts) {
 370                 List<String> artifacts = gr.getArtifacts();
 371                 List<String> artifactsUuid = gr.getArtifactsUuid();
 372                 Set<String> artifactsSet = new HashSet<>();
 373
 374                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 375                         log.info("No groups in resource {} ", resourceName);
 376                         return true;
 377                 }
 378                 artifactsSet.addAll(artifacts);
 379                 if (artifactsSet.size() < artifacts.size()) {
 380                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", gr.getName(), resourceName);
 381                         return true;
 382                 }
 383
 384                 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
 385                         log.info(" artifacts.size() < artifactsUuid.size() group {} in resource {} ", gr.getName(), resourceName);
 386                         return true;
 387                 }
 388                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 389                         log.info(
 390                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() group {} in resource {} ",
 391                                         gr.getName(), resourceName);
 392                         return true;
 393                 }
 394                 if (artifactsUuid != null && artifactsUuid.contains(null)) {
 395                         log.info(" artifactsUuid.contains(null) group {} in resource {} ", gr.getName(), resourceName);
 396                         return true;
 397                 }
 398
 399                 for (String artifactId : artifacts) {
 400                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 401                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 402                         if (artifactDefinition == null) {
 403                                 log.info(" artifactDefinition == null label {} group {} in resource {} ", artifactlabel, gr.getName(),
 404                                                 resourceName);
 405                                 return true;
 406                         }
 407                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 408                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 409                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 410                                         log.info(
 411                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} group {} in resource {} ",
 412                                                         artifactlabel, artifactId, gr.getName(), resourceName);
 413                                         return true;
 414                                 }
 415                                 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 416                                         log.info(
 417                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} group {} in resource {} ",
 418                                                         artifactlabel, gr.getName(), resourceName);
 419                                         return true;
 420                                 }
 421                         }
 422                 }
 423                 for (String artifactUUID : artifactsUuid) {
 424                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 425                         if (label != null && !label.isEmpty() && !label.equals("")) {
 426                                 return true;
 427                         }
 428                 }
 429
 430                 return false;
 431         }
 432
 433         private boolean isProblematicGroupInstance(GroupInstance gi, String instName, String servicename,
 434                         Map<String, ArtifactDefinition> deploymentArtifacts, VfModuleArtifactPayloadEx vfModule) {
 435                 List<String> artifacts = gi.getArtifacts();
 436                 List<String> artifactsUuid = gi.getArtifactsUuid();
 437                 List<String> instArtifactsUuid = gi.getGroupInstanceArtifactsUuid();
 438                 List<String> instArtifactsId = gi.getGroupInstanceArtifacts();
 439                 Set<String> instArtifatIdSet = new HashSet<>();
 440                 Set<String> artifactsSet = new HashSet<>();
 441
 442                 log.info("check group {} for instance {} ", gi.getGroupName(), instName);
 443                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 444                         log.info("No instance groups for instance {} in service {} ", instName, servicename);
 445                         return true;
 446                 }
 447                 artifactsSet.addAll(artifacts);
 448                 if (artifactsSet.size() < artifacts.size()) {
 449                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", instName, servicename);
 450                         return true;
 451                 }
 452
 453                 if (instArtifactsId != null && !instArtifactsId.isEmpty()) {
 454                         instArtifatIdSet.addAll(instArtifactsId);
 455                 }
 456
 457                 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
 458                         log.info(" artifacts.size() < artifactsUuid.size() inst {} in service {} ", instName, servicename);
 459                         return true;
 460                 }
 461                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 462                         log.info(
 463                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() inst {} in service {} ",
 464                                         instName, servicename);
 465                         return true;
 466                 }
 467                 if (artifactsUuid != null && artifactsUuid.contains(null)) {
 468                         log.info(" artifactsUuid.contains(null) inst {} in service {} ", instName, servicename);
 469                         return true;
 470                 }
 471                 if (instArtifactsId != null && instArtifatIdSet.size() < instArtifactsId.size()) {
 472                         log.info(" instArtifatIdSet.size() < instArtifactsId.size() inst {} in service {} ", instName, servicename);
 473                         return true;
 474                 }
 475
 476                 if ((instArtifactsId != null && instArtifactsUuid != null)
 477                                 && instArtifactsId.size() != instArtifactsUuid.size()) {
 478                         log.info(" instArtifactsId.size() != instArtifactsUuid.size() inst {} in service {} ", instName,
 479                                         servicename);
 480                         return true;
 481                 }
 482
 483                 for (String artifactId : artifacts) {
 484                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 485                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 486                         if (artifactDefinition == null) {
 487                                 log.info(" artifactDefinition == null label {} inst {} in service {} ", artifactlabel, instName,
 488                                                 servicename);
 489                                 return true;
 490                         }
 491                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 492                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 493                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 494                                         log.info(
 495                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} inst {} in service {} ",
 496                                                         artifactlabel, artifactId, instName, servicename);
 497                                         return true;
 498                                 }
 499                                 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 500                                         log.info(
 501                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 502                                                         artifactlabel, instName, servicename);
 503                                         return true;
 504                                 }
 505                         } else {
 506                                 if (instArtifactsUuid == null || instArtifactsUuid.isEmpty()) {
 507                                         log.info(" instArtifactsUuid empty. label {} inst {} in service {} ", artifactlabel, instName,
 508                                                         servicename);
 509                                         return true;
 510                                 }
 511                                 if (!instArtifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 512                                         log.info(
 513                                                         " instArtifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 514                                                         artifactlabel, instName, servicename);
 515                                         return true;
 516                                 }
 517                         }
 518                 }
 519                 for (String artifactUUID : artifactsUuid) {
 520                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 521                         if (label != null && !label.isEmpty() && !"".equals(label)) {
 522                                 return true;
 523                         }
 524                 }
 525                 if(vfModule != null && artifactsUuid != null){
 526                         return isProblematicVFModule(vfModule, artifactsUuid, instArtifactsUuid);
 527                 }
 528
 529                 return false;
 530         }
 531
 532         private boolean isProblematicVFModule(VfModuleArtifactPayloadEx vfModule, List<String> artifactsUuid,
 533                         List<String> instArtifactsUuid) {
 534                 log.info(" isProblematicVFModule  {}  ", vfModule.getVfModuleModelName());
 535                 List<String> vfModuleArtifacts = vfModule.getArtifacts();
 536                 List<String> allArtifacts = new ArrayList<>();
 537                 allArtifacts.addAll(artifactsUuid);
 538                 if(instArtifactsUuid != null)
 539                         allArtifacts.addAll(instArtifactsUuid);
 540                 if((vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()){
 541                         log.error(" vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()");
 542                         return true;
 543                 }
 544                 if(vfModuleArtifacts!= null){
 545                         if( vfModuleArtifacts.size() != allArtifacts.size()){
 546                                 log.error(" vfModuleArtifacts.size() != allArtifacts.size()");
 547                                 return true;
 548                         }
 549                         for(String vfModuleArtifact: vfModuleArtifacts){
 550                                 Optional<String> op = allArtifacts.stream().filter(a -> a.equals(vfModuleArtifact)).findAny();
 551                                 if(!op.isPresent()){
 552                                         log.error("failed to find artifact {} in group artifacts {}", vfModuleArtifact,  allArtifacts);
 553                                         return true;
 554                                 }
 555                         }
 556                 }
 557                 return false;
 558         }
 559
 560
 561
 562         private boolean fix(List<Resource> vfLst, List<Service> serviceList, Map<String, List<Component>> nodesToFixTosca,
 563                         Map<String, List<Component>> vfToFixTosca, Map<String, List<Component>> servicesToFixTosca) {
 564                 boolean res = true;
 565                 log.info(" Fix started ***** ");
 566                 if (vfLst != null && !vfLst.isEmpty()) {
 567                         res = fixVf(vfLst);
 568
 569                 }
 570
 571                 if (res && serviceList != null && !serviceList.isEmpty()) {
 572                         res = fixServices(serviceList);
 573
 574                 }
 575
 576                 Set<String> fixedIds = new HashSet<>();
 577
 578                 long time = System.currentTimeMillis();
 579                 String fileName = "FailedGenerateTosca" + "_" + time + ".csv";
 580                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
 581                         writer.write("componentType, name, version, UID, UUID, invariantUUID, state\n");
 582                         List<Component> failedList = new ArrayList<>();
 583
 584                         if (res && nodesToFixTosca != null && !nodesToFixTosca.isEmpty()) {
 585
 586                                 generateAndSaveToscaArtifacts(nodesToFixTosca, fixedIds, null, failedList);
 587
 588                         }
 589                         if (vfToFixTosca != null && !vfToFixTosca.isEmpty()) {
 590
 591                                 generateAndSaveToscaArtifacts(vfToFixTosca, fixedIds, vfLst, failedList);
 592
 593                         }
 594
 595                         for (Component component : vfLst) {
 596                                 res = generateToscaPerComponent(fixedIds, component);
 597                                 if (res) {
 598                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 599                                         Map<String, GroupDataDefinition> groups = topologyTemplate.getGroups();
 600                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.GROUPS, EdgeLabelEnum.GROUPS, groups);
 601                                         if (res) {
 602                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getDeploymentArtifacts();
 603                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.DEPLOYMENT_ARTIFACTS,
 604                                                                 EdgeLabelEnum.DEPLOYMENT_ARTIFACTS, arifacts);
 605                                         }
 606                                         if (res) {
 607                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 608                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 609                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 610                                         }
 611                                         janusGraphDao.commit();
 612                                 } else {
 613                                         failedList.add(component);
 614                                 }
 615                         }
 616
 617                         if (servicesToFixTosca != null && !servicesToFixTosca.isEmpty()) {
 618                                 generateAndSaveToscaArtifacts(servicesToFixTosca, fixedIds, serviceList, failedList);
 619
 620                         }
 621
 622
 623                         for (Component component : serviceList) {
 624                                 res = generateToscaPerComponent(fixedIds, component);
 625                                 if (res) {
 626                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 627                                         Map<String, MapGroupsDataDefinition> groups = topologyTemplate.getInstGroups();
 628                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_GROUPS, EdgeLabelEnum.INST_GROUPS,
 629                                                         groups);
 630
 631                                         if (res) {
 632                                                 Map<String, MapArtifactDataDefinition> artifacts = topologyTemplate
 633                                                                 .getInstDeploymentArtifacts();
 634                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_DEPLOYMENT_ARTIFACTS,
 635                                                                 EdgeLabelEnum.INST_DEPLOYMENT_ARTIFACTS, artifacts);
 636                                         }
 637                                         if (res) {
 638                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 639                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 640                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 641                                         }
 642                                         janusGraphDao.commit();
 643                                 } else {
 644                                         failedList.add(component);
 645                                 }
 646
 647                         }
 648                         if (!failedList.isEmpty()) {
 649                                 for (Component component : failedList) {
 650                                         StringBuilder sb = new StringBuilder(component.getComponentType().getValue());
 651                                         sb.append(",").append(component.getName()).append(",").append(component.getVersion()).append(",")
 652                                                         .append(component.getUniqueId()).append(",").append(component.getUUID()).append(",")
 653                                                         .append(component.getInvariantUUID()).append(",").append(component.getLifecycleState());
 654
 655                                         sb.append("\n");
 656                                         writer.write(sb.toString());
 657                                 }
 658                                 writer.flush();
 659                         }
 660                 } catch (IOException e) {
 661                     log.error(e.getMessage());
 662                 } finally {
 663                         janusGraphDao.commit();
 664                 }
 665                 log.info(" Fix finished with res {} ***** ", res);
 666                 return res;
 667         }
 668
 669         private boolean generateAndSaveToscaArtifacts(Map<String, List<Component>> nodesToFixTosca, Set<String> fixedIds,
 670                         List<? extends Component> componentsWithFailedGroups, List<Component> failedList) {
 671                 boolean res = true;
 672                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts started ");
 673                 for (Map.Entry<String, List<Component>> entry : nodesToFixTosca.entrySet()) {
 674
 675                         List<Component> component = entry.getValue();
 676                         for (Component c : component) {
 677                                 log.debug("Migration1707ArtifactUuidFix  fix tosca on component : id {},  name {} ", c.getUniqueId(),
 678                                                 c.getName());
 679                                 if (componentsWithFailedGroups != null) {
 680                                         Optional<Component> op = (Optional<Component>) componentsWithFailedGroups.stream()
 681                                                         .filter(cg -> cg.getUniqueId().equals(c.getUniqueId())).findAny();
 682                                         if (!op.isPresent())
 683                                                 res = generateToscaPerComponent(fixedIds, c);
 684                                 } else
 685                                         res = generateToscaPerComponent(fixedIds, c);
 686                                 if (res) {
 687                                         ToscaElement topologyTemplate = ModelConverter.convertToToscaElement(c);
 688                                         Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 689                                         res = fixDataOnGraph(c.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS, EdgeLabelEnum.TOSCA_ARTIFACTS,
 690                                                         arifacts);
 691                                         janusGraphDao.commit();
 692                                 } else {
 693                                         failedList.add(c);
 694                                 }
 695
 696                         }
 697                 }
 698                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts finished with res {} ", res);
 699                 return res;
 700         }
 701
 702         private boolean generateToscaPerComponent(Set<String> fixedIds, Component c) {
 703                 boolean res = true;
 704                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent started component name {} id {}",
 705                                 c.getName(), c.getUniqueId());
 706                 try {
 707                         Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
 708                                         .getToscaFullElement(c.getUniqueId());
 709                         if (toscaElement.isRight()) {
 710                                 log.info("Failed to fetch resources {} {}", c.getUniqueId(), toscaElement.right().value());
 711                                 return false;
 712                         }
 713                         Component toscaElementFull = toscaElement.left().value();
 714                         toscaElementFull.setGroups(c.getGroups());
 715                         List<ComponentInstance> ciListFull = toscaElementFull.getComponentInstances();
 716                         List<ComponentInstance> ciList = c.getComponentInstances();
 717                         if (ciListFull != null && !ciListFull.isEmpty()) {
 718                                 ciListFull.forEach(ciFull -> {
 719                                         ComponentInstance compInst = ciList.stream()
 720                                                         .filter(ci -> ci.getUniqueId().equals(ciFull.getUniqueId())).findAny().get();
 721                                         ciFull.setGroupInstances(compInst.getGroupInstances());
 722                                 });
 723                         }
 724
 725                         Either<Component, ToscaError> either = generateToscaArtifact(toscaElementFull);
 726
 727                         if (either.isRight()) {
 728                                 log.error("Couldn't generate and save tosca template component  unique id {}, name {} error: {}",
 729                                                 toscaElementFull.getUniqueId(), toscaElementFull.getName(), either.right().value());
 730                                 res = false;
 731
 732                         }
 733
 734                         if (res) {
 735                                 c.setToscaArtifacts(either.left().value().getToscaArtifacts());
 736                                 fixedIds.add(toscaElementFull.getUniqueId());
 737                         }
 738                 } finally {
 739                         if (res)
 740                                 janusGraphDao.commit();
 741                         else
 742                                 janusGraphDao.rollback();
 743                 }
 744                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent finished  component name {} id {} res {}",
 745                                 c.getName(), c.getUniqueId(), res);
 746                 return res;
 747         }
 748
 749         private <T extends ToscaDataDefinition> boolean fixDataOnGraph(String componentId, VertexTypeEnum vertexTypeEnum,
 750                         EdgeLabelEnum edgeLabelEnum, Map<String, T> groups) {
 751                 log.debug("amount groups to update: VertexTypeEnum {} EdgeLabelEnum {} data size {}", vertexTypeEnum.getName(),
 752                                 edgeLabelEnum, groups.size());
 753                 boolean res = true;
 754                 Either<GraphVertex, JanusGraphOperationStatus> getResponse = janusGraphDao.getVertexById(componentId,
 755                                 JsonParseFlagEnum.NoParse);
 756                 if (getResponse.isRight()) {
 757                         log.debug("Couldn't fetch component  unique id {}, error: {}", componentId, getResponse.right().value());
 758                         res = false;
 759
 760                 }
 761                 if (res) {
 762                         GraphVertex componentVertex = getResponse.left().value();
 763
 764                         GraphVertex toscaDataVertex = null;
 765                         Either<GraphVertex, JanusGraphOperationStatus> groupVertexEither = janusGraphDao.getChildVertex(componentVertex,
 766                                         edgeLabelEnum, JsonParseFlagEnum.ParseJson);
 767                         if (groupVertexEither.isRight() && groupVertexEither.right().value() == JanusGraphOperationStatus.NOT_FOUND) {
 768                                 log.debug("no child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum, componentId,
 769                                                 groupVertexEither.right().value());
 770                                 return true;
 771                         }
 772                         if (groupVertexEither.isRight()) {
 773                                 res = false;
 774                                 log.debug("failed to get child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum,
 775                                                 componentId, groupVertexEither.right().value());
 776                         }
 777                         if (res) {
 778                                 toscaDataVertex = groupVertexEither.left().value();
 779                                 toscaDataVertex.setJson(groups);
 780                                 Either<GraphVertex, JanusGraphOperationStatus> updatevertexEither = janusGraphDao.updateVertex(toscaDataVertex);
 781                                 if (updatevertexEither.isRight()) {
 782                                         log.debug("failed to update vertex for component  unique id {}, error: {}", componentId,
 783                                                         updatevertexEither.right().value());
 784                                         janusGraphDao.rollback();
 785                                         return false;
 786                                 }
 787                         }
 788                 }
 789                 log.debug("Fix data on graph finished: VertexTypeEnum {} EdgeLabelEnum {} res {}", vertexTypeEnum.getName(),
 790                                 res);
 791                 return res;
 792         }
 793
 794         private boolean fixServices(List<Service> serviceList) {
 795                 for (Service service : serviceList) {
 796                         log.debug("Migration1707ArtifactUuidFix  fix service: id {},  name {} ", service.getUniqueId(),
 797                                         service.getName());
 798                         List<ComponentInstance> instances = service.getComponentInstances();
 799                         for (ComponentInstance instance : instances) {
 800                                 fixComponentInstances(service, instance);
 801                         }
 802
 803                 }
 804                 return true;
 805
 806         }
 807
 808         private void fixComponentInstances(Service service, ComponentInstance instance) {
 809                 Map<String, ArtifactDefinition> artifactsMap = instance.getDeploymentArtifacts();
 810                 List<GroupInstance> groupsList = instance.getGroupInstances();
 811                 if (groupsList != null && artifactsMap != null) {
 812                         List<GroupInstance> groupsToDelete = new ArrayList<>();
 813                         for (GroupInstance group : groupsList) {
 814                                 fixGroupInstances(service, artifactsMap, groupsToDelete, group);
 815
 816                         }
 817
 818                         if (!groupsToDelete.isEmpty()) {
 819                                 log.debug("Migration1707ArtifactUuidFix  delete group:  resource id {}, group instance to delete {} ",
 820                                                 service.getUniqueId(), groupsToDelete);
 821                                 groupsList.removeAll(groupsToDelete);
 822
 823                         }
 824
 825                         Optional<ArtifactDefinition> optionalVfModuleArtifact = artifactsMap.values().stream()
 826                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 827                         ArtifactDefinition vfModuleArtifact;
 828                         if(!optionalVfModuleArtifact.isPresent()){
 829                                 vfModuleArtifact = createVfModuleArtifact(instance, service);
 830                                 artifactsMap.put(vfModuleArtifact.getArtifactLabel(), vfModuleArtifact);
 831                         }
 832                         else {
 833                                 vfModuleArtifact = optionalVfModuleArtifact.get();
 834                         }
 835                         fillVfModuleInstHeatEnvPayload(service, instance, groupsList, vfModuleArtifact);
 836                 }
 837         }
 838
 839         private void fixGroupInstances(Service service, Map<String, ArtifactDefinition> artifactsMap,
 840                         List<GroupInstance> groupsToDelete, GroupInstance group) {
 841                 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 842                         log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", service.getUniqueId(),
 843                                         group.getName());
 844                         if(group.getArtifacts() != null){
 845                                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 846                                 if(group.getGroupInstanceArtifacts() != null){
 847                                         List<String> groupInsArtifacts = new ArrayList<>(group.getGroupInstanceArtifacts());
 848                                         groupArtifactsSet.addAll(groupInsArtifacts);
 849                                 }
 850                                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 851
 852                                 clearGroupInstanceArtifacts(group);
 853
 854                                 for (String artifactId : groupArtifacts) {
 855                                         fixArtifactUndergroupInstances(artifactsMap, group, groupArtifacts, artifactId);
 856                                 }
 857                         }
 858                         if (group.getArtifacts() == null || group.getArtifacts().isEmpty()) {
 859                                 log.debug(
 860                                                 "Migration1707ArtifactUuidFix  fix groupInstance add to delete list:  resource id {} name {} , group name {} ",
 861                                                 service.getUniqueId(), service.getName(), group.getName());
 862                                 groupsToDelete.add(group);
 863                         }
 864                 }
 865         }
 866
 867         private void clearGroupInstanceArtifacts(GroupInstance group) {
 868                 if(group.getArtifacts() != null)
 869                         group.getArtifacts().clear();
 870                 else
 871                         group.setArtifacts(new ArrayList<>());
 872                 if(group.getArtifactsUuid() != null)
 873                         group.getArtifactsUuid().clear();
 874                 else{
 875                         group.setArtifactsUuid(new ArrayList<>());
 876                 }
 877                 if(group.getGroupInstanceArtifacts() != null)
 878                         group.getGroupInstanceArtifacts().clear();
 879                 else{
 880                         group.setGroupInstanceArtifacts(new ArrayList<>());
 881                 }
 882                 if(group.getGroupInstanceArtifactsUuid() != null )
 883                         group.getGroupInstanceArtifactsUuid().clear();
 884                 else
 885                         group.setGroupInstanceArtifactsUuid(new ArrayList<>());
 886         }
 887
 888         private void fixArtifactUndergroupInstances(Map<String, ArtifactDefinition> artifactsMap, GroupInstance group,
 889                         List<String> groupArtifacts, String artifactId) {
 890                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 891                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 892                                 group.getName(), artifactId, artifactlabel);
 893                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 894                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 895                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
 896                         String correctArtifactId = artifact.getUniqueId();
 897                         String correctArtifactUUID = artifact.getArtifactUUID();
 898                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 899                                 boolean isAddToGroup = true;
 900                                 if (groupArtifacts.size() == 1) {
 901                                         if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 902                                                 isAddToGroup = false;
 903                                                 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 904                                         }
 905                                 }
 906                                 if (isAddToGroup) {
 907                                         log.debug(
 908                                                         MIGRATION1707_ARTIFACT_UUID_FIX,
 909                                                         group.getName(), correctArtifactId, correctArtifactUUID);
 910                                         group.getArtifacts().add(correctArtifactId);
 911                                         if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 912                                                 group.getArtifactsUuid().add(correctArtifactUUID);
 913                                         }
 914                                 }
 915                         } else {
 916                                 log.debug(
 917                                                 MIGRATION1707_ARTIFACT_UUID_FIX,
 918                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 919                                 Set<String> tmpSet = new HashSet<>(group.getGroupInstanceArtifacts());
 920                                 tmpSet.add(correctArtifactId);
 921                                 group.setGroupInstanceArtifacts(new ArrayList<>(tmpSet));
 922                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 923                                         Set<String> tmpSetUUID = new HashSet<>(group.getGroupInstanceArtifactsUuid());
 924                                         tmpSetUUID.add(correctArtifactUUID);
 925                                         group.setGroupInstanceArtifactsUuid(new ArrayList<>(tmpSetUUID));
 926                                 }
 927                         }
 928                 }
 929         }
 930
 931         private boolean fixVf(List<Resource> vfLst) {
 932                 for (Resource resource : vfLst) {
 933                         log.debug("Migration1707ArtifactUuidFix  fix resource: id {},  name {} ", resource.getUniqueId(),
 934                                         resource.getName());
 935                         Map<String, ArtifactDefinition> artifactsMap = resource.getDeploymentArtifacts();
 936                         List<GroupDefinition> groupsList = resource.getGroups();
 937                         List<GroupDefinition> groupsToDelete = new ArrayList<>();
 938                         if (groupsList != null && artifactsMap != null) {
 939                                 for (GroupDefinition group : groupsList) {
 940                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE) && group.getArtifacts() != null) {
 941                                                 fixVfGroup(resource, artifactsMap, group);
 942                                         }
 943                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)
 944                                                         && (group.getArtifacts() == null || group.getArtifacts().isEmpty())) {
 945                                                 log.debug(
 946                                                                 "Migration1707ArtifactUuidFix  add group to delete list fix resource: id {},  name {} ",
 947                                                                 resource.getUniqueId(), resource.getName(), group.getName());
 948                                                 groupsToDelete.add(group);
 949                                         }
 950                                 }
 951
 952                                 if (!groupsToDelete.isEmpty()) {
 953                                         groupsList.removeAll(groupsToDelete);
 954
 955                                 }
 956                         }
 957
 958                 }
 959
 960                 return true;
 961         }
 962
 963         private void fixVfGroup(Resource resource, Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group) {
 964                 log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", resource.getUniqueId(),
 965                                 group.getName());
 966                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 967                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 968                 group.getArtifacts().clear();
 969                 group.getArtifactsUuid().clear();
 970
 971                 for (String artifactId : groupArtifacts) {
 972                         fixArtifactUnderGroup(artifactsMap, group, groupArtifacts, artifactId);
 973                 }
 974         }
 975
 976         private void fixArtifactUnderGroup(Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group,
 977                         List<String> groupArtifacts, String artifactId) {
 978
 979                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 980                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 981                                 group.getName(), artifactId, artifactlabel);
 982                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 983                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 984                         String correctArtifactId = artifact.getUniqueId();
 985                         String correctArtifactUUID = artifact.getArtifactUUID();
 986                         boolean isAddToGroup = true;
 987                         if (groupArtifacts.size() == 1) {
 988                                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
 989                                 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 990                                         isAddToGroup = false;
 991                                         artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 992                                 }
 993                         }
 994                         if (isAddToGroup) {
 995                                 log.debug(
 996                                                 MIGRATION1707_ARTIFACT_UUID_FIX,
 997                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 998                                 group.getArtifacts().add(correctArtifactId);
 999                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
1000                                         group.getArtifactsUuid().add(correctArtifactUUID);
1001                                 }
1002                         }
1003
1004                 }
1005         }
1006
1007         private String findArtifactLabelFromArtifactId(String artifactId) {
1008                 String artifactLabel = "";
1009
1010                 int index = artifactId.lastIndexOf('.');
1011                 if (index > 0 && index + 1 < artifactId.length())
1012                         artifactLabel = artifactId.substring(index + 1);
1013                 return artifactLabel;
1014         }
1015
1016         private void writeModuleResultToFile(Writer writer, org.openecomp.sdc.be.model.Component component,
1017                         Service service) {
1018                 try {
1019                         // "service name, service id, state, version
1020                         StringBuilder sb = new StringBuilder(component.getName());
1021                         sb.append(",").append(component.getUniqueId()).append(",").append(component.getLifecycleState()).append(",")
1022                                         .append(component.getVersion());
1023                         if (service != null) {
1024                                 sb.append(",").append(service.getName());
1025                         }
1026                         sb.append("\n");
1027                         writer.write(sb.toString());
1028                 } catch (IOException e) {
1029                         log.error(e.getMessage());
1030                 }
1031         }
1032
1033         private void writeModuleResultToFile(Writer writer, List<Component> components) {
1034                 try {
1035                         // "service name, service id, state, version
1036                         for (Component component : components) {
1037                                 StringBuilder sb = new StringBuilder(component.getName());
1038                                 sb.append(",").append(component.getUniqueId()).append(",").append(component.getInvariantUUID())
1039                                                 .append(",").append(component.getLifecycleState()).append(",").append(component.getVersion());
1040
1041                                 sb.append("\n");
1042                                 writer.write(sb.toString());
1043                         }
1044                 } catch (IOException e) {
1045
1046                     log.error(e.getMessage());
1047                 }
1048         }
1049
1050         public boolean doFixTosca(Map<String, List<Component>> nodeToFix, Map<String, List<Component>> vfToFix,
1051                         Map<String, List<Component>> serviceToFix) {
1052
1053                 Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
1054                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1055                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1056
1057                 Map<String, List<Component>> vertices = getVerticesToValidate(VertexTypeEnum.NODE_TYPE, hasProps);
1058                 boolean result = validateTosca(vertices, nodeToFix, "RESOURCE_TOSCA_ARTIFACTS");//
1059
1060                 hasProps.clear();
1061                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1062                 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF);
1063                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1064
1065                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1066                 result = validateTosca(vertices, vfToFix, "VF_TOSCA_ARTIFACTS");
1067
1068                 hasProps.clear();
1069                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
1070                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1071
1072                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1073                 result = validateTosca(vertices, serviceToFix, "SERVICE_TOSCA_ARTIFACTS");
1074
1075                 return result;
1076         }
1077
1078         public Map<String, List<Component>> getVerticesToValidate(VertexTypeEnum type,
1079                         Map<GraphPropertyEnum, Object> hasProps) {
1080
1081                 Map<String, List<Component>> result = new HashMap<>();
1082                 try {
1083
1084                         Either<List<GraphVertex>, JanusGraphOperationStatus> resultsEither = janusGraphDao
1085           .getByCriteria(type, hasProps);
1086                         if (resultsEither.isRight()) {
1087                                 log.error("getVerticesToValidate failed {} ",resultsEither.right().value());
1088                                 return result;
1089                         }
1090                         log.info("getVerticesToValidate: {}  vertices to scan", resultsEither.left().value().size());
1091                         List<GraphVertex> componentsList = resultsEither.left().value();
1092                         componentsList.forEach(vertex -> {
1093                                 String ivariantUuid = (String) vertex.getMetadataProperty(GraphPropertyEnum.INVARIANT_UUID);
1094                                 if (!result.containsKey(ivariantUuid)) {
1095                                         List<Component> compList = new ArrayList<>();
1096                                         result.put(ivariantUuid, compList);
1097                                 }
1098                                 List<Component> compList = result.get(ivariantUuid);
1099
1100                                 ComponentParametersView filter = new ComponentParametersView(true);
1101                                 filter.setIgnoreArtifacts(false);
1102
1103                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1104                                                 .getToscaElement(vertex.getUniqueId(), filter);
1105                                 if (toscaElement.isRight()) {
1106                                         log.error("getVerticesToValidate: failed to find element {}  staus is {}", vertex.getUniqueId()
1107                                                         ,toscaElement.right().value());
1108                                 } else {
1109                                         compList.add(toscaElement.left().value());
1110                                 }
1111                                 janusGraphDao.commit();
1112
1113                         });
1114
1115                 } catch (Exception e) {
1116                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1117
1118                 } finally {
1119                         janusGraphDao.commit();
1120
1121                 }
1122                 return result;
1123         }
1124
1125         public boolean validateTosca(Map<String, List<Component>> vertices, Map<String, List<Component>> compToFix,
1126                         String name) {
1127                 boolean result = true;
1128                 long time = System.currentTimeMillis();
1129                 String fileName = name + "_" + time + ".csv";
1130                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
1131                         writer.write("name, UUID, invariantUUID, state, version\n");
1132                         for (Map.Entry<String, List<Component>> entry : vertices.entrySet()) {
1133                                 List<Component> compList = entry.getValue();
1134                                 Set<String> artifactEsId = new HashSet<>();
1135                                 for (Component component : compList) {
1136                                         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
1137                                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1138                                                         .filter(a -> artifactEsId.contains(a.getEsId()) && a.getEsId() != null).findAny();
1139                                         if (op.isPresent()) {
1140                                                 result = false;
1141                                                 writeModuleResultToFile(writer, compList);
1142                                                 writer.flush();
1143                                                 break;
1144                                         } else {
1145                                                 artifactEsId.addAll(toscaArtifacts.values().stream().map(ArtifactDefinition::getEsId)
1146                                                                 .collect(Collectors.toList()));
1147                                         }
1148                                 }
1149                                 if (!result) {
1150                                         List<Component> compListfull = new ArrayList<>();
1151                                         for (Component c : compList) {
1152                                                 ComponentParametersView filter = new ComponentParametersView(true);
1153                                                 filter.setIgnoreComponentInstances(false);
1154                                                 filter.setIgnoreArtifacts(false);
1155                                                 filter.setIgnoreGroups(false);
1156
1157                                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1158                                                                 .getToscaElement(c.getUniqueId(), filter);
1159                                                 if (toscaElement.isRight()) {
1160                                                         log.debug("getVerticesToValidate: failed to find element" + c.getUniqueId()
1161                                                                         + " status is" + toscaElement.right().value());
1162                                                 } else {
1163                                                         compListfull.add(toscaElement.left().value());
1164                                                 }
1165                                                 this.janusGraphDao.commit();
1166                                         }
1167
1168                                         compToFix.put(entry.getKey(), compListfull);
1169                                         result = true;
1170                                 }
1171
1172                         }
1173
1174                 } catch (Exception e) {
1175                         log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1176                         return false;
1177                 } finally {
1178                         janusGraphDao.commit();
1179                 }
1180                 return result;
1181         }
1182
1183         private Either<Component, ToscaError> generateToscaArtifact(Component parent) {
1184                 log.debug("tosca artifact generation");
1185                 try {
1186                         Map<String, ArtifactDefinition> toscaArtifacts = parent.getToscaArtifacts();
1187
1188                         ArtifactDefinition toscaArtifact = null;
1189                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1190                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_TEMPLATE.getType())).findAny();
1191
1192                         if (op.isPresent()) {
1193                                 toscaArtifact = op.get();
1194                         }
1195                         if (toscaArtifact != null) {
1196                                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent artifact name {} id {} esId {}",
1197                                                 toscaArtifact.getArtifactName(), toscaArtifact.getUniqueId(), toscaArtifact.getEsId());
1198
1199                                 Either<ToscaRepresentation, ToscaError> exportComponent = toscaExportUtils.exportComponent(parent);
1200                                 if (exportComponent.isRight()) {
1201                                         log.debug("Failed export tosca yaml for component {} error {}", parent.getUniqueId(),
1202                                                         exportComponent.right().value());
1203
1204                                         return Either.right(exportComponent.right().value());
1205                                 }
1206                                 log.debug("Tosca yaml exported for component {} ", parent.getUniqueId());
1207                                 String payload = exportComponent.left().value().getMainYaml();
1208
1209                                 toscaArtifact.setPayloadData(payload);
1210                                 byte[] decodedPayload = toscaArtifact.getPayloadData();
1211
1212                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1213                                                 toscaArtifact.getArtifactLabel());
1214                                 toscaArtifact.setUniqueId(uniqueId);
1215                                 toscaArtifact.setEsId(toscaArtifact.getUniqueId());
1216
1217                                 toscaArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1218                                 ESArtifactData artifactData = new ESArtifactData(toscaArtifact.getEsId(), decodedPayload);
1219                                 artifactCassandraDao.saveArtifact(artifactData);
1220
1221                                 log.debug("Tosca yaml artifact esId  {} ", toscaArtifact.getEsId());
1222                         }
1223                         ArtifactDefinition csarArtifact = null;
1224                         op = toscaArtifacts.values().stream()
1225                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_CSAR.getType())).findAny();
1226
1227                         if (op.isPresent()) {
1228                                 csarArtifact = op.get();
1229                         }
1230
1231                         if (csarArtifact != null) {
1232                                 Either<byte[], ResponseFormat> generated = csarUtils.createCsar(parent, true, true);
1233
1234                                 if (generated.isRight()) {
1235                                         log.debug("Failed to export tosca csar for component {} error {}", parent.getUniqueId(),
1236                                                         generated.right().value());
1237
1238                                         return Either.right(ToscaError.GENERAL_ERROR);
1239                                 }
1240                                 byte[] value = generated.left().value();
1241                                 csarArtifact.setPayload(value);
1242                                 byte[] decodedPayload = csarArtifact.getPayloadData();
1243
1244                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1245                                                 csarArtifact.getArtifactLabel());
1246                                 csarArtifact.setUniqueId(uniqueId);
1247                                 csarArtifact.setEsId(csarArtifact.getUniqueId());
1248
1249                                 csarArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1250                                 ESArtifactData artifactData = new ESArtifactData(csarArtifact.getEsId(), decodedPayload);
1251                                 artifactCassandraDao.saveArtifact(artifactData);
1252                                 log.debug("Tosca csar artifact esId  {} ", csarArtifact.getEsId());
1253
1254                         }
1255
1256                 } catch (Exception ex) {
1257                         log.error("Failed to generate tosca atifact component id {} component name {} error {}",
1258                                         parent.getUniqueId(), parent.getName(), ex.getMessage());
1259
1260                         return Either.right(ToscaError.GENERAL_ERROR);
1261                 }
1262
1263                 return Either.left(parent);
1264         }
1265
1266            private ArtifactDefinition createVfModuleArtifact(ComponentInstance currVF, Service service) {
1267
1268                 ArtifactDefinition vfModuleArtifactDefinition = new ArtifactDefinition();
1269
1270                 vfModuleArtifactDefinition.setDescription("Auto-generated VF Modules information artifact");
1271                 vfModuleArtifactDefinition.setArtifactDisplayName("Vf Modules Metadata");
1272                 vfModuleArtifactDefinition.setArtifactType(ArtifactTypeEnum.VF_MODULES_METADATA.getType());
1273                 vfModuleArtifactDefinition.setArtifactGroupType(ArtifactGroupTypeEnum.DEPLOYMENT);
1274                 vfModuleArtifactDefinition.setArtifactLabel("vfModulesMetadata");
1275                 vfModuleArtifactDefinition.setTimeout(0);
1276                 vfModuleArtifactDefinition.setArtifactName(currVF.getNormalizedName() + "_modules.json");
1277
1278                return vfModuleArtifactDefinition;
1279             }
1280
1281
1282         private void fillVfModuleInstHeatEnvPayload(Component parent, ComponentInstance instance, List<GroupInstance> groupsForCurrVF,
1283                         ArtifactDefinition vfModuleArtifact) {
1284                 log.debug("generate new vf module for component. name  {}, id {}, Version {}", instance.getName(), instance.getUniqueId());
1285
1286                 String uniqueId = UniqueIdBuilder.buildInstanceArtifactUniqueId(parent.getUniqueId(), instance.getUniqueId(), vfModuleArtifact.getArtifactLabel());
1287
1288                 vfModuleArtifact.setUniqueId(uniqueId);
1289                 vfModuleArtifact.setEsId(vfModuleArtifact.getUniqueId());
1290
1291                 List<VfModuleArtifactPayload> vfModulePayloadForCurrVF = new ArrayList<>();
1292                 if (groupsForCurrVF != null) {
1293                         for (GroupInstance groupInstance : groupsForCurrVF) {
1294                                 VfModuleArtifactPayload modulePayload = new VfModuleArtifactPayload(groupInstance);
1295                                 vfModulePayloadForCurrVF.add(modulePayload);
1296                         }
1297                         Collections.sort(vfModulePayloadForCurrVF,
1298                                         (art1, art2) -> VfModuleArtifactPayload.compareByGroupName(art1, art2));
1299
1300                         final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1301
1302                         String vfModulePayloadString = gson.toJson(vfModulePayloadForCurrVF);
1303                         log.debug("vfModulePayloadString {}", vfModulePayloadString);
1304                         if (vfModulePayloadString != null) {
1305                                 String newCheckSum = GeneralUtility
1306                                                 .calculateMD5Base64EncodedByByteArray(vfModulePayloadString.getBytes());
1307                                 vfModuleArtifact.setArtifactChecksum(newCheckSum);
1308
1309                                 ESArtifactData artifactData = new ESArtifactData(vfModuleArtifact.getEsId(),
1310                                                 vfModulePayloadString.getBytes());
1311                                 artifactCassandraDao.saveArtifact(artifactData);
1312
1313                         }
1314
1315                 }
1316
1317         }
1318
1319         private Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> parseVFModuleJson(ArtifactDefinition vfModuleArtifact) {
1320                 log.info("Try to get vfModule json from cassandra {}", vfModuleArtifact.getEsId());
1321                 Either<ESArtifactData, CassandraOperationStatus> vfModuleData = artifactCassandraDao.getArtifact(vfModuleArtifact.getEsId());
1322
1323                 if (vfModuleData.isRight()) {
1324                         CassandraOperationStatus resourceUploadStatus = vfModuleData.right().value();
1325                         StorageOperationStatus storageResponse = DaoStatusConverter.convertCassandraStatusToStorageStatus(resourceUploadStatus);
1326                         log.error("failed to fetch vfModule json {} from cassandra. Status is {}", vfModuleArtifact.getEsId(), storageResponse);
1327                         return Either.right(storageResponse);
1328
1329                 }
1330
1331                 ESArtifactData esArtifactData = vfModuleData.left().value();
1332                 String gsonData = new String( esArtifactData.getDataAsArray());
1333                 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1334                 JsonArray jsonElement = new JsonArray();
1335                 jsonElement = gson.fromJson(gsonData, jsonElement.getClass());
1336                 List<VfModuleArtifactPayloadEx> vfModules = new ArrayList<>();
1337                 jsonElement.forEach(je ->{
1338                         VfModuleArtifactPayloadEx vfModule = ComponentsUtils.parseJsonToObject(je.toString(), VfModuleArtifactPayloadEx.class);
1339                         vfModules.add(vfModule);
1340                 });
1341
1342                 log.debug  ("parse vf module finish {}", gsonData);
1343                 return Either.left(vfModules);
1344
1345         }
1346 }
1347
1348