asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/ArtifactUuidFix.java

   1 package org.openecomp.sdc.asdctool.impl;
   2
   3 import com.google.gson.Gson;
   4 import com.google.gson.GsonBuilder;
   5 import com.google.gson.JsonArray;
   6 import fj.data.Either;
   7 import org.openecomp.sdc.asdctool.impl.validator.utils.VfModuleArtifactPayloadEx;
   8 import org.openecomp.sdc.be.components.distribution.engine.VfModuleArtifactPayload;
   9 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  10 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  11 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
  12 import org.openecomp.sdc.be.dao.jsongraph.TitanDao;
  13 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
  14 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
  15 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
  16 import org.openecomp.sdc.be.dao.titan.TitanOperationStatus;
  17 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
  18 import org.openecomp.sdc.be.datatypes.elements.GroupDataDefinition;
  19 import org.openecomp.sdc.be.datatypes.elements.MapArtifactDataDefinition;
  20 import org.openecomp.sdc.be.datatypes.elements.MapGroupsDataDefinition;
  21 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
  22 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
  23 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
  24 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  25 import org.openecomp.sdc.be.datatypes.tosca.ToscaDataDefinition;
  26 import org.openecomp.sdc.be.impl.ComponentsUtils;
  27 import org.openecomp.sdc.be.model.*;
  28 import org.openecomp.sdc.be.model.jsontitan.datamodel.TopologyTemplate;
  29 import org.openecomp.sdc.be.model.jsontitan.datamodel.ToscaElement;
  30 import org.openecomp.sdc.be.model.jsontitan.operations.ToscaOperationFacade;
  31 import org.openecomp.sdc.be.model.jsontitan.utils.ModelConverter;
  32 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  33 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
  34 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
  35 import org.openecomp.sdc.be.resources.data.ESArtifactData;
  36 import org.openecomp.sdc.be.tosca.CsarUtils;
  37 import org.openecomp.sdc.be.tosca.ToscaError;
  38 import org.openecomp.sdc.be.tosca.ToscaExportHandler;
  39 import org.openecomp.sdc.be.tosca.ToscaRepresentation;
  40 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
  41 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
  42 import org.openecomp.sdc.common.api.Constants;
  43 import org.openecomp.sdc.common.log.wrappers.Logger;
  44 import org.openecomp.sdc.common.util.GeneralUtility;
  45 import org.openecomp.sdc.exception.ResponseFormat;
  46 import org.springframework.beans.factory.annotation.Autowired;
  47
  48 import java.io.*;
  49 import java.util.*;
  50 import java.util.stream.Collectors;
  51
  52 @org.springframework.stereotype.Component("artifactUuidFix")
  53 public class ArtifactUuidFix {
  54
  55         @Autowired
  56         private TitanDao titanDao;
  57
  58         @Autowired
  59         private ToscaOperationFacade toscaOperationFacade;
  60         @Autowired
  61         private ToscaExportHandler toscaExportUtils;
  62         @Autowired
  63         private ArtifactCassandraDao artifactCassandraDao;
  64
  65
  66         @Autowired
  67         private CsarUtils csarUtils;
  68
  69         private static Logger log = Logger.getLogger(ArtifactUuidFix.class.getName());
  70
  71         public boolean doFix(String fixComponent, String runMode) {
  72                 List<Resource> vfLst = new ArrayList<>();
  73                 List<Service> serviceList = new ArrayList<>();
  74                 Map<String, List<Component>> nodeToFixTosca = new HashMap<>();
  75                 Map<String, List<Component>> vfToFixTosca = new HashMap<>();
  76                 Map<String, List<Component>> serviceToFixTosca = new HashMap<>();
  77
  78                 long time = System.currentTimeMillis();
  79
  80                 doFixTosca(nodeToFixTosca, vfToFixTosca, serviceToFixTosca);
  81
  82                 if (fixComponent.equals("vf_only")) {
  83                         if (fetchFaultVf(fixComponent, vfLst, time) == false) {
  84                                 return false;
  85                         }
  86                 } else {
  87                         if (fetchServices(fixComponent, serviceList, time) == false) {
  88                                 return false;
  89                         }
  90                 }
  91                 if (runMode.equals("service_vf") || runMode.equals("fix")) {
  92                         log.info("Mode {}. Find problem VFs", runMode);
  93                         if (fetchVf(serviceList, vfLst, time) == false) {
  94                                 log.info("Mode {}. Find problem VFs finished with failure", runMode);
  95                                 return false;
  96                         }
  97                         log.info("Mode {}. Find problem VFs finished with success", runMode);
  98                 }
  99                 if (runMode.equals("fix") || runMode.equals("fix_only_services")) {
 100                         log.info("Mode {}. Start fix", runMode);
 101                         if (fix(vfLst, serviceList, nodeToFixTosca, vfToFixTosca, serviceToFixTosca) == false) {
 102                                 log.info("Mode {}. Fix finished with failure", runMode);
 103                                 return false;
 104                         }
 105                         log.info("Mode {}. Fix finished with success", runMode);
 106                 }
 107
 108                 return true;
 109         }
 110
 111         private boolean fetchFaultVf(String fixComponent, List<Resource> vfLst, long time) {
 112                 log.info("Find fault VF ");
 113                 String fileName = "fault_" + time + ".csv";
 114                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"))) {
 115                         writer.write("vf name, vf id, state, version\n");
 116
 117                         Map<GraphPropertyEnum, Object> hasProps = new HashMap<>();
 118                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
 119                         hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF.name());
 120
 121                         Map<GraphPropertyEnum, Object> hasNotProps = new HashMap<>();
 122                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 123                         log.info("Try to fetch resources with properties {} and not {}", hasProps, hasNotProps);
 124
 125                         Either<List<GraphVertex>, TitanOperationStatus> servicesByCriteria = titanDao
 126                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 127                         if (servicesByCriteria.isRight()) {
 128                                 log.info("Failed to fetch resources {}", servicesByCriteria.right().value());
 129                                 return false;
 130                         }
 131                         List<GraphVertex> resources = servicesByCriteria.left().value();
 132                         for (GraphVertex gv : resources) {
 133                                 ComponentParametersView filter = new ComponentParametersView(true);
 134                                 filter.setIgnoreComponentInstances(false);
 135                                 filter.setIgnoreArtifacts(false);
 136                                 filter.setIgnoreGroups(false);
 137
 138                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 139                                                 .getToscaElement(gv.getUniqueId());
 140                                 if (toscaElement.isRight()) {
 141                                         log.info("Failed to fetch resources {} {}", gv.getUniqueId(), toscaElement.right().value());
 142                                         continue;
 143                                 }
 144
 145                                 Resource resource = toscaElement.left().value();
 146                                 String resourceName = resource.getName();
 147                                 Map<String, ArtifactDefinition> deploymentArtifacts = resource.getDeploymentArtifacts();
 148                                 List<GroupDefinition> groups = resource.getGroups();
 149                                 if (groups == null || groups.isEmpty()) {
 150                                         log.info("No groups for resource {} id {} ", resourceName, gv.getUniqueId());
 151                                         continue;
 152                                 }
 153                                 boolean isProblematic = false;
 154                                 for (GroupDefinition gr : groups) {
 155                                         if (gr.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 156                                                 if (isProblematicGroup(gr, resourceName, deploymentArtifacts)) {
 157                                                         isProblematic = true;
 158                                                         break;
 159                                                 }
 160                                         }
 161                                 }
 162                                 if (isProblematic) {
 163                                         vfLst.add(resource);
 164                                         writeModuleResultToFile(writer, resource, null);
 165                                         writer.flush();
 166                                 }
 167                                 titanDao.commit();
 168                         }
 169
 170                 } catch (Exception e) {
 171                         log.info("Failed to fetch vf resources ", e);
 172                         return false;
 173                 } finally {
 174                         titanDao.commit();
 175                 }
 176                 return true;
 177         }
 178
 179         private boolean fetchVf(List<Service> serviceList, List<Resource> vfLst, long time) {
 180                 log.info("Find problem VF ");
 181                 if (serviceList.isEmpty()) {
 182                         log.info("No services as input");
 183                         return true;
 184                 }
 185                 String fileName = "problemVf_" + time + ".csv";
 186                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"))) {
 187                         writer.write("vf name, vf id, state, version, example service name\n");
 188                         Set<String> vfIds = new HashSet<>();
 189                         for (Service service : serviceList) {
 190                                 List<ComponentInstance> componentInstances = service.getComponentInstances().stream()
 191                                                 .filter(ci -> ci.getOriginType().equals(OriginTypeEnum.VF)).collect(Collectors.toList());
 192                                 for (ComponentInstance ci : componentInstances) {
 193                                         if (!vfIds.contains(ci.getComponentUid())) {
 194                                                 vfIds.add(ci.getComponentUid());
 195                                                 ComponentParametersView filter = new ComponentParametersView(true);
 196                                                 filter.setIgnoreComponentInstances(false);
 197                                                 filter.setIgnoreArtifacts(false);
 198                                                 filter.setIgnoreGroups(false);
 199                                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 200                                                                 .getToscaElement(ci.getComponentUid(), filter);
 201                                                 if (toscaElement.isRight()) {
 202                                                         log.info("Failed to fetch resource {} {}", ci.getComponentUid(),
 203                                                                         toscaElement.right().value());
 204                                                         continue;
 205                                                 }
 206                                                 Resource resource = toscaElement.left().value();
 207                                                 if (resource.getResourceType().equals(ResourceTypeEnum.VF)) {
 208                                                         vfLst.add(resource);
 209                                                         writeModuleResultToFile(writer, resource, service);
 210                                                         writer.flush();
 211
 212                                                 }
 213                                                 titanDao.commit();
 214                                         }
 215                                 }
 216                         }
 217                         log.info("output file with list of Vf : {}", fileName);
 218                 } catch (Exception e) {
 219                         log.info("Failed to fetch services ", e);
 220                         return false;
 221                 } finally {
 222                         titanDao.commit();
 223                 }
 224                 return true;
 225         }
 226
 227         private boolean fetchServices(String fixServices, List<Service> serviceList, long time) {
 228                 log.info("Find problem Services {}", fixServices);
 229                 String fileName = "problemService_" + time + ".csv";
 230                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"))) {
 231                         writer.write("service name, service id, state, version\n");
 232
 233                         Map<GraphPropertyEnum, Object> hasProps = new HashMap<>();
 234                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
 235                         if (fixServices.equals("distributed_only")) {
 236                                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
 237                                 hasProps.put(GraphPropertyEnum.DISTRIBUTION_STATUS, DistributionStatusEnum.DISTRIBUTED.name());
 238                         }
 239
 240                         Map<GraphPropertyEnum, Object> hasNotProps = new HashMap<>();
 241                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 242                         log.info("Try to fetch services with properties {} and not {}", hasProps, hasNotProps);
 243
 244                         Either<List<GraphVertex>, TitanOperationStatus> servicesByCriteria = titanDao
 245                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 246                         if (servicesByCriteria.isRight()) {
 247                                 log.info("Failed to fetch services {}", servicesByCriteria.right().value());
 248                                 return false;
 249                         }
 250                         List<GraphVertex> services = servicesByCriteria.left().value();
 251                         for (GraphVertex gv : services) {
 252                                 ComponentParametersView filter = new ComponentParametersView(true);
 253                                 filter.setIgnoreComponentInstances(false);
 254                                 filter.setIgnoreArtifacts(false);
 255                                 filter.setIgnoreGroups(false);
 256
 257                                 Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade
 258                                                 .getToscaElement(gv.getUniqueId(), filter);
 259                                 if (toscaElement.isRight()) {
 260                                         log.info("Failed to fetch service {} {}", gv.getUniqueId(), toscaElement.right().value());
 261                                         continue;
 262                                 }
 263                                 Service service = toscaElement.left().value();
 264
 265                                 String serviceName = (String) gv.getMetadataProperty(GraphPropertyEnum.NAME);
 266
 267                                 boolean isProblematic = isProblematicService(service, serviceName);
 268                                 if (isProblematic) {
 269                                         serviceList.add(service);
 270                                         writeModuleResultToFile(writer, service, null);
 271                                         writer.flush();
 272
 273                                 }
 274
 275                                 titanDao.commit();
 276                         }
 277                         log.info("output file with list of services : {}", fileName);
 278                 } catch (Exception e) {
 279                         log.info("Failed to fetch services ", e);
 280                         return false;
 281                 } finally {
 282                         titanDao.commit();
 283                 }
 284                 return true;
 285         }
 286
 287         private boolean isProblematicService( Service service, String serviceName) {
 288
 289                 List<ComponentInstance> componentInstances = service.getComponentInstances();
 290
 291                 if (componentInstances == null) {
 292                         log.info("No instances for service {} ", service.getUniqueId());
 293                         return false;
 294                 }
 295                 boolean isCheckVFModules = true;
 296                 if(service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN ||
 297                                 service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT){
 298                         isCheckVFModules = false;
 299                 }
 300                 for (ComponentInstance ci : componentInstances) {
 301                         Map<String, ArtifactDefinition> deploymentArtifacts = ci.getDeploymentArtifacts();
 302                         List<GroupInstance> groupInstances = ci.getGroupInstances();
 303                         if (groupInstances == null || groupInstances.isEmpty()) {
 304                                 log.info("No instance groups for instance {} in service {} id {} ", ci.getName(), serviceName,
 305                                                 service.getUniqueId());
 306                                 continue;
 307                         }
 308                         List<VfModuleArtifactPayloadEx> vfModules = null;
 309                         if(isCheckVFModules){
 310                                 Optional<ArtifactDefinition> optionalVfModuleArtifact = deploymentArtifacts.values().stream()
 311                                                 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 312
 313                                  if(!optionalVfModuleArtifact.isPresent())
 314                                          return true;
 315
 316                                  ArtifactDefinition vfModuleArtifact = optionalVfModuleArtifact.get();
 317                                  Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> vfModulesEither = parseVFModuleJson(vfModuleArtifact);
 318                                  if(vfModulesEither.isRight()){
 319                                          log.error("Failed to parse vfModule for service {} status is {}", service.getUniqueId(), vfModulesEither.right().value());
 320                                          return true;
 321                                  }
 322                                  vfModules = vfModulesEither.left().value();
 323                                  if(vfModules == null || vfModules.isEmpty() ){
 324                                          log.info("vfModules empty for service {}", service.getUniqueId());
 325                                          return true;
 326                                  }
 327                         }
 328
 329                         for (GroupInstance gi : groupInstances) {
 330                                 if (gi.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 331                                         VfModuleArtifactPayloadEx vfModule= null;
 332                                         if(isCheckVFModules && vfModules != null){
 333                                                 Optional<VfModuleArtifactPayloadEx> op = vfModules.stream().filter(vf -> vf.getVfModuleModelName().equals(gi.getGroupName())).findAny();
 334                                                 if(!op.isPresent()){
 335                                                          log.error("Failed to find vfModule for group {}", gi.getGroupName());
 336                                                         return true;
 337                                                 }
 338                                                 vfModule = op.get();
 339                                         }
 340                                         if (isProblematicGroupInstance(gi, ci.getName(), serviceName, deploymentArtifacts, vfModule)) {
 341                                                 return true;
 342                                         }
 343                                 }
 344                         }
 345
 346                 }
 347                 return false;
 348         }
 349
 350         private boolean isProblematicGroup(GroupDefinition gr, String resourceName,
 351                         Map<String, ArtifactDefinition> deploymentArtifacts) {
 352                 List<String> artifacts = gr.getArtifacts();
 353                 List<String> artifactsUuid = gr.getArtifactsUuid();
 354                 Set<String> artifactsSet = new HashSet<>();
 355
 356                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 357                         log.info("No groups in resource {} ", resourceName);
 358                         return true;
 359                 }
 360                 artifactsSet.addAll(artifacts);
 361                 if (artifactsSet.size() < artifacts.size()) {
 362                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", gr.getName(), resourceName);
 363                         return true;
 364                 }
 365
 366                 if (artifacts.size() < artifactsUuid.size()) {
 367                         log.info(" artifacts.size() < artifactsUuid.size() group {} in resource {} ", gr.getName(), resourceName);
 368                         return true;
 369                 }
 370                 if (artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 371                         log.info(
 372                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() group {} in resource {} ",
 373                                         gr.getName(), resourceName);
 374                         return true;
 375                 }
 376                 if (artifactsUuid.contains(null)) {
 377                         log.info(" artifactsUuid.contains(null) group {} in resource {} ", gr.getName(), resourceName);
 378                         return true;
 379                 }
 380
 381                 for (String artifactId : artifacts) {
 382                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 383                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 384                         if (artifactDefinition == null) {
 385                                 log.info(" artifactDefinition == null label {} group {} in resource {} ", artifactlabel, gr.getName(),
 386                                                 resourceName);
 387                                 return true;
 388                         }
 389                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 390                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 391                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 392                                         log.info(
 393                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} group {} in resource {} ",
 394                                                         artifactlabel, artifactId, gr.getName(), resourceName);
 395                                         return true;
 396                                 }
 397                                 if (!artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 398                                         log.info(
 399                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} group {} in resource {} ",
 400                                                         artifactlabel, gr.getName(), resourceName);
 401                                         return true;
 402                                 }
 403                         }
 404                 }
 405                 for (String artifactUUID : artifactsUuid) {
 406                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 407                         if (label != null && !label.isEmpty() && !label.equals("")) {
 408                                 return true;
 409                         }
 410                 }
 411
 412                 return false;
 413         }
 414
 415         private boolean isProblematicGroupInstance(GroupInstance gi, String instName, String servicename,
 416                         Map<String, ArtifactDefinition> deploymentArtifacts, VfModuleArtifactPayloadEx vfModule) {
 417                 List<String> artifacts = gi.getArtifacts();
 418                 List<String> artifactsUuid = gi.getArtifactsUuid();
 419                 List<String> instArtifactsUuid = gi.getGroupInstanceArtifactsUuid();
 420                 List<String> instArtifactsId = gi.getGroupInstanceArtifacts();
 421                 Set<String> instArtifatIdSet = new HashSet<>();
 422                 Set<String> artifactsSet = new HashSet<>();
 423
 424                 log.info("check group {} for instance {} ", gi.getGroupName(), instName);
 425                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 426                         log.info("No instance groups for instance {} in service {} ", instName, servicename);
 427                         return true;
 428                 }
 429                 artifactsSet.addAll(artifacts);
 430                 if (artifactsSet.size() < artifacts.size()) {
 431                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", instName, servicename);
 432                         return true;
 433                 }
 434
 435                 if (instArtifactsId != null && !instArtifactsId.isEmpty()) {
 436                         instArtifatIdSet.addAll(instArtifactsId);
 437                 }
 438
 439                 if (artifacts.size() < artifactsUuid.size()) {
 440                         log.info(" artifacts.size() < artifactsUuid.size() inst {} in service {} ", instName, servicename);
 441                         return true;
 442                 }
 443                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 444                         log.info(
 445                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() inst {} in service {} ",
 446                                         instName, servicename);
 447                         return true;
 448                 }
 449                 if (artifactsUuid.contains(null)) {
 450                         log.info(" artifactsUuid.contains(null) inst {} in service {} ", instName, servicename);
 451                         return true;
 452                 }
 453                 if (instArtifactsId != null && instArtifatIdSet.size() < instArtifactsId.size()) {
 454                         log.info(" instArtifatIdSet.size() < instArtifactsId.size() inst {} in service {} ", instName, servicename);
 455                         return true;
 456                 }
 457
 458                 if ((instArtifactsId != null && instArtifactsUuid != null)
 459                                 && instArtifactsId.size() != instArtifactsUuid.size()) {
 460                         log.info(" instArtifactsId.size() != instArtifactsUuid.size() inst {} in service {} ", instName,
 461                                         servicename);
 462                         return true;
 463                 }
 464
 465                 for (String artifactId : artifacts) {
 466                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 467                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 468                         if (artifactDefinition == null) {
 469                                 log.info(" artifactDefinition == null label {} inst {} in service {} ", artifactlabel, instName,
 470                                                 servicename);
 471                                 return true;
 472                         }
 473                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 474                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 475                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 476                                         log.info(
 477                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} inst {} in service {} ",
 478                                                         artifactlabel, artifactId, instName, servicename);
 479                                         return true;
 480                                 }
 481                                 if (!artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 482                                         log.info(
 483                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 484                                                         artifactlabel, instName, servicename);
 485                                         return true;
 486                                 }
 487                         } else {
 488                                 if (instArtifactsUuid == null || instArtifactsUuid.isEmpty()) {
 489                                         log.info(" instArtifactsUuid empty. label {} inst {} in service {} ", artifactlabel, instName,
 490                                                         servicename);
 491                                         return true;
 492                                 }
 493                                 if (!instArtifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 494                                         log.info(
 495                                                         " instArtifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 496                                                         artifactlabel, instName, servicename);
 497                                         return true;
 498                                 }
 499                         }
 500                 }
 501                 for (String artifactUUID : artifactsUuid) {
 502                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 503                         if (label != null && !label.isEmpty() && !label.equals("")) {
 504                                 return true;
 505                         }
 506                 }
 507                 if(vfModule != null ){
 508                         return isProblematicVFModule(vfModule, artifactsUuid, instArtifactsUuid);
 509                 }
 510
 511                 return false;
 512         }
 513
 514         private boolean isProblematicVFModule(VfModuleArtifactPayloadEx vfModule, List<String> artifactsUuid,
 515                         List<String> instArtifactsUuid) {
 516                 log.info(" isProblematicVFModule  {}  ", vfModule.getVfModuleModelName());
 517                 List<String> vfModuleArtifacts = vfModule.getArtifacts();
 518                 List<String> allArtifacts = new ArrayList<>();
 519                 allArtifacts.addAll(artifactsUuid);
 520                 if(instArtifactsUuid != null)
 521                         allArtifacts.addAll(instArtifactsUuid);
 522                 if((vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()){
 523                         log.error(" vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()");
 524                         return true;
 525                 }
 526                 if(vfModuleArtifacts!= null){
 527                         if( vfModuleArtifacts.size() != allArtifacts.size()){
 528                                 log.error(" vfModuleArtifacts.size() != allArtifacts.size()");
 529                                 return true;
 530                         }
 531                         for(String vfModuleArtifact: vfModuleArtifacts){
 532                                 Optional<String> op = allArtifacts.stream().filter(a -> a.equals(vfModuleArtifact)).findAny();
 533                                 if(!op.isPresent()){
 534                                         log.error("failed to find artifact {} in group artifacts {}", vfModuleArtifact,  allArtifacts);
 535                                         return true;
 536                                 }
 537                         }
 538                 }
 539                 return false;
 540         }
 541
 542
 543
 544         private boolean fix(List<Resource> vfLst, List<Service> serviceList, Map<String, List<Component>> nodesToFixTosca,
 545                         Map<String, List<Component>> vfToFixTosca, Map<String, List<Component>> servicesToFixTosca) {
 546                 boolean res = true;
 547                 log.info(" Fix started ***** ");
 548                 if (vfLst != null && !vfLst.isEmpty()) {
 549                         res = fixVf(vfLst);
 550
 551                 }
 552
 553                 if (res && serviceList != null && !serviceList.isEmpty()) {
 554                         res = fixServices(serviceList);
 555
 556                 }
 557
 558                 Set<String> fixedIds = new HashSet<>();
 559
 560                 long time = System.currentTimeMillis();
 561                 String fileName = "FailedGenerateTosca" + "_" + time + ".csv";
 562                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"))) {
 563                         writer.write("componentType, name, version, UID, UUID, invariantUUID, state\n");
 564                         List<Component> failedList = new ArrayList<>();
 565
 566                         if (res && nodesToFixTosca != null && !nodesToFixTosca.isEmpty()) {
 567
 568                                 generateAndSaveToscaArtifacts(nodesToFixTosca, fixedIds, null, failedList);
 569
 570                         }
 571                         if (vfToFixTosca != null && !vfToFixTosca.isEmpty()) {
 572
 573                                 generateAndSaveToscaArtifacts(vfToFixTosca, fixedIds, vfLst, failedList);
 574
 575                         }
 576
 577                         for (Component component : vfLst) {
 578                                 res = generateToscaPerComponent(fixedIds, component);
 579                                 if (res) {
 580                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 581                                         Map<String, GroupDataDefinition> groups = topologyTemplate.getGroups();
 582                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.GROUPS, EdgeLabelEnum.GROUPS, groups);
 583                                         if (res) {
 584                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getDeploymentArtifacts();
 585                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.DEPLOYMENT_ARTIFACTS,
 586                                                                 EdgeLabelEnum.DEPLOYMENT_ARTIFACTS, arifacts);
 587                                         }
 588                                         if (res) {
 589                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 590                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 591                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 592                                         }
 593                                         titanDao.commit();
 594                                 } else {
 595                                         failedList.add(component);
 596                                 }
 597                         }
 598
 599                         if (servicesToFixTosca != null && !servicesToFixTosca.isEmpty()) {
 600                                 generateAndSaveToscaArtifacts(servicesToFixTosca, fixedIds, serviceList, failedList);
 601
 602                         }
 603
 604
 605                         for (Component component : serviceList) {
 606                                 res = generateToscaPerComponent(fixedIds, component);
 607                                 if (res) {
 608                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 609                                         Map<String, MapGroupsDataDefinition> groups = topologyTemplate.getInstGroups();
 610                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_GROUPS, EdgeLabelEnum.INST_GROUPS,
 611                                                         groups);
 612
 613                                         if (res) {
 614                                                 Map<String, MapArtifactDataDefinition> artifacts = topologyTemplate
 615                                                                 .getInstDeploymentArtifacts();
 616                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_DEPLOYMENT_ARTIFACTS,
 617                                                                 EdgeLabelEnum.INST_DEPLOYMENT_ARTIFACTS, artifacts);
 618                                         }
 619                                         if (res) {
 620                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 621                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 622                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 623                                         }
 624                                         titanDao.commit();
 625                                 } else {
 626                                         failedList.add(component);
 627                                 }
 628
 629                         }
 630                         if (!failedList.isEmpty()) {
 631                                 for (Component component : failedList) {
 632                                         StringBuilder sb = new StringBuilder(component.getComponentType().getValue());
 633                                         sb.append(",").append(component.getName()).append(",").append(component.getVersion()).append(",")
 634                                                         .append(component.getUniqueId()).append(",").append(component.getUUID()).append(",")
 635                                                         .append(component.getInvariantUUID()).append(",").append(component.getLifecycleState());
 636
 637                                         sb.append("\n");
 638                                         writer.write(sb.toString());
 639                                 }
 640                                 writer.flush();
 641                         }
 642                 } catch (IOException e) {
 643                         log.error(e.getMessage());
 644                 } finally {
 645                         titanDao.commit();
 646                 }
 647                 log.info(" Fix finished with res {} ***** ", res);
 648                 return res;
 649         }
 650
 651         private boolean generateAndSaveToscaArtifacts(Map<String, List<Component>> nodesToFixTosca, Set<String> fixedIds,
 652                         List<? extends Component> componentsWithFailedGroups, List<Component> failedList) {
 653                 boolean res = true;
 654                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts started ");
 655                 for (Map.Entry<String, List<Component>> entry : nodesToFixTosca.entrySet()) {
 656
 657                         List<Component> component = entry.getValue();
 658                         for (Component c : component) {
 659                                 log.debug("Migration1707ArtifactUuidFix  fix tosca on component : id {},  name {} ", c.getUniqueId(),
 660                                                 c.getName());
 661                                 if (componentsWithFailedGroups != null) {
 662                                         Optional<Component> op = (Optional<Component>) componentsWithFailedGroups.stream()
 663                                                         .filter(cg -> cg.getUniqueId().equals(c.getUniqueId())).findAny();
 664                                         if (!op.isPresent())
 665                                                 res = generateToscaPerComponent(fixedIds, c);
 666                                 } else
 667                                         res = generateToscaPerComponent(fixedIds, c);
 668                                 if (res) {
 669                                         ToscaElement topologyTemplate = ModelConverter.convertToToscaElement(c);
 670                                         Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 671                                         res = fixDataOnGraph(c.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS, EdgeLabelEnum.TOSCA_ARTIFACTS,
 672                                                         arifacts);
 673                                         titanDao.commit();
 674                                 } else {
 675                                         failedList.add(c);
 676                                 }
 677
 678                         }
 679                 }
 680                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts finished with res {} ", res);
 681                 return res;
 682         }
 683
 684         private boolean generateToscaPerComponent(Set<String> fixedIds, Component c) {
 685                 boolean res = true;
 686                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent started component name {} id {}",
 687                                 c.getName(), c.getUniqueId());
 688                 try {
 689                         Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
 690                                         .getToscaFullElement(c.getUniqueId());
 691                         if (toscaElement.isRight()) {
 692                                 log.info("Failed to fetch resources {} {}", c.getUniqueId(), toscaElement.right().value());
 693                                 return false;
 694                         }
 695                         Component toscaElementFull = toscaElement.left().value();
 696                         toscaElementFull.setGroups(c.getGroups());
 697                         List<ComponentInstance> ciListFull = toscaElementFull.getComponentInstances();
 698                         List<ComponentInstance> ciList = c.getComponentInstances();
 699                         if (ciListFull != null && !ciListFull.isEmpty()) {
 700                                 ciListFull.forEach(ciFull -> {
 701                                         ComponentInstance compInst = ciList.stream()
 702                                                         .filter(ci -> ci.getUniqueId().equals(ciFull.getUniqueId())).findAny().get();
 703                                         ciFull.setGroupInstances(compInst.getGroupInstances());
 704                                 });
 705                         }
 706
 707                         Either<Component, ToscaError> either = generateToscaArtifact(toscaElementFull);
 708
 709                         if (either.isRight()) {
 710                                 log.error("Couldn't generate and save tosca template component  unique id {}, name {} error: {}",
 711                                                 toscaElementFull.getUniqueId(), toscaElementFull.getName(), either.right().value());
 712                                 res = false;
 713
 714                         }
 715
 716                         if (res) {
 717                                 c.setToscaArtifacts(either.left().value().getToscaArtifacts());
 718                                 fixedIds.add(toscaElementFull.getUniqueId());
 719                         }
 720                 } finally {
 721                         if (res)
 722                                 titanDao.commit();
 723                         else
 724                                 titanDao.rollback();
 725                 }
 726                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent finished  component name {} id {} res {}",
 727                                 c.getName(), c.getUniqueId(), res);
 728                 return res;
 729         }
 730
 731         private <T extends ToscaDataDefinition> boolean fixDataOnGraph(String componentId, VertexTypeEnum vertexTypeEnum,
 732                         EdgeLabelEnum edgeLabelEnum, Map<String, T> groups) {
 733                 log.debug("amount groups to update: VertexTypeEnum {} EdgeLabelEnum {} data size {}", vertexTypeEnum.getName(),
 734                                 edgeLabelEnum, groups.size());
 735                 boolean res = true;
 736                 Either<GraphVertex, TitanOperationStatus> getResponse = titanDao.getVertexById(componentId,
 737                                 JsonParseFlagEnum.NoParse);
 738                 if (getResponse.isRight()) {
 739                         log.debug("Couldn't fetch component  unique id {}, error: {}", componentId, getResponse.right().value());
 740                         res = false;
 741
 742                 }
 743                 if (res) {
 744                         GraphVertex componentVertex = getResponse.left().value();
 745
 746                         GraphVertex toscaDataVertex = null;
 747                         Either<GraphVertex, TitanOperationStatus> groupVertexEither = titanDao.getChildVertex(componentVertex,
 748                                         edgeLabelEnum, JsonParseFlagEnum.ParseJson);
 749                         if (groupVertexEither.isRight() && groupVertexEither.right().value() == TitanOperationStatus.NOT_FOUND) {
 750                                 log.debug("no child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum, componentId,
 751                                                 groupVertexEither.right().value());
 752                                 return true;
 753                         }
 754                         if (groupVertexEither.isRight()) {
 755                                 res = false;
 756                                 log.debug("failed to get child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum,
 757                                                 componentId, groupVertexEither.right().value());
 758                         }
 759                         if (res) {
 760                                 toscaDataVertex = groupVertexEither.left().value();
 761                                 toscaDataVertex.setJson(groups);
 762                                 Either<GraphVertex, TitanOperationStatus> updatevertexEither = titanDao.updateVertex(toscaDataVertex);
 763                                 if (updatevertexEither.isRight()) {
 764                                         log.debug("failed to update vertex for component  unique id {}, error: {}", componentId,
 765                                                         updatevertexEither.right().value());
 766                                         titanDao.rollback();
 767                                         return false;
 768                                 }
 769                         }
 770                 }
 771                 log.debug("Fix data on graph finished: VertexTypeEnum {} EdgeLabelEnum {} res {}", vertexTypeEnum.getName(),
 772                                 res);
 773                 return res;
 774         }
 775
 776         private boolean fixServices(List<Service> serviceList) {
 777                 for (Service service : serviceList) {
 778                         log.debug("Migration1707ArtifactUuidFix  fix service: id {},  name {} ", service.getUniqueId(),
 779                                         service.getName());
 780                         List<ComponentInstance> instances = service.getComponentInstances();
 781                         for (ComponentInstance instance : instances) {
 782                                 fixComponentInstances(service, instance);
 783                         }
 784
 785                 }
 786                 return true;
 787
 788         }
 789
 790         private void fixComponentInstances(Service service, ComponentInstance instance) {
 791                 Map<String, ArtifactDefinition> artifactsMap = instance.getDeploymentArtifacts();
 792                 List<GroupInstance> groupsList = instance.getGroupInstances();
 793                 if (groupsList != null && artifactsMap != null) {
 794                         List<GroupInstance> groupsToDelete = new ArrayList<>();
 795                         for (GroupInstance group : groupsList) {
 796                                 fixGroupInstances(service, artifactsMap, groupsToDelete, group);
 797
 798                         }
 799
 800                         if (!groupsToDelete.isEmpty()) {
 801                                 log.debug("Migration1707ArtifactUuidFix  delete group:  resource id {}, group instance to delete {} ",
 802                                                 service.getUniqueId(), groupsToDelete);
 803                                 groupsList.removeAll(groupsToDelete);
 804
 805                         }
 806
 807                         Optional<ArtifactDefinition> optionalVfModuleArtifact = artifactsMap.values().stream()
 808                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 809                         ArtifactDefinition vfModuleAertifact = null;
 810                         if(!optionalVfModuleArtifact.isPresent()){
 811                                 vfModuleAertifact = createVfModuleArtifact(instance, service);
 812                                 artifactsMap.put(vfModuleAertifact.getArtifactLabel(), vfModuleAertifact);
 813                         }
 814                         else {
 815                                 vfModuleAertifact = optionalVfModuleArtifact.get();
 816                         }
 817                         fillVfModuleInstHeatEnvPayload(service, instance, groupsList, vfModuleAertifact);
 818                 }
 819         }
 820
 821         private void fixGroupInstances(Service service, Map<String, ArtifactDefinition> artifactsMap,
 822                         List<GroupInstance> groupsToDelete, GroupInstance group) {
 823                 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 824                         log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", service.getUniqueId(),
 825                                         group.getName());
 826                         if(group.getArtifacts() != null){
 827                                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 828                                 if(group.getGroupInstanceArtifacts() != null){
 829                                         List<String> groupInsArtifacts = new ArrayList<>(group.getGroupInstanceArtifacts());
 830                                         groupArtifactsSet.addAll(groupInsArtifacts);
 831                                 }
 832                                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 833
 834                                 clearGroupInstanceArtifacts(group);
 835
 836                                 for (String artifactId : groupArtifacts) {
 837                                         fixArtifactUndergroupInstances(artifactsMap, group, groupArtifacts, artifactId);
 838                                 }
 839                         }
 840                         if (group.getArtifacts() == null || group.getArtifacts().isEmpty()) {
 841                                 log.debug(
 842                                                 "Migration1707ArtifactUuidFix  fix groupInstance add to delete list:  resource id {} name {} , group name {} ",
 843                                                 service.getUniqueId(), service.getName(), group.getName());
 844                                 groupsToDelete.add(group);
 845                         }
 846                 }
 847         }
 848
 849         private void clearGroupInstanceArtifacts(GroupInstance group) {
 850                 if(group.getArtifacts() != null)
 851                         group.getArtifacts().clear();
 852                 else
 853                         group.setArtifacts(new ArrayList<>());
 854                 if(group.getArtifactsUuid() != null)
 855                         group.getArtifactsUuid().clear();
 856                 else{
 857                         group.setArtifactsUuid(new ArrayList<>());
 858                 }
 859                 if(group.getGroupInstanceArtifacts() != null)
 860                         group.getGroupInstanceArtifacts().clear();
 861                 else{
 862                         group.setGroupInstanceArtifacts(new ArrayList<>());
 863                 }
 864                 if(group.getGroupInstanceArtifactsUuid() != null )
 865                         group.getGroupInstanceArtifactsUuid().clear();
 866                 else
 867                         group.setGroupInstanceArtifactsUuid(new ArrayList<>());
 868         }
 869
 870         private void fixArtifactUndergroupInstances(Map<String, ArtifactDefinition> artifactsMap, GroupInstance group,
 871                         List<String> groupArtifacts, String artifactId) {
 872                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 873                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 874                                 group.getName(), artifactId, artifactlabel);
 875                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 876                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 877                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
 878                         String correctArtifactId = artifact.getUniqueId();
 879                         String correctArtifactUUID = artifact.getArtifactUUID();
 880                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 881                                 boolean isAddToGroup = true;
 882                                 if (groupArtifacts.size() == 1) {
 883                                         if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 884                                                 isAddToGroup = false;
 885                                                 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 886                                         }
 887                                 }
 888                                 if (isAddToGroup) {
 889                                         log.debug(
 890                                                         "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ",
 891                                                         group.getName(), correctArtifactId, correctArtifactUUID);
 892                                         group.getArtifacts().add(correctArtifactId);
 893                                         if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 894                                                 group.getArtifactsUuid().add(correctArtifactUUID);
 895                                         }
 896                                 }
 897                         } else {
 898                                 log.debug(
 899                                                 "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ",
 900                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 901                                 Set<String> tmpSet = new HashSet<>(group.getGroupInstanceArtifacts());
 902                                 tmpSet.add(correctArtifactId);
 903                                 group.setGroupInstanceArtifacts(new ArrayList<>(tmpSet));
 904                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 905                                         Set<String> tmpSetUUID = new HashSet<>(group.getGroupInstanceArtifactsUuid());
 906                                         tmpSetUUID.add(correctArtifactUUID);
 907                                         group.setGroupInstanceArtifactsUuid(new ArrayList<>(tmpSetUUID));
 908                                 }
 909                         }
 910                 }
 911         }
 912
 913         private boolean fixVf(List<Resource> vfLst) {
 914                 for (Resource resource : vfLst) {
 915                         log.debug("Migration1707ArtifactUuidFix  fix resource: id {},  name {} ", resource.getUniqueId(),
 916                                         resource.getName());
 917                         Map<String, ArtifactDefinition> artifactsMap = resource.getDeploymentArtifacts();
 918                         List<GroupDefinition> groupsList = resource.getGroups();
 919                         List<GroupDefinition> groupsToDelete = new ArrayList<>();
 920                         if (groupsList != null && artifactsMap != null) {
 921                                 for (GroupDefinition group : groupsList) {
 922                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE) && group.getArtifacts() != null) {
 923                                                 fixVfGroup(resource, artifactsMap, group);
 924                                         }
 925                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)
 926                                                         && (group.getArtifacts() == null || group.getArtifacts().isEmpty())) {
 927                                                 log.debug(
 928                                                                 "Migration1707ArtifactUuidFix  add group to delete list fix resource: id {},  name {} ",
 929                                                                 resource.getUniqueId(), resource.getName(), group.getName());
 930                                                 groupsToDelete.add(group);
 931                                         }
 932                                 }
 933
 934                                 if (!groupsToDelete.isEmpty()) {
 935                                         groupsList.removeAll(groupsToDelete);
 936
 937                                 }
 938                         }
 939
 940                 }
 941
 942                 return true;
 943         }
 944
 945         private void fixVfGroup(Resource resource, Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group) {
 946                 log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", resource.getUniqueId(),
 947                                 group.getName());
 948                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 949                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 950                 group.getArtifacts().clear();
 951                 group.getArtifactsUuid().clear();
 952
 953                 for (String artifactId : groupArtifacts) {
 954                         fixArtifactUnderGroup(artifactsMap, group, groupArtifacts, artifactId);
 955                 }
 956         }
 957
 958         private void fixArtifactUnderGroup(Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group,
 959                         List<String> groupArtifacts, String artifactId) {
 960
 961                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 962                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 963                                 group.getName(), artifactId, artifactlabel);
 964                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 965                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 966                         String correctArtifactId = artifact.getUniqueId();
 967                         String correctArtifactUUID = artifact.getArtifactUUID();
 968                         boolean isAddToGroup = true;
 969                         if (groupArtifacts.size() == 1) {
 970                                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
 971                                 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 972                                         isAddToGroup = false;
 973                                         artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 974                                 }
 975                         }
 976                         if (isAddToGroup) {
 977                                 log.debug(
 978                                                 "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ",
 979                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 980                                 group.getArtifacts().add(correctArtifactId);
 981                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 982                                         group.getArtifactsUuid().add(correctArtifactUUID);
 983                                 }
 984                         }
 985
 986                 }
 987         }
 988
 989         private String findArtifactLabelFromArtifactId(String artifactId) {
 990                 String artifactLabel = "";
 991
 992                 int index = artifactId.lastIndexOf('.');
 993                 if (index > 0 && index + 1 < artifactId.length())
 994                         artifactLabel = artifactId.substring(index + 1);
 995                 return artifactLabel;
 996         }
 997
 998         private void writeModuleResultToFile(Writer writer, org.openecomp.sdc.be.model.Component component,
 999                         Service service) {
1000                 try {
1001                         // "service name, service id, state, version
1002                         StringBuilder sb = new StringBuilder(component.getName());
1003                         sb.append(",").append(component.getUniqueId()).append(",").append(component.getLifecycleState()).append(",")
1004                                         .append(component.getVersion());
1005                         if (service != null) {
1006                                 sb.append(",").append(service.getName());
1007                         }
1008                         sb.append("\n");
1009                         writer.write(sb.toString());
1010                 } catch (IOException e) {
1011                         log.error(e.getMessage());
1012                 }
1013         }
1014
1015         private void writeModuleResultToFile(Writer writer, List<Component> components) {
1016                 try {
1017                         // "service name, service id, state, version
1018                         for (Component component : components) {
1019                                 StringBuilder sb = new StringBuilder(component.getName());
1020                                 sb.append(",").append(component.getUniqueId()).append(",").append(component.getInvariantUUID())
1021                                                 .append(",").append(component.getLifecycleState()).append(",").append(component.getVersion());
1022
1023                                 sb.append("\n");
1024                                 writer.write(sb.toString());
1025                         }
1026                 } catch (IOException e) {
1027
1028                         log.error(e.getMessage());
1029                 }
1030         }
1031
1032         public boolean doFixTosca(Map<String, List<Component>> nodeToFix, Map<String, List<Component>> vfToFix,
1033                         Map<String, List<Component>> serviceToFix) {
1034
1035                 Map<GraphPropertyEnum, Object> hasProps = new HashMap<>();
1036                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1037                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1038
1039                 Map<String, List<Component>> vertices = getVerticesToValidate(VertexTypeEnum.NODE_TYPE, hasProps);
1040                 boolean result = validateTosca(vertices, nodeToFix, "RESOURCE_TOSCA_ARTIFACTS");//
1041
1042                 hasProps.clear();
1043                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1044                 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF);
1045                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1046
1047                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1048                 result = validateTosca(vertices, vfToFix, "VF_TOSCA_ARTIFACTS");
1049
1050                 hasProps.clear();
1051                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
1052                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1053
1054                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1055                 result = validateTosca(vertices, serviceToFix, "SERVICE_TOSCA_ARTIFACTS");
1056
1057                 return result;
1058         }
1059
1060         public Map<String, List<Component>> getVerticesToValidate(VertexTypeEnum type,
1061                         Map<GraphPropertyEnum, Object> hasProps) {
1062
1063                 Map<String, List<Component>> result = new HashMap<>();
1064                 try {
1065
1066                         Either<List<GraphVertex>, TitanOperationStatus> resultsEither = titanDao.getByCriteria(type, hasProps);
1067                         if (resultsEither.isRight()) {
1068                                 log.error("getVerticesToValidate failed {} ",resultsEither.right().value());
1069                                 return result;
1070                         }
1071                         log.info("getVerticesToValidate: {}  vertices to scan", resultsEither.left().value().size());
1072                         List<GraphVertex> componentsList = resultsEither.left().value();
1073                         componentsList.forEach(vertex -> {
1074                                 String ivariantUuid = (String) vertex.getMetadataProperty(GraphPropertyEnum.INVARIANT_UUID);
1075                                 if (!result.containsKey(ivariantUuid)) {
1076                                         List<Component> compList = new ArrayList<Component>();
1077                                         result.put(ivariantUuid, compList);
1078                                 }
1079                                 List<Component> compList = result.get(ivariantUuid);
1080
1081                                 ComponentParametersView filter = new ComponentParametersView(true);
1082                                 filter.setIgnoreArtifacts(false);
1083
1084                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1085                                                 .getToscaElement(vertex.getUniqueId(), filter);
1086                                 if (toscaElement.isRight()) {
1087                                         log.error("getVerticesToValidate: failed to find element {}  staus is {}", vertex.getUniqueId()
1088                                                         ,toscaElement.right().value());
1089                                 } else {
1090                                         compList.add(toscaElement.left().value());
1091                                 }
1092                                 titanDao.commit();
1093
1094                         });
1095
1096                 } catch (Exception e) {
1097                         log.info("Failed to fetch vf resources ", e);
1098
1099                 } finally {
1100                         titanDao.commit();
1101
1102                 }
1103                 return result;
1104         }
1105
1106         public boolean validateTosca(Map<String, List<Component>> vertices, Map<String, List<Component>> compToFix,
1107                         String name) {
1108                 boolean result = true;
1109                 long time = System.currentTimeMillis();
1110                 String fileName = name + "_" + time + ".csv";
1111                 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"))) {
1112                         writer.write("name, UUID, invariantUUID, state, version\n");
1113                         for (Map.Entry<String, List<Component>> entry : vertices.entrySet()) {
1114                                 List<Component> compList = entry.getValue();
1115                                 Set<String> artifactEsId = new HashSet<>();
1116                                 for (Component component : compList) {
1117                                         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
1118                                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1119                                                         .filter(a -> artifactEsId.contains(a.getEsId()) && a.getEsId() != null).findAny();
1120                                         if (op.isPresent()) {
1121                                                 result = false;
1122                                                 writeModuleResultToFile(writer, compList);
1123                                                 writer.flush();
1124                                                 break;
1125                                         } else {
1126                                                 artifactEsId.addAll(toscaArtifacts.values().stream().map(ArtifactDefinition::getEsId)
1127                                                                 .collect(Collectors.toList()));
1128                                         }
1129                                 }
1130                                 if (!result) {
1131                                         List<Component> compListfull = new ArrayList<>();
1132                                         for (Component c : compList) {
1133                                                 ComponentParametersView filter = new ComponentParametersView(true);
1134                                                 filter.setIgnoreComponentInstances(false);
1135                                                 filter.setIgnoreArtifacts(false);
1136                                                 filter.setIgnoreGroups(false);
1137
1138                                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1139                                                                 .getToscaElement(c.getUniqueId(), filter);
1140                                                 if (toscaElement.isRight()) {
1141                                                         System.out.println("getVerticesToValidate: failed to find element" + c.getUniqueId()
1142                                                                         + " staus is" + toscaElement.right().value());
1143                                                 } else {
1144                                                         compListfull.add(toscaElement.left().value());
1145                                                 }
1146                                                 this.titanDao.commit();
1147                                         }
1148
1149                                         compToFix.put(entry.getKey(), compListfull);
1150                                         result = true;
1151                                 }
1152
1153                         }
1154
1155                 } catch (Exception e) {
1156                         log.info("Failed to fetch vf resources ", e);
1157                         return false;
1158                 } finally {
1159                         titanDao.commit();
1160                 }
1161                 return result;
1162         }
1163
1164         private Either<Component, ToscaError> generateToscaArtifact(Component parent) {
1165                 log.debug("tosca artifact generation");
1166                 try {
1167                         Map<String, ArtifactDefinition> toscaArtifacts = parent.getToscaArtifacts();
1168
1169                         ArtifactDefinition toscaArtifact = null;
1170                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1171                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_TEMPLATE.getType())).findAny();
1172
1173                         if (op.isPresent()) {
1174                                 toscaArtifact = op.get();
1175                         }
1176                         if (toscaArtifact != null) {
1177                                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent artifact name {} id {} esId {}",
1178                                                 toscaArtifact.getArtifactName(), toscaArtifact.getUniqueId(), toscaArtifact.getEsId());
1179
1180                                 Either<ToscaRepresentation, ToscaError> exportComponent = toscaExportUtils.exportComponent(parent);
1181                                 if (exportComponent.isRight()) {
1182                                         log.debug("Failed export tosca yaml for component {} error {}", parent.getUniqueId(),
1183                                                         exportComponent.right().value());
1184
1185                                         return Either.right(exportComponent.right().value());
1186                                 }
1187                                 log.debug("Tosca yaml exported for component {} ", parent.getUniqueId());
1188                                 String payload = exportComponent.left().value().getMainYaml();
1189
1190                                 toscaArtifact.setPayloadData(payload);
1191                                 byte[] decodedPayload = toscaArtifact.getPayloadData();
1192
1193                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1194                                                 toscaArtifact.getArtifactLabel());
1195                                 toscaArtifact.setUniqueId(uniqueId);
1196                                 toscaArtifact.setEsId(toscaArtifact.getUniqueId());
1197
1198                                 toscaArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1199                                 ESArtifactData artifactData = new ESArtifactData(toscaArtifact.getEsId(), decodedPayload);
1200                                 artifactCassandraDao.saveArtifact(artifactData);
1201
1202                                 log.debug("Tosca yaml artifact esId  {} ", toscaArtifact.getEsId());
1203                         }
1204                         ArtifactDefinition csarArtifact = null;
1205                         op = toscaArtifacts.values().stream()
1206                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_CSAR.getType())).findAny();
1207
1208                         if (op.isPresent()) {
1209                                 csarArtifact = op.get();
1210                         }
1211
1212                         if (csarArtifact != null) {
1213                                 Either<byte[], ResponseFormat> generated = csarUtils.createCsar(parent, true, true);
1214
1215                                 if (generated.isRight()) {
1216                                         log.debug("Failed to export tosca csar for component {} error {}", parent.getUniqueId(),
1217                                                         generated.right().value());
1218
1219                                         return Either.right(ToscaError.GENERAL_ERROR);
1220                                 }
1221                                 byte[] value = generated.left().value();
1222                                 csarArtifact.setPayload(value);
1223                                 byte[] decodedPayload = csarArtifact.getPayloadData();
1224
1225                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1226                                                 csarArtifact.getArtifactLabel());
1227                                 csarArtifact.setUniqueId(uniqueId);
1228                                 csarArtifact.setEsId(csarArtifact.getUniqueId());
1229
1230                                 csarArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1231                                 ESArtifactData artifactData = new ESArtifactData(csarArtifact.getEsId(), decodedPayload);
1232                                 artifactCassandraDao.saveArtifact(artifactData);
1233                                 log.debug("Tosca csar artifact esId  {} ", csarArtifact.getEsId());
1234
1235                         }
1236
1237                 } catch (Exception ex) {
1238                         log.error("Failed to generate tosca atifact component id {} component name {} error {}",
1239                                         parent.getUniqueId(), parent.getName(), ex.getMessage());
1240
1241                         return Either.right(ToscaError.GENERAL_ERROR);
1242                 }
1243
1244                 return Either.left(parent);
1245         }
1246
1247            private ArtifactDefinition createVfModuleArtifact(ComponentInstance currVF, Service service) {
1248
1249                 ArtifactDefinition vfModuleArtifactDefinition = new ArtifactDefinition();
1250                 String newCheckSum = null;
1251
1252                 vfModuleArtifactDefinition.setDescription("Auto-generated VF Modules information artifact");
1253                 vfModuleArtifactDefinition.setArtifactDisplayName("Vf Modules Metadata");
1254                 vfModuleArtifactDefinition.setArtifactType(ArtifactTypeEnum.VF_MODULES_METADATA.getType());
1255                 vfModuleArtifactDefinition.setArtifactGroupType(ArtifactGroupTypeEnum.DEPLOYMENT);
1256                 vfModuleArtifactDefinition.setArtifactLabel("vfModulesMetadata");
1257                 vfModuleArtifactDefinition.setTimeout(0);
1258                 vfModuleArtifactDefinition.setArtifactName(currVF.getNormalizedName() + "_modules.json");
1259
1260                return vfModuleArtifactDefinition;
1261             }
1262
1263
1264         private void fillVfModuleInstHeatEnvPayload(Component parent, ComponentInstance instance, List<GroupInstance> groupsForCurrVF,
1265                         ArtifactDefinition vfModuleArtifact) {
1266                 log.debug("generate new vf module for component. name  {}, id {}, Version {}", instance.getName(), instance.getUniqueId());
1267
1268                 String uniqueId = UniqueIdBuilder.buildInstanceArtifactUniqueId(parent.getUniqueId(), instance.getUniqueId(), vfModuleArtifact.getArtifactLabel());
1269
1270                 vfModuleArtifact.setUniqueId(uniqueId);
1271                 vfModuleArtifact.setEsId(vfModuleArtifact.getUniqueId());
1272
1273                 List<VfModuleArtifactPayload> vfModulePayloadForCurrVF = new ArrayList<>();
1274                 if (groupsForCurrVF != null) {
1275                         for (GroupInstance groupInstance : groupsForCurrVF) {
1276                                 VfModuleArtifactPayload modulePayload = new VfModuleArtifactPayload(groupInstance);
1277                                 vfModulePayloadForCurrVF.add(modulePayload);
1278                         }
1279                         Collections.sort(vfModulePayloadForCurrVF,
1280                                         (art1, art2) -> VfModuleArtifactPayload.compareByGroupName(art1, art2));
1281
1282                         final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1283
1284                         String vfModulePayloadString = gson.toJson(vfModulePayloadForCurrVF);
1285                         log.debug("vfModulePayloadString {}", vfModulePayloadString);
1286                         if (vfModulePayloadString != null) {
1287                                 String newCheckSum = GeneralUtility
1288                                                 .calculateMD5Base64EncodedByByteArray(vfModulePayloadString.getBytes());
1289                                 vfModuleArtifact.setArtifactChecksum(newCheckSum);
1290
1291                                 ESArtifactData artifactData = new ESArtifactData(vfModuleArtifact.getEsId(),
1292                                                 vfModulePayloadString.getBytes());
1293                                 artifactCassandraDao.saveArtifact(artifactData);
1294
1295                         }
1296
1297                 }
1298
1299         }
1300
1301         private Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> parseVFModuleJson(ArtifactDefinition vfModuleArtifact) {
1302                 log.info("Try to get vfModule json from cassandra {}", vfModuleArtifact.getEsId());
1303                 Either<ESArtifactData, CassandraOperationStatus> vfModuleData = artifactCassandraDao.getArtifact(vfModuleArtifact.getEsId());
1304
1305                 if (vfModuleData.isRight()) {
1306                         CassandraOperationStatus resourceUploadStatus = vfModuleData.right().value();
1307                         StorageOperationStatus storageResponse = DaoStatusConverter.convertCassandraStatusToStorageStatus(resourceUploadStatus);
1308                         log.error("failed to fetch vfModule json {} from cassandra. Status is {}", vfModuleArtifact.getEsId(), storageResponse);
1309                         return Either.right(storageResponse);
1310
1311                 }
1312
1313                 ESArtifactData esArtifactData = vfModuleData.left().value();
1314                 String gsonData = new String( esArtifactData.getDataAsArray());
1315                 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1316                 JsonArray jsonElement = new JsonArray();
1317                 jsonElement = gson.fromJson(gsonData, jsonElement.getClass());
1318                 List<VfModuleArtifactPayloadEx> vfModules = new ArrayList<>();
1319                 jsonElement.forEach(je ->{
1320                         VfModuleArtifactPayloadEx vfModule = ComponentsUtils.parseJsonToObject(je.toString(), VfModuleArtifactPayloadEx.class);
1321                         vfModules.add(vfModule);
1322                 });
1323
1324                 log.debug  ("parse vf module finish {}", gsonData);
1325                 return Either.left(vfModules);
1326
1327         }
1328
1329         /*public boolean manualFix() {
1330
1331                 Set<String> fixedIds = new HashSet<>();
1332                 Component component;
1333                 String componentId = "86683566-20e8-4cc5-872d-12abca1d57f0";//"9f6a6976-18e3-488a-98a4-c1aade480739";
1334                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade.getToscaFullElement(componentId);
1335                 if (toscaElement.isRight()) {
1336                         log.info("Failed to fetch resources {} {}", componentId, toscaElement.right().value());
1337
1338                 }
1339                 boolean res = generateToscaPerComponent(fixedIds, toscaElement.left().value());
1340                 log.info("Fix component return res {} ", res);
1341                 titanDao.commit();
1342
1343                 return res;
1344         }
1345
1346
1347         public boolean manualCheck() {
1348
1349         Set<String> fixedIds = new HashSet<>();
1350         Component component;
1351         String componentId = "86d50186-7b00-4bfc-abcb-9e4c6892f338";//"9f6a6976-18e3-488a-98a4-c1aade480739";
1352         Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade.getToscaFullElement(componentId);
1353         if (toscaElement.isRight()) {
1354                 log.info("Failed to fetch resources {} {}", componentId, toscaElement.right().value());
1355
1356         }
1357         boolean res = true;
1358         try {
1359                 res = isProblematicService(toscaElement.left().value(), toscaElement.left().value().getName());
1360                 if(res){
1361                         List<Service> services = new ArrayList<>();
1362                         services.add(toscaElement.left().value());
1363                         this.fix(new ArrayList<Resource>(), services, null, null, null);
1364
1365                         Either<Service, StorageOperationStatus> toscaElementNew  = toscaOperationFacade.getToscaFullElement(componentId);
1366                         if (toscaElement.isRight()) {
1367                                 log.info("Failed to fetch resources {} {}", componentId, toscaElementNew.right().value());
1368
1369                         }
1370                         res = isProblematicService(toscaElementNew.left().value(), toscaElementNew.left().value().getName());
1371                 }
1372                 titanDao.commit();
1373
1374         } catch (IOException e) {
1375                 // TODO Auto-generated catch block
1376                 e.printStackTrace();
1377         }
1378         log.info("Fix component return res {} ", res);
1379
1380         return res;
1381 }*/
1382
1383 }
1384
1385