asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/ArtifactUuidFix.java

   1 package org.openecomp.sdc.asdctool.impl;
   2
   3 import java.io.BufferedWriter;
   4 import java.io.FileOutputStream;
   5 import java.io.IOException;
   6 import java.io.OutputStreamWriter;
   7 import java.io.Writer;
   8 import java.util.ArrayList;
   9 import java.util.Collection;
  10 import java.util.Collections;
  11 import java.util.HashMap;
  12 import java.util.HashSet;
  13 import java.util.List;
  14 import java.util.Map;
  15 import java.util.Optional;
  16 import java.util.Set;
  17 import java.util.stream.Collectors;
  18
  19 import org.openecomp.sdc.asdctool.impl.validator.utils.VfModuleArtifactPayloadEx;
  20 import org.openecomp.sdc.be.components.distribution.engine.VfModuleArtifactPayload;
  21 import org.openecomp.sdc.be.dao.api.ActionStatus;
  22 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  23 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  24 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
  25 import org.openecomp.sdc.be.dao.jsongraph.TitanDao;
  26 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
  27 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
  28 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
  29 import org.openecomp.sdc.be.dao.titan.TitanOperationStatus;
  30 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
  31 import org.openecomp.sdc.be.datatypes.elements.GroupDataDefinition;
  32 import org.openecomp.sdc.be.datatypes.elements.MapArtifactDataDefinition;
  33 import org.openecomp.sdc.be.datatypes.elements.MapGroupsDataDefinition;
  34 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
  35 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
  36 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
  37 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
  38 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
  39 import org.openecomp.sdc.be.datatypes.tosca.ToscaDataDefinition;
  40 import org.openecomp.sdc.be.impl.ComponentsUtils;
  41 import org.openecomp.sdc.be.model.*;
  42 import org.openecomp.sdc.be.model.jsontitan.datamodel.TopologyTemplate;
  43 import org.openecomp.sdc.be.model.jsontitan.datamodel.ToscaElement;
  44 import org.openecomp.sdc.be.model.jsontitan.operations.ToscaOperationFacade;
  45 import org.openecomp.sdc.be.model.jsontitan.utils.ModelConverter;
  46 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  47 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
  48 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
  49 import org.openecomp.sdc.be.resources.data.ESArtifactData;
  50 import org.openecomp.sdc.be.resources.data.auditing.AuditingActionEnum;
  51 import org.openecomp.sdc.be.tosca.CsarUtils;
  52 import org.openecomp.sdc.be.tosca.ToscaError;
  53 import org.openecomp.sdc.be.tosca.ToscaExportHandler;
  54 import org.openecomp.sdc.be.tosca.ToscaRepresentation;
  55 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
  56 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
  57 import org.openecomp.sdc.common.api.Constants;
  58
  59 import org.openecomp.sdc.common.util.GeneralUtility;
  60 import org.openecomp.sdc.exception.ResponseFormat;
  61 import org.slf4j.Logger;
  62 import org.slf4j.LoggerFactory;
  63 import org.springframework.beans.factory.annotation.Autowired;
  64 import org.springframework.util.StringUtils;
  65
  66 import com.google.gson.Gson;
  67 import com.google.gson.GsonBuilder;
  68 import com.google.gson.JsonArray;
  69 import com.google.gson.JsonObject;
  70
  71 import fj.data.Either;
  72
  73 @org.springframework.stereotype.Component("artifactUuidFix")
  74 public class ArtifactUuidFix {
  75
  76         @Autowired
  77         private TitanDao titanDao;
  78
  79         @Autowired
  80         private ToscaOperationFacade toscaOperationFacade;
  81         @Autowired
  82         private ToscaExportHandler toscaExportUtils;
  83         @Autowired
  84         private ArtifactCassandraDao artifactCassandraDao;
  85
  86
  87         @Autowired
  88         private CsarUtils csarUtils;
  89
  90         private static Logger log = LoggerFactory.getLogger(ArtifactUuidFix.class.getName());
  91
  92         public boolean doFix(String fixComponent, String runMode) {
  93                 List<Resource> vfLst = new ArrayList<>();
  94                 List<Service> serviceList = new ArrayList<>();
  95                 Map<String, List<Component>> nodeToFixTosca = new HashMap<>();
  96                 Map<String, List<Component>> vfToFixTosca = new HashMap<>();
  97                 Map<String, List<Component>> serviceToFixTosca = new HashMap<>();
  98
  99                 long time = System.currentTimeMillis();
 100
 101                 doFixTosca(nodeToFixTosca, vfToFixTosca, serviceToFixTosca);
 102
 103                 if (fixComponent.equals("vf_only")) {
 104                         if (fetchFaultVf(fixComponent, vfLst, time) == false) {
 105                                 return false;
 106                         }
 107                 } else {
 108                         if (fetchServices(fixComponent, serviceList, time) == false) {
 109                                 return false;
 110                         }
 111                 }
 112                 if (runMode.equals("service_vf") || runMode.equals("fix")) {
 113                         log.info("Mode {}. Find problem VFs", runMode);
 114                         if (fetchVf(serviceList, vfLst, time) == false) {
 115                                 log.info("Mode {}. Find problem VFs finished with failure", runMode);
 116                                 return false;
 117                         }
 118                         log.info("Mode {}. Find problem VFs finished with success", runMode);
 119                 }
 120                 if (runMode.equals("fix") || runMode.equals("fix_only_services")) {
 121                         log.info("Mode {}. Start fix", runMode);
 122                         if (fix(vfLst, serviceList, nodeToFixTosca, vfToFixTosca, serviceToFixTosca) == false) {
 123                                 log.info("Mode {}. Fix finished with failure", runMode);
 124                                 return false;
 125                         }
 126                         log.info("Mode {}. Fix finished with success", runMode);
 127                 }
 128
 129                 return true;
 130         }
 131
 132         private boolean fetchFaultVf(String fixComponent, List<Resource> vfLst, long time) {
 133                 log.info("Find fault VF ");
 134                 Writer writer = null;
 135                 try {
 136                         String fileName = "fault_" + time + ".csv";
 137                         writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"));
 138                         writer.write("vf name, vf id, state, version\n");
 139
 140                         Map<GraphPropertyEnum, Object> hasProps = new HashMap<>();
 141                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
 142                         hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF.name());
 143
 144                         Map<GraphPropertyEnum, Object> hasNotProps = new HashMap<>();
 145                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 146                         log.info("Try to fetch resources with properties {} and not {}", hasProps, hasNotProps);
 147
 148                         Either<List<GraphVertex>, TitanOperationStatus> servicesByCriteria = titanDao
 149                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 150                         if (servicesByCriteria.isRight()) {
 151                                 log.info("Failed to fetch resources {}", servicesByCriteria.right().value());
 152                                 return false;
 153                         }
 154                         List<GraphVertex> resources = servicesByCriteria.left().value();
 155                         for (GraphVertex gv : resources) {
 156                                 ComponentParametersView filter = new ComponentParametersView(true);
 157                                 filter.setIgnoreComponentInstances(false);
 158                                 filter.setIgnoreArtifacts(false);
 159                                 filter.setIgnoreGroups(false);
 160
 161                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 162                                                 .getToscaElement(gv.getUniqueId());
 163                                 if (toscaElement.isRight()) {
 164                                         log.info("Failed to fetch resources {} {}", gv.getUniqueId(), toscaElement.right().value());
 165                                         continue;
 166                                 }
 167
 168                                 Resource resource = toscaElement.left().value();
 169                                 String resourceName = resource.getName();
 170                                 Map<String, ArtifactDefinition> deploymentArtifacts = resource.getDeploymentArtifacts();
 171                                 List<GroupDefinition> groups = resource.getGroups();
 172                                 if (groups == null || groups.isEmpty()) {
 173                                         log.info("No groups for resource {} id {} ", resourceName, gv.getUniqueId());
 174                                         continue;
 175                                 }
 176                                 boolean isProblematic = false;
 177                                 for (GroupDefinition gr : groups) {
 178                                         if (gr.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 179                                                 if (isProblematicGroup(gr, resourceName, deploymentArtifacts)) {
 180                                                         isProblematic = true;
 181                                                         break;
 182                                                 }
 183                                         }
 184                                 }
 185                                 if (isProblematic) {
 186                                         vfLst.add(resource);
 187                                         writeModuleResultToFile(writer, resource, null);
 188                                         writer.flush();
 189                                 }
 190                                 titanDao.commit();
 191                         }
 192
 193                 } catch (Exception e) {
 194                         log.info("Failed to fetch vf resources ", e);
 195                         return false;
 196                 } finally {
 197                         titanDao.commit();
 198                         try {
 199                                 writer.flush();
 200                                 writer.close();
 201                         } catch (Exception ex) {
 202                                 /* ignore */
 203                         }
 204                 }
 205                 return true;
 206         }
 207
 208         private boolean fetchVf(List<Service> serviceList, List<Resource> vfLst, long time) {
 209                 log.info("Find problem VF ");
 210                 if (serviceList.isEmpty()) {
 211                         log.info("No services as input");
 212                         return true;
 213                 }
 214                 Writer writer = null;
 215                 try {
 216                         String fileName = "problemVf_" + time + ".csv";
 217                         writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"));
 218                         writer.write("vf name, vf id, state, version, example service name\n");
 219                         Set<String> vfIds = new HashSet<>();
 220                         for (Service service : serviceList) {
 221                                 List<ComponentInstance> componentInstances = service.getComponentInstances().stream()
 222                                                 .filter(ci -> ci.getOriginType().equals(OriginTypeEnum.VF)).collect(Collectors.toList());
 223                                 for (ComponentInstance ci : componentInstances) {
 224                                         if (!vfIds.contains(ci.getComponentUid())) {
 225                                                 vfIds.add(ci.getComponentUid());
 226                                                 ComponentParametersView filter = new ComponentParametersView(true);
 227                                                 filter.setIgnoreComponentInstances(false);
 228                                                 filter.setIgnoreArtifacts(false);
 229                                                 filter.setIgnoreGroups(false);
 230                                                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
 231                                                                 .getToscaElement(ci.getComponentUid(), filter);
 232                                                 if (toscaElement.isRight()) {
 233                                                         log.info("Failed to fetch resource {} {}", ci.getComponentUid(),
 234                                                                         toscaElement.right().value());
 235                                                         continue;
 236                                                 }
 237                                                 Resource resource = toscaElement.left().value();
 238                                                 if (resource.getResourceType().equals(ResourceTypeEnum.VF)) {
 239                                                         vfLst.add(resource);
 240                                                         writeModuleResultToFile(writer, resource, service);
 241                                                         writer.flush();
 242
 243                                                 }
 244                                                 titanDao.commit();
 245                                         }
 246                                 }
 247                         }
 248                         log.info("output file with list of Vf : {}", fileName);
 249                 } catch (Exception e) {
 250                         log.info("Failed to fetch services ", e);
 251                         return false;
 252                 } finally {
 253                         titanDao.commit();
 254                         try {
 255                                 writer.flush();
 256                                 writer.close();
 257                         } catch (Exception ex) {
 258                                 /* ignore */
 259                         }
 260                 }
 261                 return true;
 262         }
 263
 264         private boolean fetchServices(String fixServices, List<Service> serviceList, long time) {
 265                 log.info("Find problem Services {}", fixServices);
 266                 Writer writer = null;
 267
 268                 try {
 269                         String fileName = "problemService_" + time + ".csv";
 270                         writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"));
 271                         writer.write("service name, service id, state, version\n");
 272
 273                         Map<GraphPropertyEnum, Object> hasProps = new HashMap<>();
 274                         hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
 275                         if (fixServices.equals("distributed_only")) {
 276                                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
 277                                 hasProps.put(GraphPropertyEnum.DISTRIBUTION_STATUS, DistributionStatusEnum.DISTRIBUTED.name());
 278                         }
 279
 280                         Map<GraphPropertyEnum, Object> hasNotProps = new HashMap<>();
 281                         hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
 282                         log.info("Try to fetch services with properties {} and not {}", hasProps, hasNotProps);
 283
 284                         Either<List<GraphVertex>, TitanOperationStatus> servicesByCriteria = titanDao
 285                                         .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
 286                         if (servicesByCriteria.isRight()) {
 287                                 log.info("Failed to fetch services {}", servicesByCriteria.right().value());
 288                                 return false;
 289                         }
 290                         List<GraphVertex> services = servicesByCriteria.left().value();
 291                         for (GraphVertex gv : services) {
 292                                 ComponentParametersView filter = new ComponentParametersView(true);
 293                                 filter.setIgnoreComponentInstances(false);
 294                                 filter.setIgnoreArtifacts(false);
 295                                 filter.setIgnoreGroups(false);
 296
 297                                 Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade
 298                                                 .getToscaElement(gv.getUniqueId(), filter);
 299                                 if (toscaElement.isRight()) {
 300                                         log.info("Failed to fetch service {} {}", gv.getUniqueId(), toscaElement.right().value());
 301                                         continue;
 302                                 }
 303                                 Service service = toscaElement.left().value();
 304
 305                                 String serviceName = (String) gv.getMetadataProperty(GraphPropertyEnum.NAME);
 306
 307                                 boolean isProblematic = isProblematicService(service, serviceName);
 308                                 if (isProblematic) {
 309                                         serviceList.add(service);
 310                                         writeModuleResultToFile(writer, service, null);
 311                                         writer.flush();
 312
 313                                 }
 314
 315                                 titanDao.commit();
 316                         }
 317                         log.info("output file with list of services : {}", fileName);
 318                 } catch (Exception e) {
 319                         log.info("Failed to fetch services ", e);
 320                         return false;
 321                 } finally {
 322                         titanDao.commit();
 323                         try {
 324                                 writer.flush();
 325                                 writer.close();
 326                         } catch (Exception ex) {
 327                                 /* ignore */
 328                         }
 329                 }
 330                 return true;
 331         }
 332
 333         private boolean isProblematicService( Service service, String serviceName) throws IOException {
 334
 335                 List<ComponentInstance> componentInstances = service.getComponentInstances();
 336
 337                 if (componentInstances == null) {
 338                         log.info("No instances for service {} ", service.getUniqueId());
 339                         return false;
 340                 }
 341                 boolean isCheckVFModules = true;
 342                 if(service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN ||
 343                                 service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT){
 344                         isCheckVFModules = false;
 345                 }
 346                 for (ComponentInstance ci : componentInstances) {
 347                         Map<String, ArtifactDefinition> deploymentArtifacts = ci.getDeploymentArtifacts();
 348                         List<GroupInstance> groupInstances = ci.getGroupInstances();
 349                         if (groupInstances == null || groupInstances.isEmpty()) {
 350                                 log.info("No instance groups for instance {} in service {} id {} ", ci.getName(), serviceName,
 351                                                 service.getUniqueId());
 352                                 continue;
 353                         }
 354                         List<VfModuleArtifactPayloadEx> vfModules = null;
 355                         if(isCheckVFModules){
 356                                 Optional<ArtifactDefinition> optionalVfModuleArtifact = deploymentArtifacts.values().stream()
 357                                                 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 358
 359                                  if(!optionalVfModuleArtifact.isPresent())
 360                                          return true;
 361
 362                                  ArtifactDefinition vfModuleArtifact = optionalVfModuleArtifact.get();
 363                                  Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> vfModulesEither = parseVFModuleJson(vfModuleArtifact);
 364                                  if(vfModulesEither.isRight()){
 365                                          log.error("Failed to parse vfModule for service {} status is {}", service.getUniqueId(), vfModulesEither.right().value());
 366                                          return true;
 367                                  }
 368                                  vfModules = vfModulesEither.left().value();
 369                                  if(vfModules == null || vfModules.isEmpty() ){
 370                                          log.info("vfModules empty for service {}", service.getUniqueId());
 371                                          return true;
 372                                  }
 373                         }
 374
 375                         for (GroupInstance gi : groupInstances) {
 376                                 if (gi.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 377                                         VfModuleArtifactPayloadEx vfModule= null;
 378                                         if(isCheckVFModules && vfModules != null){
 379                                                 Optional<VfModuleArtifactPayloadEx> op = vfModules.stream().filter(vf -> vf.getVfModuleModelName().equals(gi.getGroupName())).findAny();
 380                                                 if(!op.isPresent()){
 381                                                          log.error("Failed to find vfModule for group {}", gi.getGroupName());
 382                                                         return true;
 383                                                 }
 384                                                 vfModule = op.get();
 385                                         }
 386                                         if (isProblematicGroupInstance(gi, ci.getName(), serviceName, deploymentArtifacts, vfModule)) {
 387                                                 return true;
 388                                         }
 389                                 }
 390                         }
 391
 392                 }
 393                 return false;
 394         }
 395
 396         private boolean isProblematicGroup(GroupDefinition gr, String resourceName,
 397                         Map<String, ArtifactDefinition> deploymentArtifacts) {
 398                 List<String> artifacts = gr.getArtifacts();
 399                 List<String> artifactsUuid = gr.getArtifactsUuid();
 400                 Set<String> artifactsSet = new HashSet<>();
 401
 402                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 403                         log.info("No groups in resource {} ", resourceName);
 404                         return true;
 405                 }
 406                 artifactsSet.addAll(artifacts);
 407                 if (artifactsSet.size() < artifacts.size()) {
 408                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", gr.getName(), resourceName);
 409                         return true;
 410                 }
 411
 412                 if (artifacts.size() < artifactsUuid.size()) {
 413                         log.info(" artifacts.size() < artifactsUuid.size() group {} in resource {} ", gr.getName(), resourceName);
 414                         return true;
 415                 }
 416                 if (artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 417                         log.info(
 418                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() group {} in resource {} ",
 419                                         gr.getName(), resourceName);
 420                         return true;
 421                 }
 422                 if (artifactsUuid.contains(null)) {
 423                         log.info(" artifactsUuid.contains(null) group {} in resource {} ", gr.getName(), resourceName);
 424                         return true;
 425                 }
 426
 427                 for (String artifactId : artifacts) {
 428                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 429                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 430                         if (artifactDefinition == null) {
 431                                 log.info(" artifactDefinition == null label {} group {} in resource {} ", artifactlabel, gr.getName(),
 432                                                 resourceName);
 433                                 return true;
 434                         }
 435                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 436                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 437                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 438                                         log.info(
 439                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} group {} in resource {} ",
 440                                                         artifactlabel, artifactId, gr.getName(), resourceName);
 441                                         return true;
 442                                 }
 443                                 if (!artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 444                                         log.info(
 445                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} group {} in resource {} ",
 446                                                         artifactlabel, gr.getName(), resourceName);
 447                                         return true;
 448                                 }
 449                         }
 450                 }
 451                 for (String artifactUUID : artifactsUuid) {
 452                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 453                         if (label != null && !label.isEmpty() && !label.equals("")) {
 454                                 return true;
 455                         }
 456                 }
 457
 458                 return false;
 459         }
 460
 461         private boolean isProblematicGroupInstance(GroupInstance gi, String instName, String servicename,
 462                         Map<String, ArtifactDefinition> deploymentArtifacts, VfModuleArtifactPayloadEx vfModule) {
 463                 List<String> artifacts = gi.getArtifacts();
 464                 List<String> artifactsUuid = gi.getArtifactsUuid();
 465                 List<String> instArtifactsUuid = gi.getGroupInstanceArtifactsUuid();
 466                 List<String> instArtifactsId = gi.getGroupInstanceArtifacts();
 467                 Set<String> instArtifatIdSet = new HashSet<>();
 468                 Set<String> artifactsSet = new HashSet<>();
 469
 470                 log.info("check group {} for instance {} ", gi.getGroupName(), instName);
 471                 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
 472                         log.info("No instance groups for instance {} in service {} ", instName, servicename);
 473                         return true;
 474                 }
 475                 artifactsSet.addAll(artifacts);
 476                 if (artifactsSet.size() < artifacts.size()) {
 477                         log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", instName, servicename);
 478                         return true;
 479                 }
 480
 481                 if (instArtifactsId != null && !instArtifactsId.isEmpty()) {
 482                         instArtifatIdSet.addAll(instArtifactsId);
 483                 }
 484
 485                 if (artifacts.size() < artifactsUuid.size()) {
 486                         log.info(" artifacts.size() < artifactsUuid.size() inst {} in service {} ", instName, servicename);
 487                         return true;
 488                 }
 489                 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
 490                         log.info(
 491                                         " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() inst {} in service {} ",
 492                                         instName, servicename);
 493                         return true;
 494                 }
 495                 if (artifactsUuid.contains(null)) {
 496                         log.info(" artifactsUuid.contains(null) inst {} in service {} ", instName, servicename);
 497                         return true;
 498                 }
 499                 if (instArtifactsId != null && instArtifatIdSet.size() < instArtifactsId.size()) {
 500                         log.info(" instArtifatIdSet.size() < instArtifactsId.size() inst {} in service {} ", instName, servicename);
 501                         return true;
 502                 }
 503
 504                 if ((instArtifactsId != null && instArtifactsUuid != null)
 505                                 && instArtifactsId.size() != instArtifactsUuid.size()) {
 506                         log.info(" instArtifactsId.size() != instArtifactsUuid.size() inst {} in service {} ", instName,
 507                                         servicename);
 508                         return true;
 509                 }
 510
 511                 for (String artifactId : artifacts) {
 512                         String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 513                         ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
 514                         if (artifactDefinition == null) {
 515                                 log.info(" artifactDefinition == null label {} inst {} in service {} ", artifactlabel, instName,
 516                                                 servicename);
 517                                 return true;
 518                         }
 519                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
 520                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 521                                 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
 522                                         log.info(
 523                                                         " !artifactId.equals(artifactDefinition.getUniqueId() artifact {}  artId {} inst {} in service {} ",
 524                                                         artifactlabel, artifactId, instName, servicename);
 525                                         return true;
 526                                 }
 527                                 if (!artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 528                                         log.info(
 529                                                         " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 530                                                         artifactlabel, instName, servicename);
 531                                         return true;
 532                                 }
 533                         } else {
 534                                 if (instArtifactsUuid == null || instArtifactsUuid.isEmpty()) {
 535                                         log.info(" instArtifactsUuid empty. label {} inst {} in service {} ", artifactlabel, instName,
 536                                                         servicename);
 537                                         return true;
 538                                 }
 539                                 if (!instArtifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
 540                                         log.info(
 541                                                         " instArtifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
 542                                                         artifactlabel, instName, servicename);
 543                                         return true;
 544                                 }
 545                         }
 546                 }
 547                 for (String artifactUUID : artifactsUuid) {
 548                         String label = findArtifactLabelFromArtifactId(artifactUUID);
 549                         if (label != null && !label.isEmpty() && !label.equals("")) {
 550                                 return true;
 551                         }
 552                 }
 553                 if(vfModule != null ){
 554                         return isProblematicVFModule(vfModule, artifactsUuid, instArtifactsUuid);
 555                 }
 556
 557                 return false;
 558         }
 559
 560         private boolean isProblematicVFModule(VfModuleArtifactPayloadEx vfModule, List<String> artifactsUuid,
 561                         List<String> instArtifactsUuid) {
 562                 log.info(" isProblematicVFModule  {}  ", vfModule.getVfModuleModelName());
 563                 List<String> vfModuleArtifacts = vfModule.getArtifacts();
 564                 List<String> allArtifacts = new ArrayList<>();
 565                 allArtifacts.addAll(artifactsUuid);
 566                 if(instArtifactsUuid != null)
 567                         allArtifacts.addAll(instArtifactsUuid);
 568                 if((vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()){
 569                         log.error(" vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()");
 570                         return true;
 571                 }
 572                 if(vfModuleArtifacts!= null){
 573                         if( vfModuleArtifacts.size() != allArtifacts.size()){
 574                                 log.error(" vfModuleArtifacts.size() != allArtifacts.size()");
 575                                 return true;
 576                         }
 577                         for(String vfModuleArtifact: vfModuleArtifacts){
 578                                 Optional<String> op = allArtifacts.stream().filter(a -> a.equals(vfModuleArtifact)).findAny();
 579                                 if(!op.isPresent()){
 580                                         log.error("failed to find artifact {} in group artifacts {}", vfModuleArtifact,  allArtifacts);
 581                                         return true;
 582                                 }
 583                         }
 584                 }
 585                 return false;
 586         }
 587
 588
 589
 590         private boolean fix(List<Resource> vfLst, List<Service> serviceList, Map<String, List<Component>> nodesToFixTosca,
 591                         Map<String, List<Component>> vfToFixTosca, Map<String, List<Component>> servicesToFixTosca) {
 592                 boolean res = true;
 593                 log.info(" Fix started ***** ");
 594                 if (vfLst != null && !vfLst.isEmpty()) {
 595                         res = fixVf(vfLst);
 596
 597                 }
 598
 599                 if (res && serviceList != null && !serviceList.isEmpty()) {
 600                         res = fixServices(serviceList);
 601
 602                 }
 603
 604                 Set<String> fixedIds = new HashSet<>();
 605
 606                 long time = System.currentTimeMillis();
 607                 String fileName = "FailedGenerateTosca" + "_" + time + ".csv";
 608                 Writer writer = null;
 609                 try {
 610                         writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"));
 611                         writer.write("componentType, name, version, UID, UUID, invariantUUID, state\n");
 612                         List<Component> failedList = new ArrayList<>();
 613
 614                         if (res && nodesToFixTosca != null && !nodesToFixTosca.isEmpty()) {
 615
 616                                 generateAndSaveToscaArtifacts(nodesToFixTosca, fixedIds, null, failedList);
 617
 618                         }
 619                         if (vfToFixTosca != null && !vfToFixTosca.isEmpty()) {
 620
 621                                 generateAndSaveToscaArtifacts(vfToFixTosca, fixedIds, vfLst, failedList);
 622
 623                         }
 624
 625                         for (Component component : vfLst) {
 626                                 res = generateToscaPerComponent(fixedIds, component);
 627                                 if (res) {
 628                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 629                                         Map<String, GroupDataDefinition> groups = topologyTemplate.getGroups();
 630                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.GROUPS, EdgeLabelEnum.GROUPS, groups);
 631                                         if (res) {
 632                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getDeploymentArtifacts();
 633                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.DEPLOYMENT_ARTIFACTS,
 634                                                                 EdgeLabelEnum.DEPLOYMENT_ARTIFACTS, arifacts);
 635                                         }
 636                                         if (res) {
 637                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 638                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 639                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 640                                         }
 641                                         titanDao.commit();
 642                                 } else {
 643                                         failedList.add(component);
 644                                 }
 645                         }
 646
 647                         if (servicesToFixTosca != null && !servicesToFixTosca.isEmpty()) {
 648                                 generateAndSaveToscaArtifacts(servicesToFixTosca, fixedIds, serviceList, failedList);
 649
 650                         }
 651
 652
 653                         for (Component component : serviceList) {
 654                                 res = generateToscaPerComponent(fixedIds, component);
 655                                 if (res) {
 656                                         TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
 657                                         Map<String, MapGroupsDataDefinition> groups = topologyTemplate.getInstGroups();
 658                                         res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_GROUPS, EdgeLabelEnum.INST_GROUPS,
 659                                                         groups);
 660
 661                                         if (res) {
 662                                                 Map<String, MapArtifactDataDefinition> artifacts = topologyTemplate
 663                                                                 .getInstDeploymentArtifacts();
 664                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_DEPLOYMENT_ARTIFACTS,
 665                                                                 EdgeLabelEnum.INST_DEPLOYMENT_ARTIFACTS, artifacts);
 666                                         }
 667                                         if (res) {
 668                                                 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 669                                                 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
 670                                                                 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
 671                                         }
 672                                         titanDao.commit();
 673                                 } else {
 674                                         failedList.add(component);
 675                                 }
 676
 677                         }
 678                         if (!failedList.isEmpty()) {
 679                                 for (Component component : failedList) {
 680                                         StringBuilder sb = new StringBuilder(component.getComponentType().getValue());
 681                                         sb.append(",").append(component.getName()).append(",").append(component.getVersion()).append(",")
 682                                                         .append(component.getUniqueId()).append(",").append(component.getUUID()).append(",")
 683                                                         .append(component.getInvariantUUID()).append(",").append(component.getLifecycleState());
 684
 685                                         sb.append("\n");
 686                                         writer.write(sb.toString());
 687                                 }
 688                                 writer.flush();
 689                         }
 690                 } catch (IOException e) {
 691                         log.error(e.getMessage());
 692                 } finally {
 693                         titanDao.commit();
 694                         try {
 695                                 writer.flush();
 696                                 writer.close();
 697                         } catch (Exception ex) {
 698                                 /* ignore */}
 699                 }
 700                 log.info(" Fix finished with res {} ***** ", res);
 701                 return res;
 702         }
 703
 704         private boolean generateAndSaveToscaArtifacts(Map<String, List<Component>> nodesToFixTosca, Set<String> fixedIds,
 705                         List<? extends Component> componentsWithFailedGroups, List<Component> failedList) {
 706                 boolean res = true;
 707                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts started ");
 708                 for (Map.Entry<String, List<Component>> entry : nodesToFixTosca.entrySet()) {
 709
 710                         List<Component> component = entry.getValue();
 711                         for (Component c : component) {
 712                                 log.debug("Migration1707ArtifactUuidFix  fix tosca on component : id {},  name {} ", c.getUniqueId(),
 713                                                 c.getName());
 714                                 if (componentsWithFailedGroups != null) {
 715                                         Optional<Component> op = (Optional<Component>) componentsWithFailedGroups.stream()
 716                                                         .filter(cg -> cg.getUniqueId().equals(c.getUniqueId())).findAny();
 717                                         if (!op.isPresent())
 718                                                 res = generateToscaPerComponent(fixedIds, c);
 719                                 } else
 720                                         res = generateToscaPerComponent(fixedIds, c);
 721                                 if (res) {
 722                                         ToscaElement topologyTemplate = ModelConverter.convertToToscaElement(c);
 723                                         Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
 724                                         res = fixDataOnGraph(c.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS, EdgeLabelEnum.TOSCA_ARTIFACTS,
 725                                                         arifacts);
 726                                         titanDao.commit();
 727                                 } else {
 728                                         failedList.add(c);
 729                                 }
 730
 731                         }
 732                 }
 733                 log.debug("Migration1707ArtifactUuidFix  generateAndSaveToscaArtifacts finished with res {} ", res);
 734                 return res;
 735         }
 736
 737         private boolean generateToscaPerComponent(Set<String> fixedIds, Component c) {
 738                 boolean res = true;
 739                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent started component name {} id {}",
 740                                 c.getName(), c.getUniqueId());
 741                 try {
 742                         Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
 743                                         .getToscaFullElement(c.getUniqueId());
 744                         if (toscaElement.isRight()) {
 745                                 log.info("Failed to fetch resources {} {}", c.getUniqueId(), toscaElement.right().value());
 746                                 return false;
 747                         }
 748                         Component toscaElementFull = toscaElement.left().value();
 749                         toscaElementFull.setGroups(c.getGroups());
 750                         List<ComponentInstance> ciListFull = toscaElementFull.getComponentInstances();
 751                         List<ComponentInstance> ciList = c.getComponentInstances();
 752                         if (ciListFull != null && !ciListFull.isEmpty()) {
 753                                 ciListFull.forEach(ciFull -> {
 754                                         ComponentInstance compInst = ciList.stream()
 755                                                         .filter(ci -> ci.getUniqueId().equals(ciFull.getUniqueId())).findAny().get();
 756                                         ciFull.setGroupInstances(compInst.getGroupInstances());
 757                                 });
 758                         }
 759
 760                         Either<Component, ToscaError> either = generateToscaArtifact(toscaElementFull);
 761
 762                         if (either.isRight()) {
 763                                 log.error("Couldn't generate and save tosca template component  unique id {}, name {} error: {}",
 764                                                 toscaElementFull.getUniqueId(), toscaElementFull.getName(), either.right().value());
 765                                 res = false;
 766
 767                         }
 768
 769                         if (res) {
 770                                 c.setToscaArtifacts(either.left().value().getToscaArtifacts());
 771                                 fixedIds.add(toscaElementFull.getUniqueId());
 772                         }
 773                 } finally {
 774                         if (res)
 775                                 titanDao.commit();
 776                         else
 777                                 titanDao.rollback();
 778                 }
 779                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent finished  component name {} id {} res {}",
 780                                 c.getName(), c.getUniqueId(), res);
 781                 return res;
 782         }
 783
 784         private <T extends ToscaDataDefinition> boolean fixDataOnGraph(String componentId, VertexTypeEnum vertexTypeEnum,
 785                         EdgeLabelEnum edgeLabelEnum, Map<String, T> groups) {
 786                 log.debug("amount groups to update: VertexTypeEnum {} EdgeLabelEnum {} data size {}", vertexTypeEnum.getName(),
 787                                 edgeLabelEnum, groups.size());
 788                 boolean res = true;
 789                 Either<GraphVertex, TitanOperationStatus> getResponse = titanDao.getVertexById(componentId,
 790                                 JsonParseFlagEnum.NoParse);
 791                 if (getResponse.isRight()) {
 792                         log.debug("Couldn't fetch component  unique id {}, error: {}", componentId, getResponse.right().value());
 793                         res = false;
 794
 795                 }
 796                 if (res) {
 797                         GraphVertex componentVertex = getResponse.left().value();
 798
 799                         GraphVertex toscaDataVertex = null;
 800                         Either<GraphVertex, TitanOperationStatus> groupVertexEither = titanDao.getChildVertex(componentVertex,
 801                                         edgeLabelEnum, JsonParseFlagEnum.ParseJson);
 802                         if (groupVertexEither.isRight() && groupVertexEither.right().value() == TitanOperationStatus.NOT_FOUND) {
 803                                 log.debug("no child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum, componentId,
 804                                                 groupVertexEither.right().value());
 805                                 return true;
 806                         }
 807                         if (groupVertexEither.isRight()) {
 808                                 res = false;
 809                                 log.debug("failed to get child {}  vertex for component  unique id {}, error: {}", edgeLabelEnum,
 810                                                 componentId, groupVertexEither.right().value());
 811                         }
 812                         if (res) {
 813                                 toscaDataVertex = groupVertexEither.left().value();
 814                                 toscaDataVertex.setJson(groups);
 815                                 Either<GraphVertex, TitanOperationStatus> updatevertexEither = titanDao.updateVertex(toscaDataVertex);
 816                                 if (updatevertexEither.isRight()) {
 817                                         log.debug("failed to update vertex for component  unique id {}, error: {}", componentId,
 818                                                         updatevertexEither.right().value());
 819                                         titanDao.rollback();
 820                                         return false;
 821                                 }
 822                         }
 823                 }
 824                 log.debug("Fix data on graph finished: VertexTypeEnum {} EdgeLabelEnum {} res {}", vertexTypeEnum.getName(),
 825                                 res);
 826                 return res;
 827         }
 828
 829         private boolean fixServices(List<Service> serviceList) {
 830                 for (Service service : serviceList) {
 831                         log.debug("Migration1707ArtifactUuidFix  fix service: id {},  name {} ", service.getUniqueId(),
 832                                         service.getName());
 833                         List<ComponentInstance> instances = service.getComponentInstances();
 834                         for (ComponentInstance instance : instances) {
 835                                 fixComponentInstances(service, instance);
 836                         }
 837
 838                 }
 839                 return true;
 840
 841         }
 842
 843         private void fixComponentInstances(Service service, ComponentInstance instance) {
 844                 Map<String, ArtifactDefinition> artifactsMap = instance.getDeploymentArtifacts();
 845                 List<GroupInstance> groupsList = instance.getGroupInstances();
 846                 if (groupsList != null && artifactsMap != null) {
 847                         List<GroupInstance> groupsToDelete = new ArrayList<>();
 848                         for (GroupInstance group : groupsList) {
 849                                 fixGroupInstances(service, artifactsMap, groupsToDelete, group);
 850
 851                         }
 852
 853                         if (!groupsToDelete.isEmpty()) {
 854                                 log.debug("Migration1707ArtifactUuidFix  delete group:  resource id {}, group instance to delete {} ",
 855                                                 service.getUniqueId(), groupsToDelete);
 856                                 groupsList.removeAll(groupsToDelete);
 857
 858                         }
 859
 860                         Optional<ArtifactDefinition> optionalVfModuleArtifact = artifactsMap.values().stream()
 861                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
 862                         ArtifactDefinition vfModuleAertifact = null;
 863                         if(!optionalVfModuleArtifact.isPresent()){
 864                                 vfModuleAertifact = createVfModuleArtifact(instance, service);
 865                                 artifactsMap.put(vfModuleAertifact.getArtifactLabel(), vfModuleAertifact);
 866                         }
 867                         else {
 868                                 vfModuleAertifact = optionalVfModuleArtifact.get();
 869                         }
 870                         fillVfModuleInstHeatEnvPayload(service, instance, groupsList, vfModuleAertifact);
 871                 }
 872         }
 873
 874         private void fixGroupInstances(Service service, Map<String, ArtifactDefinition> artifactsMap,
 875                         List<GroupInstance> groupsToDelete, GroupInstance group) {
 876                 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
 877                         log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", service.getUniqueId(),
 878                                         group.getName());
 879                         if(group.getArtifacts() != null){
 880                                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
 881                                 if(group.getGroupInstanceArtifacts() != null){
 882                                         List<String> groupInsArtifacts = new ArrayList<>(group.getGroupInstanceArtifacts());
 883                                         groupArtifactsSet.addAll(groupInsArtifacts);
 884                                 }
 885                                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
 886
 887                                 clearGroupInstanceArtifacts(group);
 888
 889                                 for (String artifactId : groupArtifacts) {
 890                                         fixArtifactUndergroupInstances(artifactsMap, group, groupArtifacts, artifactId);
 891                                 }
 892                         }
 893                         if (group.getArtifacts() == null || group.getArtifacts().isEmpty()) {
 894                                 log.debug(
 895                                                 "Migration1707ArtifactUuidFix  fix groupInstance add to delete list:  resource id {} name {} , group name {} ",
 896                                                 service.getUniqueId(), service.getName(), group.getName());
 897                                 groupsToDelete.add(group);
 898                         }
 899                 }
 900         }
 901
 902         private void clearGroupInstanceArtifacts(GroupInstance group) {
 903                 if(group.getArtifacts() != null)
 904                         group.getArtifacts().clear();
 905                 else
 906                         group.setArtifacts(new ArrayList<>());
 907                 if(group.getArtifactsUuid() != null)
 908                         group.getArtifactsUuid().clear();
 909                 else{
 910                         group.setArtifactsUuid(new ArrayList<>());
 911                 }
 912                 if(group.getGroupInstanceArtifacts() != null)
 913                         group.getGroupInstanceArtifacts().clear();
 914                 else{
 915                         group.setGroupInstanceArtifacts(new ArrayList<>());
 916                 }
 917                 if(group.getGroupInstanceArtifactsUuid() != null )
 918                         group.getGroupInstanceArtifactsUuid().clear();
 919                 else
 920                         group.setGroupInstanceArtifactsUuid(new ArrayList<>());
 921         }
 922
 923         private void fixArtifactUndergroupInstances(Map<String, ArtifactDefinition> artifactsMap, GroupInstance group,
 924                         List<String> groupArtifacts, String artifactId) {
 925                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
 926                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
 927                                 group.getName(), artifactId, artifactlabel);
 928                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
 929                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
 930                         ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
 931                         String correctArtifactId = artifact.getUniqueId();
 932                         String correctArtifactUUID = artifact.getArtifactUUID();
 933                         if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
 934                                 boolean isAddToGroup = true;
 935                                 if (groupArtifacts.size() == 1) {
 936                                         if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
 937                                                 isAddToGroup = false;
 938                                                 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
 939                                         }
 940                                 }
 941                                 if (isAddToGroup) {
 942                                         log.debug(
 943                                                         "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ",
 944                                                         group.getName(), correctArtifactId, correctArtifactUUID);
 945                                         group.getArtifacts().add(correctArtifactId);
 946                                         if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 947                                                 group.getArtifactsUuid().add(correctArtifactUUID);
 948                                         }
 949                                 }
 950                         } else {
 951                                 log.debug(
 952                                                 "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ",
 953                                                 group.getName(), correctArtifactId, correctArtifactUUID);
 954                                 Set<String> tmpSet = new HashSet<>(group.getGroupInstanceArtifacts());
 955                                 tmpSet.add(correctArtifactId);
 956                                 group.setGroupInstanceArtifacts(new ArrayList<>(tmpSet));
 957                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
 958                                         Set<String> tmpSetUUID = new HashSet<>(group.getGroupInstanceArtifactsUuid());
 959                                         tmpSetUUID.add(correctArtifactUUID);
 960                                         group.setGroupInstanceArtifactsUuid(new ArrayList<>(tmpSetUUID));
 961                                 }
 962                         }
 963                 }
 964         }
 965
 966         private boolean fixVf(List<Resource> vfLst) {
 967                 for (Resource resource : vfLst) {
 968                         log.debug("Migration1707ArtifactUuidFix  fix resource: id {},  name {} ", resource.getUniqueId(),
 969                                         resource.getName());
 970                         Map<String, ArtifactDefinition> artifactsMap = resource.getDeploymentArtifacts();
 971                         List<GroupDefinition> groupsList = resource.getGroups();
 972                         List<GroupDefinition> groupsToDelete = new ArrayList<>();
 973                         if (groupsList != null && artifactsMap != null) {
 974                                 for (GroupDefinition group : groupsList) {
 975                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE) && group.getArtifacts() != null) {
 976                                                 fixVfGroup(resource, artifactsMap, group);
 977                                         }
 978                                         if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)
 979                                                         && (group.getArtifacts() == null || group.getArtifacts().isEmpty())) {
 980                                                 log.debug(
 981                                                                 "Migration1707ArtifactUuidFix  add group to delete list fix resource: id {},  name {} ",
 982                                                                 resource.getUniqueId(), resource.getName(), group.getName());
 983                                                 groupsToDelete.add(group);
 984                                         }
 985                                 }
 986
 987                                 if (!groupsToDelete.isEmpty()) {
 988                                         groupsList.removeAll(groupsToDelete);
 989
 990                                 }
 991                         }
 992
 993                 }
 994
 995                 return true;
 996         }
 997
 998         private void fixVfGroup(Resource resource, Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group) {
 999                 log.debug("Migration1707ArtifactUuidFix  fix group:  resource id {}, group name {} ", resource.getUniqueId(),
1000                                 group.getName());
1001                 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
1002                 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
1003                 group.getArtifacts().clear();
1004                 group.getArtifactsUuid().clear();
1005
1006                 for (String artifactId : groupArtifacts) {
1007                         fixArtifactUnderGroup(artifactsMap, group, groupArtifacts, artifactId);
1008                 }
1009         }
1010
1011         private void fixArtifactUnderGroup(Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group,
1012                         List<String> groupArtifacts, String artifactId) {
1013
1014                 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
1015                 log.debug("Migration1707ArtifactUuidFix  fix group:  group name {} artifactId for fix {} artifactlabel {} ",
1016                                 group.getName(), artifactId, artifactlabel);
1017                 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
1018                         ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
1019                         String correctArtifactId = artifact.getUniqueId();
1020                         String correctArtifactUUID = artifact.getArtifactUUID();
1021                         boolean isAddToGroup = true;
1022                         if (groupArtifacts.size() == 1) {
1023                                 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
1024                                 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
1025                                         isAddToGroup = false;
1026                                         artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
1027                                 }
1028                         }
1029                         if (isAddToGroup) {
1030                                 log.debug(
1031                                                 "Migration1707ArtifactUuidFix  fix group:  group name {} correct artifactId {} artifactUUID {} ",
1032                                                 group.getName(), correctArtifactId, correctArtifactUUID);
1033                                 group.getArtifacts().add(correctArtifactId);
1034                                 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
1035                                         group.getArtifactsUuid().add(correctArtifactUUID);
1036                                 }
1037                         }
1038
1039                 }
1040         }
1041
1042         private String findArtifactLabelFromArtifactId(String artifactId) {
1043                 String artifactLabel = "";
1044
1045                 int index = artifactId.lastIndexOf('.');
1046                 if (index > 0 && index + 1 < artifactId.length())
1047                         artifactLabel = artifactId.substring(index + 1);
1048                 return artifactLabel;
1049         }
1050
1051         private void writeModuleResultToFile(Writer writer, org.openecomp.sdc.be.model.Component component,
1052                         Service service) {
1053                 try {
1054                         // "service name, service id, state, version
1055                         StringBuilder sb = new StringBuilder(component.getName());
1056                         sb.append(",").append(component.getUniqueId()).append(",").append(component.getLifecycleState()).append(",")
1057                                         .append(component.getVersion());
1058                         if (service != null) {
1059                                 sb.append(",").append(service.getName());
1060                         }
1061                         sb.append("\n");
1062                         writer.write(sb.toString());
1063                 } catch (IOException e) {
1064                         log.error(e.getMessage());
1065                 }
1066         }
1067
1068         private void writeModuleResultToFile(Writer writer, List<Component> components) {
1069                 try {
1070                         // "service name, service id, state, version
1071                         for (Component component : components) {
1072                                 StringBuilder sb = new StringBuilder(component.getName());
1073                                 sb.append(",").append(component.getUniqueId()).append(",").append(component.getInvariantUUID())
1074                                                 .append(",").append(component.getLifecycleState()).append(",").append(component.getVersion());
1075
1076                                 sb.append("\n");
1077                                 writer.write(sb.toString());
1078                         }
1079                 } catch (IOException e) {
1080
1081                         log.error(e.getMessage());
1082                 }
1083         }
1084
1085         public boolean doFixTosca(Map<String, List<Component>> nodeToFix, Map<String, List<Component>> vfToFix,
1086                         Map<String, List<Component>> serviceToFix) {
1087
1088                 Map<GraphPropertyEnum, Object> hasProps = new HashMap<>();
1089                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1090                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1091
1092                 Map<String, List<Component>> vertices = getVerticesToValidate(VertexTypeEnum.NODE_TYPE, hasProps);
1093                 boolean result = validateTosca(vertices, nodeToFix, "RESOURCE_TOSCA_ARTIFACTS");//
1094
1095                 hasProps.clear();
1096                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1097                 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF);
1098                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1099
1100                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1101                 result = validateTosca(vertices, vfToFix, "VF_TOSCA_ARTIFACTS");
1102
1103                 hasProps.clear();
1104                 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
1105                 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1106
1107                 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1108                 result = validateTosca(vertices, serviceToFix, "SERVICE_TOSCA_ARTIFACTS");
1109
1110                 return result;
1111         }
1112
1113         public Map<String, List<Component>> getVerticesToValidate(VertexTypeEnum type,
1114                         Map<GraphPropertyEnum, Object> hasProps) {
1115
1116                 Map<String, List<Component>> result = new HashMap<>();
1117                 try {
1118
1119                         Either<List<GraphVertex>, TitanOperationStatus> resultsEither = titanDao.getByCriteria(type, hasProps);
1120                         if (resultsEither.isRight()) {
1121                                 log.error("getVerticesToValidate failed {} ",resultsEither.right().value());
1122                                 return result;
1123                         }
1124                         log.info("getVerticesToValidate: {}  vertices to scan", resultsEither.left().value().size());
1125                         List<GraphVertex> componentsList = resultsEither.left().value();
1126                         componentsList.forEach(vertex -> {
1127                                 String ivariantUuid = (String) vertex.getMetadataProperty(GraphPropertyEnum.INVARIANT_UUID);
1128                                 if (!result.containsKey(ivariantUuid)) {
1129                                         List<Component> compList = new ArrayList<Component>();
1130                                         result.put(ivariantUuid, compList);
1131                                 }
1132                                 List<Component> compList = result.get(ivariantUuid);
1133
1134                                 ComponentParametersView filter = new ComponentParametersView(true);
1135                                 filter.setIgnoreArtifacts(false);
1136
1137                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1138                                                 .getToscaElement(vertex.getUniqueId(), filter);
1139                                 if (toscaElement.isRight()) {
1140                                         log.error("getVerticesToValidate: failed to find element {}  staus is {}", vertex.getUniqueId()
1141                                                         ,toscaElement.right().value());
1142                                 } else {
1143                                         compList.add(toscaElement.left().value());
1144                                 }
1145                                 titanDao.commit();
1146
1147                         });
1148
1149                 } catch (Exception e) {
1150                         log.info("Failed to fetch vf resources ", e);
1151
1152                 } finally {
1153                         titanDao.commit();
1154
1155                 }
1156                 return result;
1157         }
1158
1159         public boolean validateTosca(Map<String, List<Component>> vertices, Map<String, List<Component>> compToFix,
1160                         String name) {
1161                 boolean result = true;
1162                 long time = System.currentTimeMillis();
1163                 String fileName = name + "_" + time + ".csv";
1164                 Writer writer = null;
1165                 try {
1166                         writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), "utf-8"));
1167                         writer.write("name, UUID, invariantUUID, state, version\n");
1168                         for (Map.Entry<String, List<Component>> entry : vertices.entrySet()) {
1169                                 List<Component> compList = entry.getValue();
1170                                 Set<String> artifactEsId = new HashSet<>();
1171                                 for (Component component : compList) {
1172                                         Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
1173                                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1174                                                         .filter(a -> artifactEsId.contains(a.getEsId()) && a.getEsId() != null).findAny();
1175                                         if (op.isPresent()) {
1176                                                 result = false;
1177                                                 writeModuleResultToFile(writer, compList);
1178                                                 writer.flush();
1179                                                 break;
1180                                         } else {
1181                                                 artifactEsId.addAll(toscaArtifacts.values().stream().map(ArtifactDefinition::getEsId)
1182                                                                 .collect(Collectors.toList()));
1183                                         }
1184                                 }
1185                                 if (!result) {
1186                                         List<Component> compListfull = new ArrayList<>();
1187                                         for (Component c : compList) {
1188                                                 ComponentParametersView filter = new ComponentParametersView(true);
1189                                                 filter.setIgnoreComponentInstances(false);
1190                                                 filter.setIgnoreArtifacts(false);
1191                                                 filter.setIgnoreGroups(false);
1192
1193                                                 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1194                                                                 .getToscaElement(c.getUniqueId(), filter);
1195                                                 if (toscaElement.isRight()) {
1196                                                         System.out.println("getVerticesToValidate: failed to find element" + c.getUniqueId()
1197                                                                         + " staus is" + toscaElement.right().value());
1198                                                 } else {
1199                                                         compListfull.add(toscaElement.left().value());
1200                                                 }
1201                                                 this.titanDao.commit();
1202                                         }
1203
1204                                         compToFix.put(entry.getKey(), compListfull);
1205                                         result = true;
1206                                 }
1207
1208                         }
1209
1210                 } catch (Exception e) {
1211                         log.info("Failed to fetch vf resources ", e);
1212                         return false;
1213                 } finally {
1214                         titanDao.commit();
1215                         try {
1216                                 writer.flush();
1217                                 writer.close();
1218                         } catch (Exception ex) {
1219                                 /* ignore */}
1220                 }
1221                 return result;
1222         }
1223
1224         private Either<Component, ToscaError> generateToscaArtifact(Component parent) {
1225                 log.debug("tosca artifact generation");
1226                 try {
1227                         Map<String, ArtifactDefinition> toscaArtifacts = parent.getToscaArtifacts();
1228
1229                         ArtifactDefinition toscaArtifact = null;
1230                         Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1231                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_TEMPLATE.getType())).findAny();
1232
1233                         if (op.isPresent()) {
1234                                 toscaArtifact = op.get();
1235                         }
1236                         if (toscaArtifact != null) {
1237                                 log.debug("Migration1707ArtifactUuidFix  generateToscaPerComponent artifact name {} id {} esId {}",
1238                                                 toscaArtifact.getArtifactName(), toscaArtifact.getUniqueId(), toscaArtifact.getEsId());
1239
1240                                 Either<ToscaRepresentation, ToscaError> exportComponent = toscaExportUtils.exportComponent(parent);
1241                                 if (exportComponent.isRight()) {
1242                                         log.debug("Failed export tosca yaml for component {} error {}", parent.getUniqueId(),
1243                                                         exportComponent.right().value());
1244
1245                                         return Either.right(exportComponent.right().value());
1246                                 }
1247                                 log.debug("Tosca yaml exported for component {} ", parent.getUniqueId());
1248                                 String payload = exportComponent.left().value().getMainYaml();
1249
1250                                 toscaArtifact.setPayloadData(payload);
1251                                 byte[] decodedPayload = toscaArtifact.getPayloadData();
1252
1253                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1254                                                 toscaArtifact.getArtifactLabel());
1255                                 toscaArtifact.setUniqueId(uniqueId);
1256                                 toscaArtifact.setEsId(toscaArtifact.getUniqueId());
1257
1258                                 toscaArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1259                                 ESArtifactData artifactData = new ESArtifactData(toscaArtifact.getEsId(), decodedPayload);
1260                                 artifactCassandraDao.saveArtifact(artifactData);
1261
1262                                 log.debug("Tosca yaml artifact esId  {} ", toscaArtifact.getEsId());
1263                         }
1264                         ArtifactDefinition csarArtifact = null;
1265                         op = toscaArtifacts.values().stream()
1266                                         .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_CSAR.getType())).findAny();
1267
1268                         if (op.isPresent()) {
1269                                 csarArtifact = op.get();
1270                         }
1271
1272                         if (csarArtifact != null) {
1273                                 Either<byte[], ResponseFormat> generated = csarUtils.createCsar(parent, true, true);
1274
1275                                 if (generated.isRight()) {
1276                                         log.debug("Failed to export tosca csar for component {} error {}", parent.getUniqueId(),
1277                                                         generated.right().value());
1278
1279                                         return Either.right(ToscaError.GENERAL_ERROR);
1280                                 }
1281                                 byte[] value = generated.left().value();
1282                                 csarArtifact.setPayload(value);
1283                                 byte[] decodedPayload = csarArtifact.getPayloadData();
1284
1285                                 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1286                                                 csarArtifact.getArtifactLabel());
1287                                 csarArtifact.setUniqueId(uniqueId);
1288                                 csarArtifact.setEsId(csarArtifact.getUniqueId());
1289
1290                                 csarArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1291                                 ESArtifactData artifactData = new ESArtifactData(csarArtifact.getEsId(), decodedPayload);
1292                                 artifactCassandraDao.saveArtifact(artifactData);
1293                                 log.debug("Tosca csar artifact esId  {} ", csarArtifact.getEsId());
1294
1295                         }
1296
1297                 } catch (Exception ex) {
1298                         log.error("Failed to generate tosca atifact component id {} component name {} error {}",
1299                                         parent.getUniqueId(), parent.getName(), ex.getMessage());
1300
1301                         return Either.right(ToscaError.GENERAL_ERROR);
1302                 }
1303
1304                 return Either.left(parent);
1305         }
1306
1307            private ArtifactDefinition createVfModuleArtifact(ComponentInstance currVF, Service service) {
1308
1309                 ArtifactDefinition vfModuleArtifactDefinition = new ArtifactDefinition();
1310                 String newCheckSum = null;
1311
1312                 vfModuleArtifactDefinition.setDescription("Auto-generated VF Modules information artifact");
1313                 vfModuleArtifactDefinition.setArtifactDisplayName("Vf Modules Metadata");
1314                 vfModuleArtifactDefinition.setArtifactType(ArtifactTypeEnum.VF_MODULES_METADATA.getType());
1315                 vfModuleArtifactDefinition.setArtifactGroupType(ArtifactGroupTypeEnum.DEPLOYMENT);
1316                 vfModuleArtifactDefinition.setArtifactLabel("vfModulesMetadata");
1317                 vfModuleArtifactDefinition.setTimeout(0);
1318                 vfModuleArtifactDefinition.setArtifactName(currVF.getNormalizedName() + "_modules.json");
1319
1320                return vfModuleArtifactDefinition;
1321             }
1322
1323
1324         private void fillVfModuleInstHeatEnvPayload(Component parent, ComponentInstance instance, List<GroupInstance> groupsForCurrVF,
1325                         ArtifactDefinition vfModuleArtifact) {
1326                 log.debug("generate new vf module for component. name  {}, id {}, Version {}", instance.getName(), instance.getUniqueId());
1327
1328                 String uniqueId = UniqueIdBuilder.buildInstanceArtifactUniqueId(parent.getUniqueId(), instance.getUniqueId(), vfModuleArtifact.getArtifactLabel());
1329
1330                 vfModuleArtifact.setUniqueId(uniqueId);
1331                 vfModuleArtifact.setEsId(vfModuleArtifact.getUniqueId());
1332
1333                 List<VfModuleArtifactPayload> vfModulePayloadForCurrVF = new ArrayList<>();
1334                 if (groupsForCurrVF != null) {
1335                         for (GroupInstance groupInstance : groupsForCurrVF) {
1336                                 VfModuleArtifactPayload modulePayload = new VfModuleArtifactPayload(groupInstance);
1337                                 vfModulePayloadForCurrVF.add(modulePayload);
1338                         }
1339                         Collections.sort(vfModulePayloadForCurrVF,
1340                                         (art1, art2) -> VfModuleArtifactPayload.compareByGroupName(art1, art2));
1341
1342                         final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1343
1344                         String vfModulePayloadString = gson.toJson(vfModulePayloadForCurrVF);
1345                         log.debug("vfModulePayloadString {}", vfModulePayloadString);
1346                         if (vfModulePayloadString != null) {
1347                                 String newCheckSum = GeneralUtility
1348                                                 .calculateMD5Base64EncodedByByteArray(vfModulePayloadString.getBytes());
1349                                 vfModuleArtifact.setArtifactChecksum(newCheckSum);
1350
1351                                 ESArtifactData artifactData = new ESArtifactData(vfModuleArtifact.getEsId(),
1352                                                 vfModulePayloadString.getBytes());
1353                                 artifactCassandraDao.saveArtifact(artifactData);
1354
1355                         }
1356
1357                 }
1358
1359         }
1360
1361         private Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> parseVFModuleJson(ArtifactDefinition vfModuleArtifact) {
1362                 log.info("Try to get vfModule json from cassandra {}", vfModuleArtifact.getEsId());
1363                 Either<ESArtifactData, CassandraOperationStatus> vfModuleData = artifactCassandraDao.getArtifact(vfModuleArtifact.getEsId());
1364
1365                 if (vfModuleData.isRight()) {
1366                         CassandraOperationStatus resourceUploadStatus = vfModuleData.right().value();
1367                         StorageOperationStatus storageResponse = DaoStatusConverter.convertCassandraStatusToStorageStatus(resourceUploadStatus);
1368                         log.error("failed to fetch vfModule json {} from cassandra. Status is {}", vfModuleArtifact.getEsId(), storageResponse);
1369                         return Either.right(storageResponse);
1370
1371                 }
1372
1373                 ESArtifactData esArtifactData = vfModuleData.left().value();
1374                 String gsonData = new String( esArtifactData.getDataAsArray());
1375                 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1376                 JsonArray jsonElement = new JsonArray();
1377                 jsonElement = gson.fromJson(gsonData, jsonElement.getClass());
1378                 List<VfModuleArtifactPayloadEx> vfModules = new ArrayList<>();
1379                 jsonElement.forEach(je ->{
1380                         VfModuleArtifactPayloadEx vfModule = ComponentsUtils.parseJsonToObject(je.toString(), VfModuleArtifactPayloadEx.class);
1381                         vfModules.add(vfModule);
1382                 });
1383
1384                 log.debug  ("parse vf module finish {}", gsonData);
1385                 return Either.left(vfModules);
1386
1387         }
1388
1389         /*public boolean manualFix() {
1390
1391                 Set<String> fixedIds = new HashSet<>();
1392                 Component component;
1393                 String componentId = "86683566-20e8-4cc5-872d-12abca1d57f0";//"9f6a6976-18e3-488a-98a4-c1aade480739";
1394                 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade.getToscaFullElement(componentId);
1395                 if (toscaElement.isRight()) {
1396                         log.info("Failed to fetch resources {} {}", componentId, toscaElement.right().value());
1397
1398                 }
1399                 boolean res = generateToscaPerComponent(fixedIds, toscaElement.left().value());
1400                 log.info("Fix component return res {} ", res);
1401                 titanDao.commit();
1402
1403                 return res;
1404         }
1405
1406
1407         public boolean manualCheck() {
1408
1409         Set<String> fixedIds = new HashSet<>();
1410         Component component;
1411         String componentId = "86d50186-7b00-4bfc-abcb-9e4c6892f338";//"9f6a6976-18e3-488a-98a4-c1aade480739";
1412         Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade.getToscaFullElement(componentId);
1413         if (toscaElement.isRight()) {
1414                 log.info("Failed to fetch resources {} {}", componentId, toscaElement.right().value());
1415
1416         }
1417         boolean res = true;
1418         try {
1419                 res = isProblematicService(toscaElement.left().value(), toscaElement.left().value().getName());
1420                 if(res){
1421                         List<Service> services = new ArrayList<>();
1422                         services.add(toscaElement.left().value());
1423                         this.fix(new ArrayList<Resource>(), services, null, null, null);
1424
1425                         Either<Service, StorageOperationStatus> toscaElementNew  = toscaOperationFacade.getToscaFullElement(componentId);
1426                         if (toscaElement.isRight()) {
1427                                 log.info("Failed to fetch resources {} {}", componentId, toscaElementNew.right().value());
1428
1429                         }
1430                         res = isProblematicService(toscaElementNew.left().value(), toscaElementNew.left().value().getName());
1431                 }
1432                 titanDao.commit();
1433
1434         } catch (IOException e) {
1435                 // TODO Auto-generated catch block
1436                 e.printStackTrace();
1437         }
1438         log.info("Fix component return res {} ", res);
1439
1440         return res;
1441 }*/
1442
1443 }
1444
1445