vid-app-common/src/main/java/org/onap/vid/services/AsyncInstantiationBusinessLogicImpl.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.services;
  22
  23 import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
  24 import static org.onap.vid.controller.MsoController.SVC_INSTANCE_ID;
  25 import static org.onap.vid.controller.MsoController.VNF_INSTANCE_ID;
  26 import static org.onap.vid.utils.KotlinUtilsKt.JACKSON_OBJECT_MAPPER;
  27
  28 import com.google.common.collect.ImmutableMap;
  29 import java.io.IOException;
  30 import java.time.ZonedDateTime;
  31 import java.util.ArrayList;
  32 import java.util.Calendar;
  33 import java.util.Date;
  34 import java.util.List;
  35 import java.util.Map;
  36 import java.util.UUID;
  37 import java.util.function.Consumer;
  38 import org.apache.commons.lang3.StringUtils;
  39 import org.hibernate.SessionFactory;
  40 import org.onap.portalsdk.core.logging.logic.EELFLoggerDelegate;
  41 import org.onap.vid.aai.AaiClientInterface;
  42 import org.onap.vid.aai.model.ResourceType;
  43 import org.onap.vid.dal.AsyncInstantiationRepository;
  44 import org.onap.vid.exceptions.DbFailureUncheckedException;
  45 import org.onap.vid.exceptions.GenericUncheckedException;
  46 import org.onap.vid.exceptions.MaxRetriesException;
  47 import org.onap.vid.exceptions.OperationNotAllowedException;
  48 import org.onap.vid.job.Job;
  49 import org.onap.vid.job.Job.JobStatus;
  50 import org.onap.vid.job.JobAdapter;
  51 import org.onap.vid.job.JobType;
  52 import org.onap.vid.job.JobsBrokerService;
  53 import org.onap.vid.job.impl.JobSharedData;
  54 import org.onap.vid.model.Action;
  55 import org.onap.vid.model.NameCounter;
  56 import org.onap.vid.model.ResourceInfo;
  57 import org.onap.vid.model.ServiceInfo;
  58 import org.onap.vid.model.serviceInstantiation.BaseResource;
  59 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
  60 import org.onap.vid.mso.MsoBusinessLogicImpl;
  61 import org.onap.vid.mso.MsoProperties;
  62 import org.onap.vid.mso.MsoUtil;
  63 import org.onap.vid.mso.RestObject;
  64 import org.onap.vid.mso.rest.AsyncRequestStatus;
  65 import org.onap.vid.mso.rest.RequestStatus;
  66 import org.onap.vid.properties.Features;
  67 import org.onap.vid.utils.DaoUtils;
  68 import org.onap.vid.utils.TimeUtils;
  69 import org.springframework.beans.factory.annotation.Autowired;
  70 import org.springframework.stereotype.Service;
  71 import org.togglz.core.manager.FeatureManager;
  72
  73 @Service
  74 public class AsyncInstantiationBusinessLogicImpl implements
  75         AsyncInstantiationBusinessLogic {
  76
  77     private static final int MAX_RETRIES_GETTING_COUNTER = 100;
  78     private static final int MAX_RETRIES_GETTING_FREE_NAME_FROM_AAI = 10000;
  79     public static final String NAME_FOR_CHECK_AAI_STATUS = "NAME_FOR_CHECK_AAI_STATUS";
  80
  81     private final JobAdapter jobAdapter;
  82
  83     private final JobsBrokerService jobService;
  84
  85     private final CloudOwnerService cloudOwnerService;
  86
  87     private final AsyncInstantiationRepository asyncInstantiationRepository;
  88
  89     private SessionFactory sessionFactory;
  90
  91     private AaiClientInterface aaiClient;
  92
  93     private FeatureManager featureManager;
  94
  95     private AuditService auditService;
  96
  97
  98     private int maxRetriesGettingFreeNameFromAai = MAX_RETRIES_GETTING_FREE_NAME_FROM_AAI;
  99
 100     private static final EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(AsyncInstantiationBusinessLogicImpl.class);
 101     private Map<String, JobStatus> msoStateToJobStatusMap = ImmutableMap.<String, JobStatus>builder()
 102             .put("inprogress", JobStatus.IN_PROGRESS)
 103             .put("failed", JobStatus.FAILED)
 104             .put("pause", JobStatus.PAUSE)
 105             .put("paused", JobStatus.PAUSE)
 106             .put("complete", JobStatus.COMPLETED)
 107             .put("pending", JobStatus.IN_PROGRESS)
 108             .put("pendingmanualtask", JobStatus.PAUSE)
 109             .put("unlocked", JobStatus.IN_PROGRESS)
 110             .put("aborted", JobStatus.COMPLETED_WITH_ERRORS)
 111             .put("rolledback", JobStatus.FAILED)
 112             .put("rolledbacktoassigned", JobStatus.FAILED)
 113             .put("rolledbacktocreated", JobStatus.FAILED)
 114             .build();
 115
 116
 117     @Autowired
 118     public AsyncInstantiationBusinessLogicImpl(JobAdapter jobAdapter,
 119                                                JobsBrokerService jobService,
 120                                                SessionFactory sessionFactory,
 121                                                AaiClientInterface aaiClient,
 122                                                FeatureManager featureManager,
 123                                                CloudOwnerService cloudOwnerService, AsyncInstantiationRepository asyncInstantiationRepository,
 124                                                AuditService auditService) {
 125         this.jobAdapter = jobAdapter;
 126         this.jobService = jobService;
 127         this.sessionFactory = sessionFactory;
 128         this.aaiClient = aaiClient;
 129         this.featureManager = featureManager;
 130         this.cloudOwnerService = cloudOwnerService;
 131         this.asyncInstantiationRepository = asyncInstantiationRepository;
 132         this.auditService = auditService;
 133     }
 134
 135     @Override
 136     public List<ServiceInfo> getAllServicesInfo() {
 137         return asyncInstantiationRepository.getAllServicesInfo();
 138     }
 139
 140     JobType getJobType(ServiceInstantiation request) {
 141         if (request.isALaCarte()) {
 142             switch (defaultIfNull(request.getAction(), Action.Create)) {
 143                 case Delete:
 144                     return JobType.ALaCarteService;
 145                 case None:
 146                     return JobType.ALaCarteService;
 147                 default:
 148                     return JobType.ALaCarteServiceInstantiation;
 149             }
 150         } else {
 151             return JobType.MacroServiceInstantiation;
 152         }
 153     }
 154
 155     @Override
 156     public List<UUID> pushBulkJob(ServiceInstantiation request, String userId) {
 157
 158         List<UUID> uuids = new ArrayList<>();
 159         Date createdBulkDate = Calendar.getInstance().getTime();
 160         int bulkSize = request.getBulkSize();
 161         UUID templateId = UUID.randomUUID();
 162         for (int i = 0; i < bulkSize; i++) {
 163             ServiceInstantiation requestPerJob = prepareServiceToBeUnique(request);
 164             ServiceInfo.ServiceAction serviceAction = getAction(requestPerJob);
 165             JobType jobType = getJobType(requestPerJob);
 166             final String optimisticUniqueServiceInstanceName = bulkSize>1 ? //only bulk with more than 1 service need to get multiple names
 167                     getOptimisticUniqueServiceInstanceName(requestPerJob.getInstanceName()) : requestPerJob.getInstanceName();
 168             Job job = jobAdapter.createServiceInstantiationJob(jobType, requestPerJob, templateId, userId, request.getTestApi(), optimisticUniqueServiceInstanceName, i);
 169             UUID jobId = job.getUuid();
 170
 171             asyncInstantiationRepository.saveServiceInfo(createServiceInfo(userId, requestPerJob, jobId, templateId, createdBulkDate, optimisticUniqueServiceInstanceName, serviceAction));
 172             asyncInstantiationRepository.addJobRequest(jobId, requestPerJob);
 173             auditService.auditVidStatus(jobId, job.getStatus());
 174             uuids.add(jobId);
 175
 176             jobService.add(job);
 177         }
 178         return uuids;
 179     }
 180
 181     private ServiceInfo.ServiceAction getAction(ServiceInstantiation request) {
 182         if (request.getAction() == null) {
 183             //throw new GenericUncheckedException("Required 'action' field not provided at service level");
 184             return Action.Create.getServiceAction();
 185         }
 186         return request.getAction().getServiceAction();
 187     }
 188
 189
 190     private String getOptimisticUniqueServiceInstanceName(String instanceName) {
 191         return StringUtils.isNotEmpty(instanceName) ? getUniqueNameFromDbOnly(instanceName) : instanceName;
 192     }
 193
 194     protected ServiceInfo createServiceInfo(String userId, ServiceInstantiation serviceInstantiation, UUID jobId, UUID templateId, Date createdBulkDate, String optimisticUniqueServiceInstanceName, ServiceInfo.ServiceAction serviceAction) {
 195         return new ServiceInfo(
 196                 userId,
 197                 serviceInstantiation.isALaCarte(),
 198                 Job.JobStatus.PENDING, serviceInstantiation.isPause(), jobId, templateId,
 199                 serviceInstantiation.getOwningEntityId(),
 200                 serviceInstantiation.getOwningEntityName(),
 201                 serviceInstantiation.getProjectName(),
 202                 serviceInstantiation.getAicZoneId(),
 203                 serviceInstantiation.getAicZoneName(),
 204                 serviceInstantiation.getTenantId(),
 205                 serviceInstantiation.getTenantName(),
 206                 serviceInstantiation.getLcpCloudRegionId(),
 207                 null,
 208                 serviceInstantiation.getSubscriptionServiceType(),
 209                 serviceInstantiation.getSubscriberName(),
 210                 serviceInstantiation.getGlobalSubscriberId(),
 211                 serviceInstantiation.getInstanceId(),
 212                 optimisticUniqueServiceInstanceName,
 213                 serviceInstantiation.getModelInfo().getModelVersionId(),
 214                 serviceInstantiation.getModelInfo().getModelName(),
 215                 serviceInstantiation.getModelInfo().getModelVersion(),
 216                 createdBulkDate,
 217                 serviceAction,
 218                 false);
 219     }
 220
 221     @Override
 222     public boolean isPartOfBulk(UUID jobId) {
 223         if (jobId == null) {
 224             return false;
 225     }
 226         ServiceInfo serviceInfo = asyncInstantiationRepository.getServiceInfoByJobId(jobId);
 227         UUID templateId = serviceInfo.getTemplateId();
 228         if (templateId != null) {
 229             return getNumberOfJobsInBulk(templateId) > 1;
 230     }
 231         return false;
 232
 233     }
 234
 235     private int getNumberOfJobsInBulk(UUID templateId) {
 236         String hqlSelectJob = "from JobDaoImpl where templateId = :templateId";
 237         return DaoUtils.tryWithSessionAndTransaction(sessionFactory, session ->
 238             session.createQuery(hqlSelectJob)
 239                     .setText("templateId", templateId.toString())
 240                     .list()
 241                     .size()
 242         );
 243     }
 244
 245     @Override
 246     public String getServiceInstantiationPath(ServiceInstantiation serviceInstantiationRequest) {
 247         //in case pause flag is true - use assign , else - use create.
 248         return MsoBusinessLogicImpl.validateEndpointPath(
 249                 serviceInstantiationRequest.isPause() ?
 250                         MsoProperties.MSO_REST_API_SERVICE_INSTANCE_ASSIGN : MsoProperties.MSO_RESTAPI_SERVICE_INSTANCE
 251         );
 252     }
 253
 254     @Override
 255     public String getServiceDeletionPath(String serviceInstanceId) {
 256         return MsoBusinessLogicImpl.validateEndpointPath( MsoProperties.MSO_RESTAPI_SERVICE_INSTANCE)  + "/" + serviceInstanceId;
 257     }
 258
 259     @Override
 260     public String getVnfInstantiationPath(String serviceInstanceId) {
 261         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_VNF_INSTANCE).
 262                 replaceFirst(SVC_INSTANCE_ID, serviceInstanceId);
 263     }
 264
 265     @Override
 266     public String getVnfDeletionPath(String serviceInstanceId, String vnfInstanceId) {
 267         return (MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_VNF_INSTANCE)
 268                 + '/' + vnfInstanceId)
 269                 .replaceFirst(SVC_INSTANCE_ID, serviceInstanceId).replaceFirst(VNF_INSTANCE_ID, vnfInstanceId);
 270     }
 271
 272     @Override
 273     public String getNetworkInstantiationPath(String serviceInstanceId) {
 274         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_NETWORK_INSTANCE).
 275                 replaceFirst(SVC_INSTANCE_ID, serviceInstanceId);
 276     }
 277
 278     @Override
 279     public String getVfmoduleInstantiationPath(String serviceInstanceId, String vnfInstanceId) {
 280         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_VF_MODULE_INSTANCE)
 281                 .replaceFirst(SVC_INSTANCE_ID, serviceInstanceId)
 282                 .replaceFirst(VNF_INSTANCE_ID, vnfInstanceId);
 283     }
 284
 285     @Override
 286     public String getVfModuleReplacePath(String serviceInstanceId, String vnfInstanceId, String vfModuleInstanceId)
 287     {
 288         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_VF_MODULE_INSTANCE)
 289                 .replaceFirst(SVC_INSTANCE_ID, serviceInstanceId)
 290                 .replaceFirst(VNF_INSTANCE_ID, vnfInstanceId)
 291                 + "/" + vfModuleInstanceId
 292                 + "/replace";
 293     }
 294
 295     @Override
 296     public String getVfModuleDeletePath(String serviceInstanceId, String vnfInstanceId, String vfModuleInstanceId) {
 297         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_VF_MODULE_INSTANCE)
 298                 .replaceFirst(SVC_INSTANCE_ID, serviceInstanceId)
 299                 .replaceFirst(VNF_INSTANCE_ID, vnfInstanceId)
 300                 + "/" + vfModuleInstanceId;
 301     }
 302
 303     @Override
 304     public String getVolumeGroupInstantiationPath(String serviceInstanceId, String vnfInstanceId) {
 305         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_VOLUME_GROUP_INSTANCE)
 306                 .replaceFirst(SVC_INSTANCE_ID, serviceInstanceId)
 307                 .replaceFirst(VNF_INSTANCE_ID, vnfInstanceId);
 308     }
 309
 310     @Override
 311     public String getInstanceGroupInstantiationPath() {
 312         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_INSTANCE_GROUP);
 313     }
 314
 315     @Override
 316     public String getInstanceGroupMemberInstantiationPath(String vnfGroupInstanceId) {
 317         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_INSTANCE_GROUP)
 318                 + '/' + vnfGroupInstanceId + "/addMembers";
 319     }
 320
 321     @Override
 322     public String getInstanceGroupDeletePath(String instanceGroupId) {
 323         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_INSTANCE_GROUP)
 324                 + '/' + instanceGroupId;
 325     }
 326
 327     @Override
 328     public String getInstanceGroupMemberDeletePath(String vnfGroupInstanceId){
 329         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_INSTANCE_GROUP)
 330                 + '/' + vnfGroupInstanceId + "/removeMembers";
 331     }
 332
 333     @Override
 334     public String getNetworkDeletePath(String serviceInstanceId, String networkInstanceId) {
 335         return (MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_NETWORK_INSTANCE)
 336                 + "/" + networkInstanceId)
 337                 .replaceFirst(SVC_INSTANCE_ID, serviceInstanceId);
 338     }
 339
 340     @Override
 341     public String getResumeRequestPath(String requestId) {
 342         return MsoBusinessLogicImpl.validateEndpointPath("mso.restapi.resume.orc.req")
 343                 .replaceFirst("<request_id>", requestId);
 344     }
 345
 346     @Override
 347     public String getOrchestrationRequestsPath() {
 348         return MsoBusinessLogicImpl.validateEndpointPath(MsoProperties.MSO_REST_API_GET_ORC_REQ);
 349     }
 350
 351     @Override
 352     public ServiceInfo updateServiceInfo(UUID jobUUID, Consumer<ServiceInfo> serviceUpdater) {
 353         ServiceInfo serviceInfo = asyncInstantiationRepository.getServiceInfoByJobId(jobUUID);
 354         serviceUpdater.accept(serviceInfo);
 355         asyncInstantiationRepository.saveServiceInfo(serviceInfo);
 356         return serviceInfo;
 357     }
 358
 359     @Override
 360     public ServiceInfo updateServiceInfoAndAuditStatus(UUID jobUuid, JobStatus jobStatus) {
 361         auditService.auditVidStatus(jobUuid, jobStatus);
 362         return updateServiceInfo(jobUuid, x -> setServiceInfoStatus(x, jobStatus));
 363     }
 364
 365     private boolean isRetryEnabledForStatus(JobStatus jobStatus) {
 366         return featureManager.isActive(Features.FLAG_1902_RETRY_JOB) &&
 367                 (jobStatus==JobStatus.COMPLETED_WITH_ERRORS || jobStatus==JobStatus.FAILED);
 368     }
 369
 370     private void setServiceInfoStatus(ServiceInfo serviceInfo, JobStatus jobStatus) {
 371         serviceInfo.setJobStatus(jobStatus);
 372         serviceInfo.setStatusModifiedDate(new Date());
 373         serviceInfo.setRetryEnabled(isRetryEnabledForStatus(jobStatus));
 374     }
 375
 376     public Job.JobStatus calcStatus(AsyncRequestStatus asyncRequestStatus) {
 377         String msoRequestState = asyncRequestStatus.request.requestStatus.getRequestState().toLowerCase().replaceAll("[^a-z]+", "");
 378         JobStatus jobStatus = msoStateToJobStatusMap.get(msoRequestState);
 379         return (jobStatus != null ? jobStatus : JobStatus.IN_PROGRESS);
 380     }
 381
 382     @Override
 383     public void handleFailedInstantiation(UUID jobUUID) {
 384         ServiceInfo serviceInfo = asyncInstantiationRepository.getServiceInfoByJobId(jobUUID);
 385         List<ServiceInfo> serviceInfoList = asyncInstantiationRepository.getServiceInfoByTemplateIdAndJobStatus(serviceInfo.getTemplateId(), JobStatus.PENDING);
 386         serviceInfoList.forEach(si -> updateServiceInfoAndAuditStatus(si.getJobId(), JobStatus.STOPPED));
 387     }
 388
 389     @Override
 390     public void deleteJob(UUID jobId) {
 391         jobService.delete(jobId);
 392         Date now = new Date();
 393         updateServiceInfo(jobId, x -> x.setDeletedAt(now));
 394     }
 395
 396     @Override
 397     public void hideServiceInfo(UUID jobUUID) {
 398         ServiceInfo serviceInfo = asyncInstantiationRepository.getServiceInfoByJobId(jobUUID);
 399         if (!serviceInfo.getJobStatus().isFinal()) {
 400             String message = String.format("jobId %s: Service status does not allow hide service, status = %s",
 401                     serviceInfo.getJobId(),
 402                     serviceInfo.getJobStatus());
 403             logger.error(EELFLoggerDelegate.errorLogger, message);
 404             throw new OperationNotAllowedException(message);
 405         }
 406         serviceInfo.setHidden(true);
 407         asyncInstantiationRepository.saveServiceInfo(serviceInfo);
 408     }
 409
 410     @Override
 411     public int getCounterForName(String name) {
 412
 413         String hqlSelectNC = "from NameCounter where name = :name";
 414         String hqlUpdateCounter = "update NameCounter set counter = :newCounter " +
 415                 "where name= :name " +
 416                 "and counter= :prevCounter";
 417
 418         Integer counter = null;
 419         GenericUncheckedException lastException = null;
 420         for (int i = 0; i< MAX_RETRIES_GETTING_COUNTER && counter==null; i++) {
 421             try {
 422                 counter = calcCounter(name, hqlSelectNC, hqlUpdateCounter);
 423             }
 424             catch (GenericUncheckedException exception) {
 425                 lastException = exception; //do nothing, we will try again in the loop
 426             }
 427         }
 428
 429         if (counter!=null) {
 430             return counter;
 431         }
 432
 433         throw lastException!=null ? new DbFailureUncheckedException(lastException) :
 434                 new DbFailureUncheckedException("Failed to get counter for "+name+" due to unknown error");
 435
 436     }
 437
 438     private Integer calcCounter(String name, String hqlSelectNC, String hqlUpdateCounter) {
 439         Integer counter;
 440         counter = DaoUtils.tryWithSessionAndTransaction(sessionFactory, session -> {
 441             NameCounter nameCounter = (NameCounter) session.createQuery(hqlSelectNC)
 442                     .setText("name", name)
 443                     .uniqueResult();
 444             if (nameCounter != null) {
 445                 int updatedRows = session.createQuery(hqlUpdateCounter)
 446                         .setText("name", nameCounter.getName())
 447                         .setInteger("prevCounter", nameCounter.getCounter())
 448                         .setInteger("newCounter", nameCounter.getCounter() + 1)
 449                         .executeUpdate();
 450                 if (updatedRows == 1) {
 451                     return nameCounter.getCounter() + 1;
 452                 }
 453             } else {
 454                 Object nameAsId = session.save(new NameCounter(name));
 455                 //if save success
 456                 if (nameAsId != null) {
 457                     return 0;
 458                 }
 459             }
 460             //in case of failure return null, in order to continue the loop
 461             return null;
 462         });
 463         return counter;
 464     }
 465
 466     @Override
 467     public int getMaxRetriesGettingFreeNameFromAai() {
 468         return maxRetriesGettingFreeNameFromAai;
 469     }
 470
 471     @Override
 472     public void setMaxRetriesGettingFreeNameFromAai(int maxRetriesGettingFreeNameFromAai) {
 473         this.maxRetriesGettingFreeNameFromAai = maxRetriesGettingFreeNameFromAai;
 474     }
 475
 476     @Override
 477     public String getUniqueName(String name, ResourceType resourceType) {
 478         //check that name aai response well before increasing counter from DB
 479         //Prevents unnecessary increasing of the counter while AAI doesn't response
 480         isNameFreeInAai(NAME_FOR_CHECK_AAI_STATUS, resourceType);
 481
 482         for (int i=0; i<getMaxRetriesGettingFreeNameFromAai(); i++) {
 483             String newName = getUniqueNameFromDbOnly(name);
 484             if (isNameFreeInAai(newName, resourceType)) {
 485                 return newName;
 486             }
 487         }
 488
 489         throw new MaxRetriesException("can't find unused name for "+name, getMaxRetriesGettingFreeNameFromAai());
 490     }
 491
 492     @Override
 493     public ServiceInstantiation prepareServiceToBeUnique(ServiceInstantiation serviceInstantiation) {
 494         try {
 495             ServiceInstantiation clonedServiceInstantiation = JACKSON_OBJECT_MAPPER.readValue(
 496                     JACKSON_OBJECT_MAPPER.writeValueAsBytes(serviceInstantiation), ServiceInstantiation.class);
 497             clonedServiceInstantiation.setBulkSize(1);
 498             return replaceAllTrackById(clonedServiceInstantiation);
 499         } catch (IOException e) {
 500             throw new GenericUncheckedException(e);
 501         }
 502
 503     }
 504
 505     private<T extends BaseResource> T replaceAllTrackById(T resource) {
 506         resource.setTrackById(UUID.randomUUID().toString());
 507         resource.getChildren().forEach(this::replaceAllTrackById);
 508         return resource;
 509     }
 510
 511     @Override
 512     public List<UUID> retryJob(ServiceInstantiation request, UUID jobId, String userId ) {
 513         updateServiceInfo(jobId, si->si.setRetryEnabled(false));
 514         return pushBulkJob(request, userId);
 515     }
 516
 517     @Override
 518     public List<UUID> retryJob(UUID jobId, String userId) {
 519         ServiceInstantiation serviceInstantiationRequest = asyncInstantiationRepository.getJobRequest(jobId);
 520         enrichBulkForRetry(serviceInstantiationRequest, jobId);
 521
 522         try {
 523             logger.debug(EELFLoggerDelegate.debugLogger, "retry ServiceInstantiation request: "+
 524                     JACKSON_OBJECT_MAPPER.writeValueAsString(serviceInstantiationRequest));
 525         } catch (Exception e) {
 526             logger.error(EELFLoggerDelegate.errorLogger, "failed to log retry of ServiceInstantiation request ", e);
 527         }
 528         return retryJob(serviceInstantiationRequest, jobId, userId);
 529     }
 530
 531     @Override
 532     public ServiceInstantiation getBulkForRetry(UUID jobId) {
 533          return enrichBulkForRetry( asyncInstantiationRepository.getJobRequest(jobId), jobId);
 534     }
 535
 536     @Override
 537     public void addResourceInfo(JobSharedData sharedData, Job.JobStatus jobStatus, String instanceId) {
 538         String trackById = ((BaseResource) sharedData.getRequest()).getTrackById();
 539         ResourceInfo resourceInfo = new ResourceInfo(trackById, sharedData.getRootJobId(), instanceId, jobStatus, null);
 540         asyncInstantiationRepository.saveResourceInfo(resourceInfo);
 541     }
 542
 543     @Override
 544     public void addFailedResourceInfo(JobSharedData sharedData, RestObject msoResponse) {
 545         String trackById = ((BaseResource) sharedData.getRequest()).getTrackById();
 546         String errorMessage = MsoUtil.formatExceptionAdditionalInfo(msoResponse.getStatusCode(), msoResponse.getRaw());
 547         AsyncRequestStatus asyncRequestStatus = convertMessageToAsyncRequestStatus(errorMessage);
 548         ResourceInfo resourceInfo = new ResourceInfo(trackById, sharedData.getRootJobId(), null, JobStatus.FAILED, asyncRequestStatus);
 549
 550         asyncInstantiationRepository.saveResourceInfo(resourceInfo);
 551     }
 552
 553     @Override
 554     public void updateResourceInfo(JobSharedData sharedData, JobStatus jobStatus, AsyncRequestStatus message) {
 555         ResourceInfo resourceInfo = asyncInstantiationRepository.getResourceInfoByTrackId(((BaseResource) sharedData.getRequest()).getTrackById());
 556         resourceInfo.setJobStatus(jobStatus);
 557         if (jobStatus.isFailure()) {
 558             resourceInfo.setErrorMessage(message);
 559         }
 560         asyncInstantiationRepository.saveResourceInfo(resourceInfo);
 561     }
 562
 563     public AsyncRequestStatus convertMessageToAsyncRequestStatus(String message) {
 564         RequestStatus requestStatus = new RequestStatus("FAILED", message, TimeUtils.zonedDateTimeToString(ZonedDateTime.now()));
 565         AsyncRequestStatus.Request request = new AsyncRequestStatus.Request(requestStatus);
 566         return new AsyncRequestStatus(request);
 567     }
 568
 569     protected String getUniqueNameFromDbOnly(String name) {
 570         int counter = getCounterForName(name);
 571         return formatNameAndCounter(name, counter);
 572     }
 573
 574     //the method is protected so we can call it in the UT
 575     protected String formatNameAndCounter(String name, int counter) {
 576         return counter==0 ? name : name + "_" + String.format("%03d", counter);
 577     }
 578
 579     private boolean isNameFreeInAai(String name, ResourceType resourceType){
 580         return !aaiClient.isNodeTypeExistsByName(name, resourceType);
 581     }
 582
 583     @Override
 584     public ServiceInstantiation enrichBulkForRetry(ServiceInstantiation serviceInstantiation, UUID jobId){
 585         Map<String, ResourceInfo> resourceInfoByTrackId = asyncInstantiationRepository.getResourceInfoByRootJobId(jobId);
 586
 587         return setResourceStatus(resourceInfoByTrackId, serviceInstantiation);
 588     }
 589
 590     protected String readStatusMsg(ResourceInfo resourceInfo){
 591         if(resourceInfo!=null && resourceInfo.getErrorMessage()!=null && resourceInfo.getErrorMessage().request != null &&resourceInfo.getErrorMessage().request.requestStatus != null ) {
 592             return resourceInfo.getErrorMessage().request.requestStatus.getStatusMessage();
 593         }
 594         return null;
 595     }
 596
 597     private<T extends BaseResource> T setResourceStatus(Map<String, ResourceInfo> resourceInfoByTrackId, T resource) {
 598         ResourceInfo resourceInfo = resourceInfoByTrackId.get(resource.getTrackById());
 599         if(resourceInfo != null) {
 600             boolean failed = resourceInfo.getJobStatus().isFailure();
 601             resource.setIsFailed(failed);
 602             resource.setStatusMessage(readStatusMsg(resourceInfo));
 603             if (!failed) {
 604                 // if(resource.getAction().equals(Action.Delete)){
 605                 // TODO not yet implemented- completed after delete should remove the node
 606                 resource.setAction(Action.None);
 607                 resource.setInstanceId(resourceInfo.getInstanceId());
 608             }
 609         }
 610         resource.getChildren().forEach(child -> setResourceStatus(resourceInfoByTrackId, child));
 611         return resource;
 612     }
 613
 614 }