vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import static java.util.stream.Collectors.counting;
  24 import static java.util.stream.Collectors.groupingBy;
  25 import static java.util.stream.Collectors.joining;
  26 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  27 import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals;
  28 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  29 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartMatches;
  30 import static org.hamcrest.CoreMatchers.equalTo;
  31 import static org.hamcrest.CoreMatchers.is;
  32 import static org.hamcrest.CoreMatchers.notNullValue;
  33 import static org.hamcrest.CoreMatchers.nullValue;
  34 import static org.hamcrest.MatcherAssert.assertThat;
  35 import static org.hamcrest.Matchers.hasProperty;
  36 import static org.hamcrest.Matchers.hasSize;
  37 import static org.hamcrest.core.Every.everyItem;
  38 import static org.mockito.ArgumentMatchers.any;
  39 import static org.mockito.ArgumentMatchers.argThat;
  40 import static org.mockito.ArgumentMatchers.endsWith;
  41 import static org.mockito.ArgumentMatchers.eq;
  42 import static org.mockito.ArgumentMatchers.isNull;
  43 import static org.mockito.Mockito.reset;
  44 import static org.mockito.Mockito.times;
  45 import static org.mockito.Mockito.verify;
  46 import static org.mockito.Mockito.when;
  47 import static org.onap.vid.job.Job.JobStatus.COMPLETED;
  48 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_ERRORS;
  49 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_NO_ACTION;
  50 import static org.onap.vid.job.Job.JobStatus.FAILED;
  51 import static org.onap.vid.job.Job.JobStatus.IN_PROGRESS;
  52 import static org.onap.vid.job.Job.JobStatus.PAUSE;
  53 import static org.onap.vid.job.Job.JobStatus.PENDING;
  54 import static org.onap.vid.job.Job.JobStatus.PENDING_RESOURCE;
  55 import static org.onap.vid.job.Job.JobStatus.RESOURCE_IN_PROGRESS;
  56 import static org.onap.vid.job.Job.JobStatus.STOPPED;
  57 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  58 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  59 import static org.testng.AssertJUnit.assertEquals;
  60 import static org.testng.AssertJUnit.assertFalse;
  61 import static org.testng.AssertJUnit.assertTrue;
  62
  63 import com.fasterxml.jackson.databind.JsonNode;
  64 import com.google.common.collect.ImmutableList;
  65 import com.google.common.collect.ImmutableMap;
  66 import java.io.IOException;
  67 import java.lang.reflect.Method;
  68 import java.util.Collection;
  69 import java.util.Collections;
  70 import java.util.HashMap;
  71 import java.util.List;
  72 import java.util.Map;
  73 import java.util.Optional;
  74 import java.util.Stack;
  75 import java.util.UUID;
  76 import java.util.function.BiConsumer;
  77 import java.util.function.Supplier;
  78 import java.util.stream.Collectors;
  79 import java.util.stream.IntStream;
  80 import java.util.stream.Stream;
  81 import javax.inject.Inject;
  82 import javax.ws.rs.ProcessingException;
  83 import org.apache.commons.lang3.StringUtils;
  84 import org.apache.commons.lang3.mutable.MutableInt;
  85 import org.jetbrains.annotations.NotNull;
  86 import org.mockito.ArgumentCaptor;
  87 import org.mockito.Mockito;
  88 import org.mockito.hamcrest.MockitoHamcrest;
  89 import org.onap.portalsdk.core.service.DataAccessService;
  90 import org.onap.portalsdk.core.util.SystemProperties;
  91 import org.onap.vid.asdc.AsdcCatalogException;
  92 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  93 import org.onap.vid.config.DataSourceConfig;
  94 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  95 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  96 import org.onap.vid.job.Job;
  97 import org.onap.vid.job.Job.JobStatus;
  98 import org.onap.vid.job.JobType;
  99 import org.onap.vid.job.JobsBrokerService;
 100 import org.onap.vid.job.command.CommandUtils;
 101 import org.onap.vid.job.command.InternalState;
 102 import org.onap.vid.model.Action;
 103 import org.onap.vid.model.JobAuditStatus;
 104 import org.onap.vid.model.NameCounter;
 105 import org.onap.vid.model.RequestReferencesContainer;
 106 import org.onap.vid.model.Service;
 107 import org.onap.vid.model.ServiceInfo;
 108 import org.onap.vid.model.ServiceModel;
 109 import org.onap.vid.model.VNF;
 110 import org.onap.vid.model.VfModule;
 111 import org.onap.vid.model.serviceInstantiation.BaseResource;
 112 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
 113 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
 114 import org.onap.vid.mso.RestMsoImplementation;
 115 import org.onap.vid.mso.RestObject;
 116 import org.onap.vid.mso.model.RequestReferences;
 117 import org.onap.vid.mso.rest.AsyncRequestStatus;
 118 import org.onap.vid.mso.rest.AsyncRequestStatusList;
 119 import org.onap.vid.properties.Features;
 120 import org.onap.vid.services.AsyncInstantiationBaseTest;
 121 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
 122 import org.onap.vid.services.AuditService;
 123 import org.onap.vid.services.VersionService;
 124 import org.onap.vid.testUtils.TestUtils;
 125 import org.onap.vid.utils.DaoUtils;
 126 import org.springframework.http.HttpMethod;
 127 import org.springframework.test.context.ContextConfiguration;
 128 import org.testng.annotations.BeforeClass;
 129 import org.testng.annotations.BeforeMethod;
 130 import org.testng.annotations.DataProvider;
 131 import org.testng.annotations.Test;
 132 import org.togglz.core.manager.FeatureManager;
 133
 134 //it's more like integration test than UT
 135 //But it's very hard to test in API test so I use UT
 136 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 137 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 138
 139     private static final String FAILED_STR = "FAILED";
 140     private static final String COMPLETE_STR = "COMPLETE";
 141     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 142     private static final String REQUESTED = "REQUESTED";
 143     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 144     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 145     private static String USER_ID =  "123";
 146     public static String REQUEST_ID = UUID.randomUUID().toString();
 147     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 148
 149     @Inject
 150     private VersionService versionService;
 151
 152     @Inject
 153     private JobsBrokerService jobsBrokerService;
 154
 155     @Inject
 156     private JobWorker jobWorker;
 157
 158     @Inject
 159     private FeatureManager featureManager;
 160
 161     @Inject
 162     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 163
 164     @Inject
 165     private AuditService auditService;
 166
 167     @Inject
 168     private RestMsoImplementation restMso;
 169
 170     @Inject
 171     private DataAccessService dataAccessService;
 172
 173     @Inject
 174     private CommandUtils commandUtils;
 175
 176     @BeforeClass
 177     void initServicesInfoService() {
 178         createInstanceParamsMaps();
 179         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 180     }
 181
 182     @BeforeMethod
 183     void clearDb() {
 184         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 185         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 186         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 187     }
 188
 189     @BeforeMethod
 190     void defineMocks() {
 191         Mockito.reset(restMso);
 192         Mockito.reset(aaiClient);
 193         mockAaiClientAnyNameFree();
 194     }
 195
 196     @Test
 197     public void whenPushNewBulk_thenAllServicesAreInPending() {
 198
 199         pushMacroBulk();
 200         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 201         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 202     }
 203
 204     private List<UUID> pushMacroBulk() {
 205         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 206             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 207             3, true,PROJECT_NAME, true);
 208         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 209     }
 210
 211     private UUID pushALaCarteWithVnf() {
 212         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 213         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 214         assertThat(uuids, hasSize(1));
 215         return uuids.get(0);
 216     }
 217
 218     private UUID pushALaCarteUpdateWithGroups() {
 219         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 220         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 221         assertThat(uuids, hasSize(1));
 222         return uuids.get(0);
 223     }
 224
 225     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 226         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 227     }
 228
 229     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 230         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 231     }
 232
 233     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 234         RequestReferences requestReferences = new RequestReferences();
 235         requestReferences.setRequestId(requestId);
 236         requestReferences.setInstanceId(instanceId);
 237         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 238         restObject.set(new RequestReferencesContainer(requestReferences));
 239         restObject.setStatusCode(statusCode);
 240         restObject.setRaw(RAW_DATA_FROM_MSO);
 241         return restObject;
 242     }
 243
 244     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 245         return Stream.of(jobStatuses).map(
 246             Enum::toString).collect(ImmutableList.toImmutableList());
 247     }
 248
 249     /*
 250     Make sure service state is in progress once request has sent to MSO
 251     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 252     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 253     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 254     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 255     */
 256     @Test
 257     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 258
 259         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 260         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 261         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 262         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 263
 264         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 265             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 266
 267         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 268             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 269             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 270             statusesToStrings(PENDING, STOPPED)
 271         );
 272
 273         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 274             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 275             ImmutableList.of(REQUESTED, FAILED_STR),
 276             ImmutableList.of()
 277         );
 278
 279         List<UUID> uuids = pushMacroBulk();
 280         UUID firstJobUuid = uuids.get(0);
 281         UUID secondJobUuid = uuids.get(1);
 282         //assert that when get ProcessingException from restMso, status remain the same
 283         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 284         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 285
 286         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 287         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 288         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 289
 290         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 291         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 292         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 293
 294         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 295         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 296         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 297
 298         //when get job COMPLETE from MSO, service status become COMPLETED
 299         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 300         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 301         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 302
 303
 304         //for use later in the test
 305         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 306             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 307         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 308
 309         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 310         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 311             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 312         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 313         processJobsCountTimes(10);
 314
 315         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 316         listServicesAndAssertStatus(expectedJobStatusMap);
 317
 318
 319         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 320         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 321         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 322         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 323         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 324             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 325         ));
 326
 327         listServicesAndAssertStatus(expectedJobStatusMap);
 328         IntStream.range(0, uuids.size()).forEach(i -> {
 329             UUID uuid = uuids.get(i);
 330             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 331             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 332         });
 333
 334         //assert no more jobs to pull
 335         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 336         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 337     }
 338
 339
 340     @DataProvider
 341     public static Object[][] AlaCarteStatuses(Method test) {
 342         return new Object[][]{
 343             {COMPLETE_STR, JobStatus.COMPLETED},
 344             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 345         };
 346     }
 347
 348     /*
 349     Make sure service state is in progress once request has sent to MSO
 350     Make sure service state is watching until state changes to complemented
 351     Make sure service state is watching until vnf state changes to completed
 352     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 353     status Creating
 354      */
 355     @Test(dataProvider = "AlaCarteStatuses")
 356     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 357         /*
 358             [v]  + push alacarte with 1 vnf
 359             [v]    verify STATUS pending
 360             [v]  + pull+execute  (should post to MSO)
 361             [v]    verify STATUS in progress
 362             [v]  + pull+execute  (should GET completed from MSO)
 363             [v]    verify STATUS in progress; TYPE watching
 364             [v]    verify job#2 *new* VNF job STATUS creating
 365             [v]  + pull+execute job#2 (should post to MSO)
 366             [v]    verify job#2 STATUS resource in progress
 367             [v]    verify job#1 STATUS in progress
 368             [v]  + pull+execute job#2 (should GET completed from MSO)
 369             [v]    verify job#2 STATUS completed
 370             [v]  + pull+execute job#1
 371             [v]    verify job#1 STATUS completed
 372
 373            * not looking on audit (yet)
 374         */
 375         reset(restMso);
 376         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 377         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false);
 378         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 379         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 380         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 381
 382
 383         //push alacarte with 1 vnf, verify STATUS pending
 384         UUID uuid = pushALaCarteWithVnf();
 385         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 386         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 387         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 388             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 389         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 390         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 391             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 392         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 393         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 394             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 395         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 396             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 397
 398         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 399         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 400         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 401         verify(restMso, times(2)).GetForObject(any(), any());
 402
 403     }
 404
 405     /*
 406     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 407
 408     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 409     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 410
 411     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 412     And union these tests to single one.
 413      */
 414
 415     @Test
 416     public void testALaCarteLifeCycle1Vnf2VfModules() {
 417
 418
 419         String msoVnfStatus = COMPLETE_STR;
 420         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 421         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 422         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 423         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 424         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 425         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 426         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 427         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 428         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 429         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 430
 431
 432         //push alacarte with 1 vnf, verify STATUS pending
 433         UUID uuid = pushALaCarteWithVnf();
 434         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 435         reset(restMso);
 436
 437         /*---------- service -----------*/
 438
 439         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 440         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 441             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 442
 443         //mock mso to answer COMPLETE for service instance create
 444         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 445             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 446
 447         /*---------- vnf -----------*/
 448
 449         //mock mso to answer 200 of create vnf instance request
 450         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 451             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 452
 453         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 454         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 455             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 456
 457         try {
 458             reset(commandUtils);
 459             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true);
 460             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false);
 461         } catch (AsdcCatalogException e) {
 462
 463         }
 464
 465         /*---------- vf Module without volume group name (base) -----------*/
 466
 467         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 468         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 469             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 470         //mock mso to answer for vf module orchestration request
 471         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 472             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 473
 474         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 475             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 476         //mock mso to answer for volume group orchestration request
 477         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 478             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 479
 480         /*---------- vfModule -----------*/
 481
 482         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 483         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 484             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 485
 486         //mock mso to answer for vf module orchestration request
 487         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 488             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 489
 490         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 491         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 492         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 493         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 494         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 495         verify(restMso, times(5)).GetForObject(any(), any());
 496     }
 497
 498     @Test
 499     public void testALaCarteLifeCycle2Networks() {
 500
 501         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 502         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 503         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 504         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 505
 506         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 507         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 508         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 509         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 510         //TODO use them later for different networks
 511         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 512         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 513
 514
 515         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 516         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 517
 518         reset(restMso);
 519
 520         /*---------- service -----------*/
 521
 522         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 523         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 524             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 525
 526         //mock mso to answer COMPLETE for service instance create
 527         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 528             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 529
 530         final MutableInt secondNetworkCode = new MutableInt(500);
 531         final MutableInt inProgressCount = new MutableInt(0);
 532
 533         /*---------- network 1-----------*/
 534
 535         //mock mso to answer 200 of first create network instance request
 536         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 537             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 538             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 539             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 540
 541         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 542         //Once COMPLETE, second network creation will return 200
 543         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 544             thenAnswer(x->{
 545                 String status;
 546                 if (inProgressCount.getValue()<10) {
 547                     status = IN_PROGRESS_STR;
 548                 } else {
 549                     secondNetworkCode.setValue(200);
 550                     status = COMPLETE_STR;
 551                 }
 552                 inProgressCount.add(1);
 553                 return asyncRequestStatusResponseAsRestObject(status);
 554             });
 555
 556         /*---------- network 2-----------*/
 557
 558         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 559         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 560             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 561             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 562             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 563
 564 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 565 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 566         //mock mso to answer COMPLETE for network creation status,
 567
 568         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 569             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 570
 571
 572         /*---------- Create request and process it -----------*/
 573         //push alacarte with 2 networks, verify STATUS pending
 574         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 575         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 576         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 577         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 578
 579         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 580
 581         //validate the mso request id is the right one
 582         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 583         ServiceInfo serviceInfo = serviceInfoList.get(0);
 584         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 585
 586         /*---------- verify -----------*/
 587         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 588         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 589         //get status
 590         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 591         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 592         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 593     }
 594
 595     @Test
 596     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 597         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 598         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 599         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 600             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 601
 602         List<UUID> uuids = pushMacroBulk();
 603         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 604
 605         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 606         Mockito.reset(aaiClient); // must forget the "thenThrow"
 607         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 608         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 609
 610     }
 611
 612     @Test
 613     public void testAaiResponseNameUsedTillMaxRetries() {
 614         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 615         //simulate MSO to return good result, for making sure we failed because of AAI error
 616         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 617         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 618             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 619
 620         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 621         List<UUID> uuids = pushMacroBulk();
 622         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 623     }
 624
 625     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 626         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 627     }
 628
 629     //return the pulled job (and not the pushed job)
 630     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 631         Optional<Job> job = pullJob(topic, pullingAssertion);
 632
 633         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 634
 635         try {
 636             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 637
 638             if (pullingAssertion) {
 639                 //assert another pulling on same topic return no result (before push back)
 640                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 641             }
 642
 643         } finally {
 644             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 645         }
 646         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 647         return job.get();
 648     }
 649
 650     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 651         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 652         optionalJob.ifPresent(job->{
 653             Job nextJob = jobWorker.executeJobAndGetNext(job);
 654             jobsBrokerService.pushBack(nextJob);
 655         });
 656     }
 657
 658     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 659         JobType expectedNextJobType) {
 660         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 661         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 662         return job;
 663     }
 664
 665     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 666         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 667         return retryWithAssertionsLimit(retries, () -> {
 668             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 669             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 670             if (internalState != null) {
 671                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 672             }
 673             return job;
 674         });
 675     }
 676
 677     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 678         java.util.Stack<AssertionError> history = new Stack<>();
 679
 680         do {
 681             try {
 682                 return supplier.get();
 683             } catch (AssertionError assertionError) {
 684                 history.push(assertionError);
 685             }
 686         } while (history.size() < retries);
 687
 688         // No success:
 689         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 690             .map(Throwable::getMessage)
 691             .map(s -> s.replace('\n', ' '))
 692             .map(s -> s.replaceAll("\\s{2,}"," "))
 693             .distinct()
 694             .collect(joining("\n   ", "\n   ", "")), history.peek());
 695     }
 696
 697     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 698         if (pullingAssertion) {
 699             //assert pulling on inverse topic return no result
 700             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 701         }
 702
 703         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 704         assertTrue("no job fetched", job.isPresent());
 705
 706         if (pullingAssertion) {
 707             //assert another pulling on same topic return no result
 708             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 709         }
 710
 711         return job;
 712     }
 713
 714     private JobStatus inverseTopic(JobStatus topic) {
 715         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 716     }
 717
 718
 719     @Test
 720     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 721         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 722         //assert that when get ProcessingException from restMso, status remain the same
 723         List<UUID> uuids = pushMacroBulk();
 724         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 725     }
 726
 727     @Test
 728     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 729         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 730         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 731             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 732
 733         //assert that when get ProcessingException from restMso, status remain the same
 734         List<UUID> uuids = pushMacroBulk();
 735         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 736
 737
 738         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 739         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 740             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 741         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 742
 743         //the job get COMPLETE response this job is copmpleted and then also other jobs
 744         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 745             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 746         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 747
 748         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 749         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 750         assertThat(vidStatuses, is(expectedStatusesForVid));
 751     }
 752
 753     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 754         pushMacroBulk();
 755         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 756     }
 757
 758     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 759         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 760         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 761         return job;
 762     }
 763
 764     @Test
 765     public void test2BulksLifeCyclesAreIndependent() {
 766
 767         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 768         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 769         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 770         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 771         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 772         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 773
 774
 775         //create first bulk and make one job in progress
 776         List<UUID> firstBulksIDs = pushMacroBulk();
 777         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 778             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 779         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 780             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 781         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 782
 783         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 784         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 785             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 786         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 787             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 788         List<UUID> secondBulksIDs = pushMacroBulk();
 789         processJobsCountTimes(30);
 790         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 791         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 792         assertThat(statusCount.get(PENDING), is(4L));
 793
 794         //return failed to first job
 795         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 796         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 797         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 798             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 799         processJobsCountTimes(30);
 800         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 801             asyncInstantiationBL.getAllServicesInfo()
 802                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 803         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 804         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 805         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 806         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 807
 808         //return completed to all other jobs
 809         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 810             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 811         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 812             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 813         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 814             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 815
 816         processJobsCountTimes(30);
 817         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 818         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 819         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 820         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 821         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 822         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 823     }
 824
 825     protected Map<JobStatus, Long> getJobStatusesCount() {
 826         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 827     }
 828
 829     @Test
 830     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 831
 832         /*
 833         new feature: skip service (existing impl) and skip group (new impl)
 834         service+group aren't touched, 2 groups ARE created
 835
 836         [v]  success if all GROUPs success
 837
 838         Next test variation should:
 839         [ ]  error if all GROUPs error
 840         [ ]  completed with error if 1 GROUP error
 841
 842
 843         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 844         [v]    verify STATUS pending
 845         [v]  + pull+execute  (should NOT post to MSO)
 846         [v]    verify STATUS in progress; TYPE watching
 847                ...
 848
 849         [v]  + pull+execute job#1
 850         [v]    verify job#1 STATUS in progress; TYPE watching
 851
 852         [v]  + pull+execute job#6 (should post to MSO)
 853         [v]    verify job#6 STATUS resource in progress
 854         [v]  + pull+execute job#1
 855         [v]    verify job#1 STATUS in progress; TYPE watching
 856         [v]  + pull+execute job#6 (should get from MSO)
 857         [v]    verify job#6 STATUS completed
 858         [v]  + pull+execute job#1
 859         [v]    verify job#1 STATUS in progress; TYPE watching
 860
 861         [v]  + pull+execute job#7 (should post to MSO)
 862         [v]    verify job#7 STATUS resource in progress
 863         [v]  + pull+execute job#1
 864         [v]    verify job#1 STATUS in progress; TYPE watching
 865         [v]  + pull+execute job#7 (should get from MSO)
 866         [v]    verify job#7 STATUS completed
 867         [v]  + pull+execute job#1
 868         [v]    verify job#1 STATUS completed
 869
 870          */
 871
 872         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 873         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 874         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 875         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 876
 877         // Utility method
 878         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 879             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 880         };
 881
 882         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 883         UUID uuid = pushALaCarteUpdateWithGroups();
 884         singleServicesAndAssertStatus(PENDING, uuid);
 885
 886         // take from pending, put in-progress -> 3 delete-child were born
 887         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 888         verifyQueueSizes(ImmutableMap.of(
 889             IN_PROGRESS, 1
 890         ));
 891
 892         // take job #1 from phase delete to phase create -> 3 create-child were born
 893         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 894         verifyQueueSizes(ImmutableMap.of(
 895             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 896         ));
 897
 898         // prepare MSO mock
 899         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 900             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 901             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 902             .thenReturn(null);
 903         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 904             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 905
 906         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 907         // note there's no concrete mechanism that makes the first child be
 908         // the "action=None" case, but that's what happens, and following lines
 909         // relies on that fact.
 910         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 911
 912         // take each of next two children from creating to in-progress, then to completed
 913         // verify job #1 is watching, and MSO is getting requests
 914         Stream.of(1, 2).forEach(i -> {
 915             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 916             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 917             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 918
 919             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 920             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 921             verify(restMso, times(i)).GetForObject(any(), any());
 922         });
 923
 924         // job #1 is done as all children are done
 925         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 926         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 927     }
 928
 929     @DataProvider
 930     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 931         return new Object[][]{
 932             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 933             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 934                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 935             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 936                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 937             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 938                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 939             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 940                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 941             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 942                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 943             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 944                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 945         };
 946     }
 947
 948     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 949     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 950     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 951     public void vnfGropingIntegrationTest(
 952         String desc,
 953         RestObject<RequestReferencesContainer> createGroupResponse,
 954         RestObject<RequestReferencesContainer> deleteGroupResponse,
 955         RestObject<AsyncRequestStatus> createStatusResponse,
 956         RestObject<AsyncRequestStatus> deleteStatusResponse,
 957         JobStatus expectedJobStatus,
 958         int getStatusCounter) throws IOException {
 959
 960         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 961             "/serviceInstantiation/v7/instanceGroups",
 962             createGroupResponse,
 963             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 964             deleteGroupResponse,
 965             createStatusResponse,
 966             deleteStatusResponse,
 967             expectedJobStatus,
 968             getStatusCounter);
 969
 970         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 971         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 972         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 973
 974         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 975             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 976             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 977         }
 978
 979         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 980             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 981             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 982         }
 983     }
 984
 985     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 986     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 987     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 988     public void vnfsIntegrationTest(
 989         String desc,
 990         RestObject<RequestReferencesContainer> createVnfResponse,
 991         RestObject<RequestReferencesContainer> deleteVnfResponse,
 992         RestObject<AsyncRequestStatus> createStatusResponse,
 993         RestObject<AsyncRequestStatus> deleteStatusResponse,
 994         JobStatus expectedJobStatus,
 995         int getStatusCounter) throws IOException {
 996
 997         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
 998             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
 999             createVnfResponse,
1000             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
1001             deleteVnfResponse,
1002             createStatusResponse,
1003             deleteStatusResponse,
1004             expectedJobStatus,
1005             getStatusCounter);
1006     }
1007
1008     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1009     public void vfModulesIntegrationTest(
1010         String desc,
1011         RestObject<RequestReferencesContainer> createVfModuleResponse,
1012         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
1013         RestObject<AsyncRequestStatus> createStatusResponse,
1014         RestObject<AsyncRequestStatus> deleteStatusResponse,
1015         JobStatus expectedJobStatus,
1016         int getStatusCounter) throws IOException, AsdcCatalogException {
1017
1018         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1019         reset(commandUtils);
1020         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "f8360508-3f17-4414-a2ed-6bc71161e8db")).thenReturn(true);
1021         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "25284168-24bb-4698-8cb4-3f509146eca5")).thenReturn(false);
1022
1023         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
1024             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
1025             createVfModuleResponse,
1026             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
1027             deleteVfModuleResponse,
1028             createStatusResponse,
1029             deleteStatusResponse,
1030             expectedJobStatus,
1031             getStatusCounter);
1032     }
1033
1034     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
1035     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
1036     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1037     public void networksIntegrationTest(
1038         String desc,
1039         RestObject<RequestReferencesContainer> createNetworkResponse,
1040         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1041         RestObject<AsyncRequestStatus> createStatusResponse,
1042         RestObject<AsyncRequestStatus> deleteStatusResponse,
1043         JobStatus expectedJobStatus,
1044         int getStatusCounter) throws IOException {
1045
1046         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1047             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1048             createNetworkResponse,
1049             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1050             deleteNetworkResponse,
1051             createStatusResponse,
1052             deleteStatusResponse,
1053             expectedJobStatus,
1054             getStatusCounter);
1055     }
1056
1057     private UUID createAndDeleteIntegrationTest(String payload,
1058         String createPath,
1059         RestObject<RequestReferencesContainer> createResponse,
1060         String deletePath,
1061         RestObject<RequestReferencesContainer> deleteResponse,
1062         RestObject<AsyncRequestStatus> createStatusResponse,
1063         RestObject<AsyncRequestStatus> deleteStatusResponse,
1064         JobStatus expectedJobStatus,
1065         int getStatusCounter) throws IOException {
1066         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1067             TestUtils.readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1068             .get(0);
1069
1070         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1071         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1072         if  (createStatusResponse!=null) {
1073             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1074         }
1075         if  (deleteStatusResponse!=null) {
1076             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1077         }
1078
1079         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1080
1081         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1082         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1083         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1084
1085         return jobUUID;
1086     }
1087
1088     @Test
1089     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() throws IOException {
1090         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1091         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1092
1093         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1094             .thenReturn(createResponse);
1095         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1096             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1097         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1098
1099         JsonNode expectedJson = TestUtils.readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1100         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1101         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1102         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1103     }
1104
1105     private void assertErrorForResource(BaseResource resource,
1106         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1107         RestObject<AsyncRequestStatus> statusResponse) {
1108         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1109         assertThat(auditStatus, is(notNullValue()));
1110         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1111         if (statusResponse == null) {
1112             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1113             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1114             assertThat(auditStatus.getRequestId(), is(nullValue()));
1115         } else {
1116             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1117         }
1118     }
1119
1120     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1121         processJobsCountTimes(times);
1122         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1123     }
1124
1125     private void processJobsCountTimes(int times) {
1126         for (int i = 0; i < times; i++) {
1127             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1128         }
1129     }
1130
1131     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1132         processJobsCountTimes(times);
1133         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1134     }
1135
1136
1137     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1138         final Collection<Job> peek = jobsBrokerService.peek();
1139         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1140         assertThat(jobTypes, jsonEquals(expected));
1141     }
1142
1143     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1144         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1145         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1146
1147         return serviceInfoList;
1148     }
1149
1150     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1151         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1152         assertEquals(1, serviceInfoList.size());
1153         ServiceInfo serviceInfo = serviceInfoList.get(0);
1154         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1155         assertThat(serviceInfo.getJobId(), is(jobUUID));
1156         return serviceInfo;
1157     }
1158
1159     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1160         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1161         assertEquals(1, serviceInfoList.size());
1162         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1163     }
1164
1165     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1166         serviceInfoList.forEach(si->{
1167             if (si.getJobId().equals(jobUUID)) {
1168                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1169             }
1170             else {
1171                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1172             }
1173         });
1174     }
1175
1176     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1177         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1178             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1179         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1180     }
1181
1182     private String randomUuid() {
1183         return UUID.randomUUID().toString();
1184     }
1185
1186     @Test
1187     public void whenResumeService_thanExpectedResumeRequestSent() throws IOException {
1188         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1189         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1190         String resumeRequestId = randomUuid();
1191         String userId = TestUtils.generateRandomAlphaNumeric(6);
1192
1193         //prepare mocks for get all requests for instance id
1194         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1195         when(restMso.GetForObject(
1196             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1197             eq(AsyncRequestStatusList.class)))
1198             .thenReturn(getRequestByIdResponse);
1199
1200         //prepare mocks resume request
1201         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1202             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1203
1204         //prepare mocks for get resume status
1205         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1206             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1207                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1208                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1209
1210
1211         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1212         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1213         verify(restMso).GetForObject(
1214             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1215             eq(AsyncRequestStatusList.class));
1216         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1217         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1218     }
1219
1220     @Test
1221     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() throws IOException {
1222         //there is no mocks for restMSO which means restMso return bad response...
1223         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1224         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1225     }
1226
1227     @NotNull
1228     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() throws IOException {
1229         AsyncRequestStatusList asyncRequestStatusList = TestUtils.readJsonResourceFileAsObject(
1230             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1231             AsyncRequestStatusList.class);
1232         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1233         getRequestByIdResponse.set(asyncRequestStatusList);
1234         getRequestByIdResponse.setStatusCode(200);
1235         return getRequestByIdResponse;
1236     }
1237
1238     private ServiceInstantiation generateResumeMacroPayload() throws IOException {
1239         return TestUtils.readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1240     }
1241
1242     @Test
1243     public void whenUpgradingAvfModule_thanExpectedReplaceRequestSent() throws IOException, AsdcCatalogException {
1244         String instanceId = "5d49c3b1-fc90-4762-8c98-e800170baa55"; //from replace_vfmodule_fe_input.json
1245         String replaceRequestId = randomUuid();
1246         String userId = "az2016";
1247
1248
1249         //prepare mocks for newest model
1250         String newestModelUuid = "newest-model-uuid";
1251         when(commandUtils.getNewestModelUuid(eq("b16a9398-ffa3-4041-b78c-2956b8ad9c7b"))).thenReturn(newestModelUuid);
1252
1253         when(commandUtils.getServiceModel(eq(newestModelUuid))).thenReturn(generateMockLatestModelForUpgrade());
1254
1255         //prepare mocks resume request
1256         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances/e9993045-cc96-4f3f-bf9a-71b2a400a956/vnfs/5c9c2896-1fe6-4055-b7ec-d0a01e5f9bf5/vfModules/5d49c3b1-fc90-4762-8c98-e800170baa55/replace"), eq(Optional.of(userId))))
1257             .thenReturn(createResponse(202, instanceId, replaceRequestId));
1258
1259
1260         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1261             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1262                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1263                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1264
1265         ///orchestrationRequests/v7/0174b25a-dd81-45b7-b4af-0057bcc30857
1266
1267         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1268         enableAddCloudOwnerOnMsoRequest();
1269
1270         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateReplaceVfModulePayload(), userId).get(0);
1271         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1272
1273
1274
1275         JsonNode expectedJson = TestUtils.readJsonResourceFileAsObject("/payload_jsons/vfmodule/replace_vfmodule.json", JsonNode.class);
1276         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1277         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), requestCaptor.capture(), eq("/serviceInstantiation/v7/serviceInstances/e9993045-cc96-4f3f-bf9a-71b2a400a956/vnfs/5c9c2896-1fe6-4055-b7ec-d0a01e5f9bf5/vfModules/5d49c3b1-fc90-4762-8c98-e800170baa55/replace"), eq(Optional.of(userId)));
1278         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1279     }
1280
1281     private ServiceModel generateMockLatestModelForUpgrade() {
1282         ServiceModel expectedNewestModel = new ServiceModel();
1283
1284
1285         VfModule vfm = new VfModule();
1286         vfm.setModelCustomizationName("newest-model-customization-name-vfm");
1287         vfm.setCustomizationUuid("newest-model-customization-uuid-vfm");
1288         vfm.setVersion("newest-model-version-vfm");
1289         vfm.setUuid("newest-model-uuid-vfm");
1290         vfm.setName("newest-model-name-vfm");
1291         vfm.setInvariantUuid("f7a867f2-596b-4f4a-a128-421e825a6190");
1292
1293
1294         Map<String,VfModule> vfms = new HashMap<>();
1295         vfms.put("074c64d0-7e13-4bcc-8bdb-ea922331102d", vfm);
1296
1297
1298         VNF vnf = new VNF();
1299         vnf.setModelCustomizationName("newest-model-customization-name-vnf");
1300         vnf.setCustomizationUuid("newest-model-customization-uuid-vnf");
1301         vnf.setVersion("newest-model-version-vnf");
1302         vnf.setUuid("newest-model-uuid-vnf");
1303         vnf.setName("newest-model-name-vnf");
1304         vnf.setInvariantUuid("23122c9b-dd7f-483f-bf0a-e069303db2f7");
1305         vnf.setVfModules(vfms);
1306         expectedNewestModel.setVfModules(vfms);
1307
1308         Map<String,VNF> vnfs = new HashMap<>();
1309         vnfs.put("96c23a4a-6887-4b2c-9cce-1e4ea35eaade", vnf);
1310
1311         Service svc = new Service();
1312         svc.setInvariantUuid("b16a9398-ffa3-4041-b78c-2956b8ad9c7b");
1313         svc.setUuid("newest-model-uuid-service");
1314         svc.setVersion("newest-model-version-service");
1315         svc.setName("newest-model-name-service");
1316
1317         expectedNewestModel.setService(svc);
1318
1319         expectedNewestModel.setVnfs(vnfs);
1320
1321         return expectedNewestModel;
1322
1323
1324     }
1325
1326     private ServiceInstantiation generateReplaceVfModulePayload() throws IOException {
1327         return TestUtils.readJsonResourceFileAsObject("/payload_jsons/vfmodule/replace_vfmodule_fe_input.json", ServiceInstantiation.class);
1328     }
1329
1330 }