vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import com.fasterxml.jackson.databind.JsonNode;
  24 import com.google.common.collect.ImmutableList;
  25 import com.google.common.collect.ImmutableMap;
  26 import org.apache.commons.lang3.StringUtils;
  27 import org.apache.commons.lang3.mutable.MutableInt;
  28 import org.jetbrains.annotations.NotNull;
  29 import org.mockito.ArgumentCaptor;
  30 import org.mockito.Mockito;
  31 import org.mockito.hamcrest.MockitoHamcrest;
  32 import org.onap.portalsdk.core.service.DataAccessService;
  33 import org.onap.portalsdk.core.util.SystemProperties;
  34 import org.onap.vid.asdc.AsdcCatalogException;
  35 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  36 import org.onap.vid.job.Job;
  37 import org.onap.vid.job.Job.JobStatus;
  38 import org.onap.vid.job.JobType;
  39 import org.onap.vid.job.JobsBrokerService;
  40 import org.onap.vid.job.command.CommandUtils;
  41 import org.onap.vid.job.command.InternalState;
  42 import org.onap.vid.model.*;
  43 import org.onap.vid.model.serviceInstantiation.BaseResource;
  44 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
  45 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
  46 import org.onap.vid.mso.RestMsoImplementation;
  47 import org.onap.vid.mso.RestObject;
  48 import org.onap.vid.mso.model.RequestReferences;
  49 import org.onap.vid.mso.rest.AsyncRequestStatus;
  50 import org.onap.vid.mso.rest.AsyncRequestStatusList;
  51 import org.onap.vid.properties.Features;
  52 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
  53 import org.onap.vid.services.AuditService;
  54 import org.onap.vid.services.VersionService;
  55 import org.onap.vid.utils.DaoUtils;
  56 import org.onap.vid.config.DataSourceConfig;
  57 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  58 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  59 import org.onap.vid.services.AsyncInstantiationBaseTest;
  60 import org.onap.vid.testUtils.TestUtils;
  61 import org.springframework.http.HttpMethod;
  62 import org.springframework.test.context.ContextConfiguration;
  63 import org.testng.annotations.BeforeClass;
  64 import org.testng.annotations.BeforeMethod;
  65 import org.testng.annotations.DataProvider;
  66 import org.testng.annotations.Test;
  67 import org.togglz.core.manager.FeatureManager;
  68
  69 import javax.inject.Inject;
  70 import javax.ws.rs.ProcessingException;
  71 import java.io.IOException;
  72 import java.lang.reflect.Method;
  73 import java.util.*;
  74 import java.util.function.BiConsumer;
  75 import java.util.function.Supplier;
  76 import java.util.stream.Collectors;
  77 import java.util.stream.IntStream;
  78 import java.util.stream.Stream;
  79
  80 import static java.util.stream.Collectors.*;
  81 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  82 import static net.javacrumbs.jsonunit.JsonMatchers.*;
  83 import static org.hamcrest.CoreMatchers.*;
  84 import static org.hamcrest.MatcherAssert.assertThat;
  85 import static org.hamcrest.Matchers.hasProperty;
  86 import static org.hamcrest.Matchers.hasSize;
  87 import static org.hamcrest.core.Every.everyItem;
  88 import static org.mockito.ArgumentMatchers.any;
  89 import static org.mockito.ArgumentMatchers.endsWith;
  90 import static org.mockito.ArgumentMatchers.*;
  91 import static org.mockito.Mockito.*;
  92 import static org.onap.vid.job.Job.JobStatus.*;
  93 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  94 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  95 import static org.testng.AssertJUnit.*;
  96
  97 //it's more like integration test than UT
  98 //But it's very hard to test in API test so I use UT
  99 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 100 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 101
 102     private static final String FAILED_STR = "FAILED";
 103     private static final String COMPLETE_STR = "COMPLETE";
 104     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 105     private static final String REQUESTED = "REQUESTED";
 106     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 107     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 108     private static String USER_ID =  "123";
 109     public static String REQUEST_ID = UUID.randomUUID().toString();
 110     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 111
 112     @Inject
 113     private VersionService versionService;
 114
 115     @Inject
 116     private JobsBrokerService jobsBrokerService;
 117
 118     @Inject
 119     private JobWorker jobWorker;
 120
 121     @Inject
 122     private FeatureManager featureManager;
 123
 124     @Inject
 125     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 126
 127     @Inject
 128     private AuditService auditService;
 129
 130     @Inject
 131     private RestMsoImplementation restMso;
 132
 133     @Inject
 134     private DataAccessService dataAccessService;
 135
 136     @Inject
 137     private CommandUtils commandUtils;
 138
 139     @BeforeClass
 140     void initServicesInfoService() {
 141         createInstanceParamsMaps();
 142         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 143     }
 144
 145     @BeforeMethod
 146     void clearDb() {
 147         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 148         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 149         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 150     }
 151
 152     @BeforeMethod
 153     void defineMocks() {
 154         Mockito.reset(restMso);
 155         Mockito.reset(aaiClient);
 156         mockAaiClientAnyNameFree();
 157     }
 158
 159     @Test
 160     public void whenPushNewBulk_thenAllServicesAreInPending() {
 161
 162         pushMacroBulk();
 163         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 164         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 165     }
 166
 167     private List<UUID> pushMacroBulk() {
 168         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 169             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 170             3, true,PROJECT_NAME, true);
 171         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 172     }
 173
 174     private UUID pushALaCarteWithVnf() {
 175         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 176         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 177         assertThat(uuids, hasSize(1));
 178         return uuids.get(0);
 179     }
 180
 181     private UUID pushALaCarteUpdateWithGroups() {
 182         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 183         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 184         assertThat(uuids, hasSize(1));
 185         return uuids.get(0);
 186     }
 187
 188     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 189         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 190     }
 191
 192     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 193         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 194     }
 195
 196     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 197         RequestReferences requestReferences = new RequestReferences();
 198         requestReferences.setRequestId(requestId);
 199         requestReferences.setInstanceId(instanceId);
 200         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 201         restObject.set(new RequestReferencesContainer(requestReferences));
 202         restObject.setStatusCode(statusCode);
 203         restObject.setRaw(RAW_DATA_FROM_MSO);
 204         return restObject;
 205     }
 206
 207     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 208         return Stream.of(jobStatuses).map(
 209             Enum::toString).collect(ImmutableList.toImmutableList());
 210     }
 211
 212     /*
 213     Make sure service state is in progress once request has sent to MSO
 214     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 215     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 216     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 217     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 218     */
 219     @Test
 220     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 221
 222         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 223         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 224         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 225         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 226
 227         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 228             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 229
 230         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 231             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 232             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 233             statusesToStrings(PENDING, STOPPED)
 234         );
 235
 236         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 237             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 238             ImmutableList.of(REQUESTED, FAILED_STR),
 239             ImmutableList.of()
 240         );
 241
 242         List<UUID> uuids = pushMacroBulk();
 243         UUID firstJobUuid = uuids.get(0);
 244         UUID secondJobUuid = uuids.get(1);
 245         //assert that when get ProcessingException from restMso, status remain the same
 246         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 247         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 248
 249         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 250         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 251         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 252
 253         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 254         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 255         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 256
 257         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 258         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 259         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 260
 261         //when get job COMPLETE from MSO, service status become COMPLETED
 262         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 263         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 264         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 265
 266
 267         //for use later in the test
 268         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 269             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 270         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 271
 272         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 273         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 274             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 275         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 276         processJobsCountTimes(10);
 277
 278         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 279         listServicesAndAssertStatus(expectedJobStatusMap);
 280
 281
 282         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 283         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 284         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 285         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 286         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 287             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 288         ));
 289
 290         listServicesAndAssertStatus(expectedJobStatusMap);
 291         IntStream.range(0, uuids.size()).forEach(i -> {
 292             UUID uuid = uuids.get(i);
 293             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 294             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 295         });
 296
 297         //assert no more jobs to pull
 298         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 299         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 300     }
 301
 302
 303     @DataProvider
 304     public static Object[][] AlaCarteStatuses(Method test) {
 305         return new Object[][]{
 306             {COMPLETE_STR, JobStatus.COMPLETED},
 307             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 308         };
 309     }
 310
 311     /*
 312     Make sure service state is in progress once request has sent to MSO
 313     Make sure service state is watching until state changes to complemented
 314     Make sure service state is watching until vnf state changes to completed
 315     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 316     status Creating
 317      */
 318     @Test(dataProvider = "AlaCarteStatuses")
 319     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 320         /*
 321             [v]  + push alacarte with 1 vnf
 322             [v]    verify STATUS pending
 323             [v]  + pull+execute  (should post to MSO)
 324             [v]    verify STATUS in progress
 325             [v]  + pull+execute  (should GET completed from MSO)
 326             [v]    verify STATUS in progress; TYPE watching
 327             [v]    verify job#2 *new* VNF job STATUS creating
 328             [v]  + pull+execute job#2 (should post to MSO)
 329             [v]    verify job#2 STATUS resource in progress
 330             [v]    verify job#1 STATUS in progress
 331             [v]  + pull+execute job#2 (should GET completed from MSO)
 332             [v]    verify job#2 STATUS completed
 333             [v]  + pull+execute job#1
 334             [v]    verify job#1 STATUS completed
 335
 336            * not looking on audit (yet)
 337         */
 338         reset(restMso);
 339         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 340         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false);
 341         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 342         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 343         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 344
 345
 346         //push alacarte with 1 vnf, verify STATUS pending
 347         UUID uuid = pushALaCarteWithVnf();
 348         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 349         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 350         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 351             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 352         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 353         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 354             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 355         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 356         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 357             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 358         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 359             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 360
 361         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 362         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 363         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 364         verify(restMso, times(2)).GetForObject(any(), any());
 365
 366     }
 367
 368     /*
 369     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 370
 371     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 372     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 373
 374     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 375     And union these tests to single one.
 376      */
 377
 378     @Test
 379     public void testALaCarteLifeCycle1Vnf2VfModules() {
 380
 381
 382         String msoVnfStatus = COMPLETE_STR;
 383         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 384         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 385         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 386         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 387         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 388         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 389         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 390         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 391         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 392         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 393
 394
 395         //push alacarte with 1 vnf, verify STATUS pending
 396         UUID uuid = pushALaCarteWithVnf();
 397         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 398         reset(restMso);
 399
 400         /*---------- service -----------*/
 401
 402         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 403         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 404             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 405
 406         //mock mso to answer COMPLETE for service instance create
 407         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 408             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 409
 410         /*---------- vnf -----------*/
 411
 412         //mock mso to answer 200 of create vnf instance request
 413         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 414             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 415
 416         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 417         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 418             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 419
 420         try {
 421             reset(commandUtils);
 422             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true);
 423             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false);
 424         } catch (AsdcCatalogException e) {
 425
 426         }
 427
 428         /*---------- vf Module without volume group name (base) -----------*/
 429
 430         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 431         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 432             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 433         //mock mso to answer for vf module orchestration request
 434         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 435             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 436
 437         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 438             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 439         //mock mso to answer for volume group orchestration request
 440         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 441             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 442
 443         /*---------- vfModule -----------*/
 444
 445         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 446         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 447             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 448
 449         //mock mso to answer for vf module orchestration request
 450         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 451             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 452
 453         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 454         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 455         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 456         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 457         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 458         verify(restMso, times(5)).GetForObject(any(), any());
 459     }
 460
 461     @Test
 462     public void testALaCarteLifeCycle2Networks() {
 463
 464         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 465         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 466         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 467         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 468
 469         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 470         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 471         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 472         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 473         //TODO use them later for different networks
 474         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 475         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 476
 477
 478         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 479         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 480
 481         reset(restMso);
 482
 483         /*---------- service -----------*/
 484
 485         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 486         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 487             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 488
 489         //mock mso to answer COMPLETE for service instance create
 490         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 491             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 492
 493         final MutableInt secondNetworkCode = new MutableInt(500);
 494         final MutableInt inProgressCount = new MutableInt(0);
 495
 496         /*---------- network 1-----------*/
 497
 498         //mock mso to answer 200 of first create network instance request
 499         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 500             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 501             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 502             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 503
 504         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 505         //Once COMPLETE, second network creation will return 200
 506         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 507             thenAnswer(x->{
 508                 String status;
 509                 if (inProgressCount.getValue()<10) {
 510                     status = IN_PROGRESS_STR;
 511                 } else {
 512                     secondNetworkCode.setValue(200);
 513                     status = COMPLETE_STR;
 514                 }
 515                 inProgressCount.add(1);
 516                 return asyncRequestStatusResponseAsRestObject(status);
 517             });
 518
 519         /*---------- network 2-----------*/
 520
 521         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 522         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 523             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 524             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 525             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 526
 527 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 528 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 529         //mock mso to answer COMPLETE for network creation status,
 530
 531         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 532             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 533
 534
 535         /*---------- Create request and process it -----------*/
 536         //push alacarte with 2 networks, verify STATUS pending
 537         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 538         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 539         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 540         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 541
 542         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 543
 544         //validate the mso request id is the right one
 545         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 546         ServiceInfo serviceInfo = serviceInfoList.get(0);
 547         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 548
 549         /*---------- verify -----------*/
 550         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 551         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 552         //get status
 553         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 554         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 555         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 556     }
 557
 558     @Test
 559     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 560         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 561         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 562         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 563             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 564
 565         List<UUID> uuids = pushMacroBulk();
 566         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 567
 568         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 569         Mockito.reset(aaiClient); // must forget the "thenThrow"
 570         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 571         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 572
 573     }
 574
 575     @Test
 576     public void testAaiResponseNameUsedTillMaxRetries() {
 577         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 578         //simulate MSO to return good result, for making sure we failed because of AAI error
 579         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 580         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 581             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 582
 583         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 584         List<UUID> uuids = pushMacroBulk();
 585         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 586     }
 587
 588     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 589         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 590     }
 591
 592     //return the pulled job (and not the pushed job)
 593     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 594         Optional<Job> job = pullJob(topic, pullingAssertion);
 595
 596         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 597
 598         try {
 599             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 600
 601             if (pullingAssertion) {
 602                 //assert another pulling on same topic return no result (before push back)
 603                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 604             }
 605
 606         } finally {
 607             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 608         }
 609         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 610         return job.get();
 611     }
 612
 613     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 614         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 615         optionalJob.ifPresent(job->{
 616             Job nextJob = jobWorker.executeJobAndGetNext(job);
 617             jobsBrokerService.pushBack(nextJob);
 618         });
 619     }
 620
 621     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 622         JobType expectedNextJobType) {
 623         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 624         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 625         return job;
 626     }
 627
 628     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 629         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 630         return retryWithAssertionsLimit(retries, () -> {
 631             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 632             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 633             if (internalState != null) {
 634                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 635             }
 636             return job;
 637         });
 638     }
 639
 640     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 641         java.util.Stack<AssertionError> history = new Stack<>();
 642
 643         do {
 644             try {
 645                 return supplier.get();
 646             } catch (AssertionError assertionError) {
 647                 history.push(assertionError);
 648             }
 649         } while (history.size() < retries);
 650
 651         // No success:
 652         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 653             .map(Throwable::getMessage)
 654             .map(s -> s.replace('\n', ' '))
 655             .map(s -> s.replaceAll("\\s{2,}"," "))
 656             .distinct()
 657             .collect(joining("\n   ", "\n   ", "")), history.peek());
 658     }
 659
 660     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 661         if (pullingAssertion) {
 662             //assert pulling on inverse topic return no result
 663             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 664         }
 665
 666         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 667         assertTrue("no job fetched", job.isPresent());
 668
 669         if (pullingAssertion) {
 670             //assert another pulling on same topic return no result
 671             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 672         }
 673
 674         return job;
 675     }
 676
 677     private JobStatus inverseTopic(JobStatus topic) {
 678         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 679     }
 680
 681
 682     @Test
 683     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 684         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 685         //assert that when get ProcessingException from restMso, status remain the same
 686         List<UUID> uuids = pushMacroBulk();
 687         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 688     }
 689
 690     @Test
 691     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 692         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 693         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 694             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 695
 696         //assert that when get ProcessingException from restMso, status remain the same
 697         List<UUID> uuids = pushMacroBulk();
 698         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 699
 700
 701         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 702         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 703             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 704         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 705
 706         //the job get COMPLETE response this job is copmpleted and then also other jobs
 707         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 708             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 709         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 710
 711         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 712         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 713         assertThat(vidStatuses, is(expectedStatusesForVid));
 714     }
 715
 716     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 717         pushMacroBulk();
 718         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 719     }
 720
 721     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 722         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 723         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 724         return job;
 725     }
 726
 727     @Test
 728     public void test2BulksLifeCyclesAreIndependent() {
 729
 730         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 731         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 732         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 733         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 734         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 735         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 736
 737
 738         //create first bulk and make one job in progress
 739         List<UUID> firstBulksIDs = pushMacroBulk();
 740         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 741             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 742         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 743             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 744         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 745
 746         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 747         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 748             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 749         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 750             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 751         List<UUID> secondBulksIDs = pushMacroBulk();
 752         processJobsCountTimes(30);
 753         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 754         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 755         assertThat(statusCount.get(PENDING), is(4L));
 756
 757         //return failed to first job
 758         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 759         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 760         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 761             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 762         processJobsCountTimes(30);
 763         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 764             asyncInstantiationBL.getAllServicesInfo()
 765                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 766         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 767         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 768         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 769         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 770
 771         //return completed to all other jobs
 772         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 773             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 774         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 775             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 776         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 777             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 778
 779         processJobsCountTimes(30);
 780         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 781         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 782         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 783         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 784         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 785         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 786     }
 787
 788     protected Map<JobStatus, Long> getJobStatusesCount() {
 789         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 790     }
 791
 792     @Test
 793     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 794
 795         /*
 796         new feature: skip service (existing impl) and skip group (new impl)
 797         service+group aren't touched, 2 groups ARE created
 798
 799         [v]  success if all GROUPs success
 800
 801         Next test variation should:
 802         [ ]  error if all GROUPs error
 803         [ ]  completed with error if 1 GROUP error
 804
 805
 806         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 807         [v]    verify STATUS pending
 808         [v]  + pull+execute  (should NOT post to MSO)
 809         [v]    verify STATUS in progress; TYPE watching
 810                ...
 811
 812         [v]  + pull+execute job#1
 813         [v]    verify job#1 STATUS in progress; TYPE watching
 814
 815         [v]  + pull+execute job#6 (should post to MSO)
 816         [v]    verify job#6 STATUS resource in progress
 817         [v]  + pull+execute job#1
 818         [v]    verify job#1 STATUS in progress; TYPE watching
 819         [v]  + pull+execute job#6 (should get from MSO)
 820         [v]    verify job#6 STATUS completed
 821         [v]  + pull+execute job#1
 822         [v]    verify job#1 STATUS in progress; TYPE watching
 823
 824         [v]  + pull+execute job#7 (should post to MSO)
 825         [v]    verify job#7 STATUS resource in progress
 826         [v]  + pull+execute job#1
 827         [v]    verify job#1 STATUS in progress; TYPE watching
 828         [v]  + pull+execute job#7 (should get from MSO)
 829         [v]    verify job#7 STATUS completed
 830         [v]  + pull+execute job#1
 831         [v]    verify job#1 STATUS completed
 832
 833          */
 834
 835         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 836         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 837         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 838         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 839
 840         // Utility method
 841         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 842             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 843         };
 844
 845         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 846         UUID uuid = pushALaCarteUpdateWithGroups();
 847         singleServicesAndAssertStatus(PENDING, uuid);
 848
 849         // take from pending, put in-progress -> 3 delete-child were born
 850         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 851         verifyQueueSizes(ImmutableMap.of(
 852             IN_PROGRESS, 1
 853         ));
 854
 855         // take job #1 from phase delete to phase create -> 3 create-child were born
 856         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 857         verifyQueueSizes(ImmutableMap.of(
 858             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 859         ));
 860
 861         // prepare MSO mock
 862         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 863             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 864             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 865             .thenReturn(null);
 866         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 867             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 868
 869         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 870         // note there's no concrete mechanism that makes the first child be
 871         // the "action=None" case, but that's what happens, and following lines
 872         // relies on that fact.
 873         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 874
 875         // take each of next two children from creating to in-progress, then to completed
 876         // verify job #1 is watching, and MSO is getting requests
 877         Stream.of(1, 2).forEach(i -> {
 878             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 879             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 880             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 881
 882             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 883             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 884             verify(restMso, times(i)).GetForObject(any(), any());
 885         });
 886
 887         // job #1 is done as all children are done
 888         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 889         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 890     }
 891
 892     @DataProvider
 893     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 894         return new Object[][]{
 895             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 896             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 897                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 898             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 899                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 900             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 901                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 902             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 903                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 904             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 905                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 906             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 907                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 908         };
 909     }
 910
 911     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 912     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 913     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 914     public void vnfGropingIntegrationTest(
 915         String desc,
 916         RestObject<RequestReferencesContainer> createGroupResponse,
 917         RestObject<RequestReferencesContainer> deleteGroupResponse,
 918         RestObject<AsyncRequestStatus> createStatusResponse,
 919         RestObject<AsyncRequestStatus> deleteStatusResponse,
 920         JobStatus expectedJobStatus,
 921         int getStatusCounter) throws IOException {
 922
 923         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 924             "/serviceInstantiation/v7/instanceGroups",
 925             createGroupResponse,
 926             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 927             deleteGroupResponse,
 928             createStatusResponse,
 929             deleteStatusResponse,
 930             expectedJobStatus,
 931             getStatusCounter);
 932
 933         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 934         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 935         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 936
 937         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 938             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 939             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 940         }
 941
 942         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 943             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 944             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 945         }
 946     }
 947
 948     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 949     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 950     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 951     public void vnfsIntegrationTest(
 952         String desc,
 953         RestObject<RequestReferencesContainer> createVnfResponse,
 954         RestObject<RequestReferencesContainer> deleteVnfResponse,
 955         RestObject<AsyncRequestStatus> createStatusResponse,
 956         RestObject<AsyncRequestStatus> deleteStatusResponse,
 957         JobStatus expectedJobStatus,
 958         int getStatusCounter) throws IOException {
 959
 960         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
 961             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
 962             createVnfResponse,
 963             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
 964             deleteVnfResponse,
 965             createStatusResponse,
 966             deleteStatusResponse,
 967             expectedJobStatus,
 968             getStatusCounter);
 969     }
 970
 971     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 972     public void vfModulesIntegrationTest(
 973         String desc,
 974         RestObject<RequestReferencesContainer> createVfModuleResponse,
 975         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
 976         RestObject<AsyncRequestStatus> createStatusResponse,
 977         RestObject<AsyncRequestStatus> deleteStatusResponse,
 978         JobStatus expectedJobStatus,
 979         int getStatusCounter) throws IOException, AsdcCatalogException {
 980
 981         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 982         reset(commandUtils);
 983         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "f8360508-3f17-4414-a2ed-6bc71161e8db")).thenReturn(true);
 984         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "25284168-24bb-4698-8cb4-3f509146eca5")).thenReturn(false);
 985
 986         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
 987             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
 988             createVfModuleResponse,
 989             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
 990             deleteVfModuleResponse,
 991             createStatusResponse,
 992             deleteStatusResponse,
 993             expectedJobStatus,
 994             getStatusCounter);
 995     }
 996
 997     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
 998     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 999     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1000     public void networksIntegrationTest(
1001         String desc,
1002         RestObject<RequestReferencesContainer> createNetworkResponse,
1003         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1004         RestObject<AsyncRequestStatus> createStatusResponse,
1005         RestObject<AsyncRequestStatus> deleteStatusResponse,
1006         JobStatus expectedJobStatus,
1007         int getStatusCounter) throws IOException {
1008
1009         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1010             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1011             createNetworkResponse,
1012             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1013             deleteNetworkResponse,
1014             createStatusResponse,
1015             deleteStatusResponse,
1016             expectedJobStatus,
1017             getStatusCounter);
1018     }
1019
1020     private UUID createAndDeleteIntegrationTest(String payload,
1021         String createPath,
1022         RestObject<RequestReferencesContainer> createResponse,
1023         String deletePath,
1024         RestObject<RequestReferencesContainer> deleteResponse,
1025         RestObject<AsyncRequestStatus> createStatusResponse,
1026         RestObject<AsyncRequestStatus> deleteStatusResponse,
1027         JobStatus expectedJobStatus,
1028         int getStatusCounter) throws IOException {
1029         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1030             TestUtils.readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1031             .get(0);
1032
1033         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1034         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1035         if  (createStatusResponse!=null) {
1036             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1037         }
1038         if  (deleteStatusResponse!=null) {
1039             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1040         }
1041
1042         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1043
1044         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1045         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1046         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1047
1048         return jobUUID;
1049     }
1050
1051     @Test
1052     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() throws IOException {
1053         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1054         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1055
1056         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1057             .thenReturn(createResponse);
1058         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1059             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1060         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1061
1062         JsonNode expectedJson = TestUtils.readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1063         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1064         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1065         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1066     }
1067
1068     private void assertErrorForResource(BaseResource resource,
1069         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1070         RestObject<AsyncRequestStatus> statusResponse) {
1071         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1072         assertThat(auditStatus, is(notNullValue()));
1073         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1074         if (statusResponse == null) {
1075             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1076             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1077             assertThat(auditStatus.getRequestId(), is(nullValue()));
1078         } else {
1079             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1080         }
1081     }
1082
1083     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1084         processJobsCountTimes(times);
1085         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1086     }
1087
1088     private void processJobsCountTimes(int times) {
1089         for (int i = 0; i < times; i++) {
1090             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1091         }
1092     }
1093
1094     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1095         processJobsCountTimes(times);
1096         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1097     }
1098
1099
1100     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1101         final Collection<Job> peek = jobsBrokerService.peek();
1102         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1103         assertThat(jobTypes, jsonEquals(expected));
1104     }
1105
1106     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1107         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1108         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1109
1110         return serviceInfoList;
1111     }
1112
1113     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1114         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1115         assertEquals(1, serviceInfoList.size());
1116         ServiceInfo serviceInfo = serviceInfoList.get(0);
1117         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1118         assertThat(serviceInfo.getJobId(), is(jobUUID));
1119         return serviceInfo;
1120     }
1121
1122     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1123         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1124         assertEquals(1, serviceInfoList.size());
1125         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1126     }
1127
1128     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1129         serviceInfoList.forEach(si->{
1130             if (si.getJobId().equals(jobUUID)) {
1131                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1132             }
1133             else {
1134                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1135             }
1136         });
1137     }
1138
1139     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1140         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1141             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1142         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1143     }
1144
1145     private String randomUuid() {
1146         return UUID.randomUUID().toString();
1147     }
1148
1149     @Test
1150     public void whenResumeService_thanExpectedResumeRequestSent() throws IOException {
1151         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1152         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1153         String resumeRequestId = randomUuid();
1154         String userId = TestUtils.generateRandomAlphaNumeric(6);
1155
1156         //prepare mocks for get all requests for instance id
1157         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1158         when(restMso.GetForObject(
1159             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1160             eq(AsyncRequestStatusList.class)))
1161             .thenReturn(getRequestByIdResponse);
1162
1163         //prepare mocks resume request
1164         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1165             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1166
1167         //prepare mocks for get resume status
1168         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1169             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1170                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1171                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1172
1173
1174         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1175         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1176         verify(restMso).GetForObject(
1177             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1178             eq(AsyncRequestStatusList.class));
1179         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1180         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1181     }
1182
1183     @Test
1184     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() throws IOException {
1185         //there is no mocks for restMSO which means restMso return bad response...
1186         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1187         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1188     }
1189
1190     @NotNull
1191     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() throws IOException {
1192         AsyncRequestStatusList asyncRequestStatusList = TestUtils.readJsonResourceFileAsObject(
1193             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1194             AsyncRequestStatusList.class);
1195         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1196         getRequestByIdResponse.set(asyncRequestStatusList);
1197         getRequestByIdResponse.setStatusCode(200);
1198         return getRequestByIdResponse;
1199     }
1200
1201     private ServiceInstantiation generateResumeMacroPayload() throws IOException {
1202         return TestUtils.readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1203     }
1204
1205     @Test
1206     public void whenUpgradingAvfModule_thanExpectedReplaceRequestSent() throws IOException {
1207         String instanceId = "5d49c3b1-fc90-4762-8c98-e800170baa55"; //from feRequestResumeMacroService.json
1208         String replaceRequestId = randomUuid();
1209         String userId = "az2016";
1210
1211
1212         //prepare mocks resume request
1213         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances/e9993045-cc96-4f3f-bf9a-71b2a400a956/vnfs/5c9c2896-1fe6-4055-b7ec-d0a01e5f9bf5/vfModules/5d49c3b1-fc90-4762-8c98-e800170baa55/replace"), eq(Optional.of(userId))))
1214             .thenReturn(createResponse(202, instanceId, replaceRequestId));
1215
1216
1217         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1218             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1219                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1220                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1221
1222         ///orchestrationRequests/v7/0174b25a-dd81-45b7-b4af-0057bcc30857
1223
1224         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1225         enableAddCloudOwnerOnMsoRequest();
1226
1227         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateReplaceVfModulePayload(), userId).get(0);
1228         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1229
1230
1231
1232         JsonNode expectedJson = TestUtils.readJsonResourceFileAsObject("/payload_jsons/vfmodule/replace_vfmodule.json", JsonNode.class);
1233         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1234         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), requestCaptor.capture(), eq("/serviceInstantiation/v7/serviceInstances/e9993045-cc96-4f3f-bf9a-71b2a400a956/vnfs/5c9c2896-1fe6-4055-b7ec-d0a01e5f9bf5/vfModules/5d49c3b1-fc90-4762-8c98-e800170baa55/replace"), eq(Optional.of(userId)));
1235         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1236     }
1237
1238     private ServiceInstantiation generateReplaceVfModulePayload() throws IOException {
1239         return TestUtils.readJsonResourceFileAsObject("/payload_jsons/vfmodule/replace_vfmodule_fe_input.json", ServiceInstantiation.class);
1240     }
1241 }