vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import static java.util.stream.Collectors.counting;
  24 import static java.util.stream.Collectors.groupingBy;
  25 import static java.util.stream.Collectors.joining;
  26 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  27 import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals;
  28 import static net.javacrumbs.jsonunit.JsonMatchers.jsonNodePresent;
  29 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  30 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartMatches;
  31 import static net.javacrumbs.jsonunit.core.Option.IGNORING_ARRAY_ORDER;
  32 import static org.hamcrest.CoreMatchers.equalTo;
  33 import static org.hamcrest.CoreMatchers.is;
  34 import static org.hamcrest.CoreMatchers.notNullValue;
  35 import static org.hamcrest.CoreMatchers.nullValue;
  36 import static org.hamcrest.MatcherAssert.assertThat;
  37 import static org.hamcrest.Matchers.hasProperty;
  38 import static org.hamcrest.Matchers.hasSize;
  39 import static org.hamcrest.core.Every.everyItem;
  40 import static org.mockito.ArgumentMatchers.any;
  41 import static org.mockito.ArgumentMatchers.argThat;
  42 import static org.mockito.ArgumentMatchers.endsWith;
  43 import static org.mockito.ArgumentMatchers.eq;
  44 import static org.mockito.ArgumentMatchers.isNull;
  45 import static org.mockito.Mockito.reset;
  46 import static org.mockito.Mockito.times;
  47 import static org.mockito.Mockito.verify;
  48 import static org.mockito.Mockito.when;
  49 import static org.onap.vid.job.Job.JobStatus.COMPLETED;
  50 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_ERRORS;
  51 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_NO_ACTION;
  52 import static org.onap.vid.job.Job.JobStatus.FAILED;
  53 import static org.onap.vid.job.Job.JobStatus.IN_PROGRESS;
  54 import static org.onap.vid.job.Job.JobStatus.PAUSE;
  55 import static org.onap.vid.job.Job.JobStatus.PENDING;
  56 import static org.onap.vid.job.Job.JobStatus.PENDING_RESOURCE;
  57 import static org.onap.vid.job.Job.JobStatus.RESOURCE_IN_PROGRESS;
  58 import static org.onap.vid.job.Job.JobStatus.STOPPED;
  59 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  60 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  61 import static org.onap.vid.testUtils.TestUtils.readJsonResourceFileAsObject;
  62 import static org.testng.AssertJUnit.assertEquals;
  63 import static org.testng.AssertJUnit.assertFalse;
  64 import static org.testng.AssertJUnit.assertTrue;
  65
  66 import com.fasterxml.jackson.databind.JsonNode;
  67 import com.google.common.collect.ImmutableList;
  68 import com.google.common.collect.ImmutableMap;
  69 import java.io.IOException;
  70 import java.lang.reflect.Method;
  71 import java.util.Collection;
  72 import java.util.Collections;
  73 import java.util.List;
  74 import java.util.Map;
  75 import java.util.Optional;
  76 import java.util.Stack;
  77 import java.util.UUID;
  78 import java.util.function.BiConsumer;
  79 import java.util.function.Supplier;
  80 import java.util.stream.Collectors;
  81 import java.util.stream.IntStream;
  82 import java.util.stream.Stream;
  83 import javax.inject.Inject;
  84 import javax.ws.rs.ProcessingException;
  85 import org.apache.commons.lang3.StringUtils;
  86 import org.apache.commons.lang3.mutable.MutableInt;
  87 import org.jetbrains.annotations.NotNull;
  88 import org.mockito.ArgumentCaptor;
  89 import org.mockito.Mockito;
  90 import org.mockito.hamcrest.MockitoHamcrest;
  91 import org.onap.portalsdk.core.service.DataAccessService;
  92 import org.onap.portalsdk.core.util.SystemProperties;
  93 import org.onap.vid.asdc.AsdcCatalogException;
  94 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  95 import org.onap.vid.config.DataSourceConfig;
  96 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  97 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  98 import org.onap.vid.job.Job;
  99 import org.onap.vid.job.Job.JobStatus;
 100 import org.onap.vid.job.JobType;
 101 import org.onap.vid.job.JobsBrokerService;
 102 import org.onap.vid.job.command.CommandUtils;
 103 import org.onap.vid.job.command.InternalState;
 104 import org.onap.vid.model.Action;
 105 import org.onap.vid.model.JobAuditStatus;
 106 import org.onap.vid.model.NameCounter;
 107 import org.onap.vid.model.RequestReferencesContainer;
 108 import org.onap.vid.model.ServiceInfo;
 109 import org.onap.vid.model.ServiceModel;
 110 import org.onap.vid.model.serviceInstantiation.BaseResource;
 111 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
 112 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
 113 import org.onap.vid.mso.RestMsoImplementation;
 114 import org.onap.vid.mso.RestObject;
 115 import org.onap.vid.mso.model.RequestReferences;
 116 import org.onap.vid.mso.rest.AsyncRequestStatus;
 117 import org.onap.vid.mso.rest.AsyncRequestStatusList;
 118 import org.onap.vid.properties.Features;
 119 import org.onap.vid.services.AsyncInstantiationBaseTest;
 120 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
 121 import org.onap.vid.services.AuditService;
 122 import org.onap.vid.services.VersionService;
 123 import org.onap.vid.testUtils.TestUtils;
 124 import org.onap.vid.utils.DaoUtils;
 125 import org.springframework.http.HttpMethod;
 126 import org.springframework.test.context.ContextConfiguration;
 127 import org.testng.annotations.BeforeClass;
 128 import org.testng.annotations.BeforeMethod;
 129 import org.testng.annotations.DataProvider;
 130 import org.testng.annotations.Test;
 131 import org.togglz.core.manager.FeatureManager;
 132
 133 //it's more like integration test than UT
 134 //But it's very hard to test in API test so I use UT
 135 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 136 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 137
 138     private static final String FAILED_STR = "FAILED";
 139     private static final String COMPLETE_STR = "COMPLETE";
 140     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 141     private static final String REQUESTED = "REQUESTED";
 142     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 143     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 144     private static String USER_ID =  "123";
 145     public static String REQUEST_ID = UUID.randomUUID().toString();
 146     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 147
 148     @Inject
 149     private VersionService versionService;
 150
 151     @Inject
 152     private JobsBrokerService jobsBrokerService;
 153
 154     @Inject
 155     private JobWorker jobWorker;
 156
 157     @Inject
 158     private FeatureManager featureManager;
 159
 160     @Inject
 161     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 162
 163     @Inject
 164     private AuditService auditService;
 165
 166     @Inject
 167     private RestMsoImplementation restMso;
 168
 169     @Inject
 170     private DataAccessService dataAccessService;
 171
 172     @Inject
 173     private CommandUtils commandUtils;
 174
 175     @BeforeClass
 176     void initServicesInfoService() {
 177         createInstanceParamsMaps();
 178         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 179     }
 180
 181     @BeforeMethod
 182     void clearDb() {
 183         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 184         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 185         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 186     }
 187
 188     @BeforeMethod
 189     void defineMocks() {
 190         Mockito.reset(restMso);
 191         Mockito.reset(aaiClient);
 192         mockAaiClientAnyNameFree();
 193     }
 194
 195     @Test
 196     public void whenPushNewBulk_thenAllServicesAreInPending() {
 197
 198         pushMacroBulk();
 199         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 200         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 201     }
 202
 203     private List<UUID> pushMacroBulk() {
 204         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 205             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 206             3, true,PROJECT_NAME, true);
 207         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 208     }
 209
 210     private UUID pushALaCarteWithVnf() {
 211         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 212         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 213         assertThat(uuids, hasSize(1));
 214         return uuids.get(0);
 215     }
 216
 217     private UUID pushALaCarteUpdateWithGroups() {
 218         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 219         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 220         assertThat(uuids, hasSize(1));
 221         return uuids.get(0);
 222     }
 223
 224     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 225         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 226     }
 227
 228     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 229         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 230     }
 231
 232     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 233         RequestReferences requestReferences = new RequestReferences();
 234         requestReferences.setRequestId(requestId);
 235         requestReferences.setInstanceId(instanceId);
 236         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 237         restObject.set(new RequestReferencesContainer(requestReferences));
 238         restObject.setStatusCode(statusCode);
 239         restObject.setRaw(RAW_DATA_FROM_MSO);
 240         return restObject;
 241     }
 242
 243     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 244         return Stream.of(jobStatuses).map(
 245             Enum::toString).collect(ImmutableList.toImmutableList());
 246     }
 247
 248     /*
 249     Make sure service state is in progress once request has sent to MSO
 250     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 251     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 252     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 253     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 254     */
 255     @Test
 256     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 257
 258         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 259         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 260         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 261         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 262
 263         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 264             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 265
 266         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 267             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 268             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 269             statusesToStrings(PENDING, STOPPED)
 270         );
 271
 272         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 273             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 274             ImmutableList.of(REQUESTED, FAILED_STR),
 275             ImmutableList.of()
 276         );
 277
 278         List<UUID> uuids = pushMacroBulk();
 279         UUID firstJobUuid = uuids.get(0);
 280         UUID secondJobUuid = uuids.get(1);
 281         //assert that when get ProcessingException from restMso, status remain the same
 282         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 283         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 284
 285         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 286         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 287         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 288
 289         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 290         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 291         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 292
 293         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 294         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 295         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 296
 297         //when get job COMPLETE from MSO, service status become COMPLETED
 298         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 299         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 300         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 301
 302
 303         //for use later in the test
 304         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 305             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 306         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 307
 308         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 309         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 310             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 311         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 312         processJobsCountTimes(10);
 313
 314         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 315         listServicesAndAssertStatus(expectedJobStatusMap);
 316
 317
 318         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 319         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 320         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 321         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 322         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 323             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 324         ));
 325
 326         listServicesAndAssertStatus(expectedJobStatusMap);
 327         IntStream.range(0, uuids.size()).forEach(i -> {
 328             UUID uuid = uuids.get(i);
 329             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 330             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 331         });
 332
 333         //assert no more jobs to pull
 334         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 335         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 336     }
 337
 338
 339     @DataProvider
 340     public static Object[][] AlaCarteStatuses(Method test) {
 341         return new Object[][]{
 342             {COMPLETE_STR, JobStatus.COMPLETED},
 343             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 344         };
 345     }
 346
 347     /*
 348     Make sure service state is in progress once request has sent to MSO
 349     Make sure service state is watching until state changes to complemented
 350     Make sure service state is watching until vnf state changes to completed
 351     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 352     status Creating
 353      */
 354     @Test(dataProvider = "AlaCarteStatuses")
 355     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 356         /*
 357             [v]  + push alacarte with 1 vnf
 358             [v]    verify STATUS pending
 359             [v]  + pull+execute  (should post to MSO)
 360             [v]    verify STATUS in progress
 361             [v]  + pull+execute  (should GET completed from MSO)
 362             [v]    verify STATUS in progress; TYPE watching
 363             [v]    verify job#2 *new* VNF job STATUS creating
 364             [v]  + pull+execute job#2 (should post to MSO)
 365             [v]    verify job#2 STATUS resource in progress
 366             [v]    verify job#1 STATUS in progress
 367             [v]  + pull+execute job#2 (should GET completed from MSO)
 368             [v]    verify job#2 STATUS completed
 369             [v]  + pull+execute job#1
 370             [v]    verify job#1 STATUS completed
 371
 372            * not looking on audit (yet)
 373         */
 374         reset(restMso);
 375         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 376         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false);
 377         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 378         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 379         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 380
 381
 382         //push alacarte with 1 vnf, verify STATUS pending
 383         UUID uuid = pushALaCarteWithVnf();
 384         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 385         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 386         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 387             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 388         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 389         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 390             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 391         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 392         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 393             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 394         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 395             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 396
 397         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 398         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 399         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 400         verify(restMso, times(2)).GetForObject(any(), any());
 401
 402     }
 403
 404     /*
 405     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 406
 407     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 408     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 409
 410     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 411     And union these tests to single one.
 412      */
 413
 414     @Test
 415     public void testALaCarteLifeCycle1Vnf2VfModules() {
 416
 417
 418         String msoVnfStatus = COMPLETE_STR;
 419         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 420         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 421         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 422         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 423         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 424         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 425         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 426         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 427         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 428         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 429
 430
 431         //push alacarte with 1 vnf, verify STATUS pending
 432         UUID uuid = pushALaCarteWithVnf();
 433         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 434         reset(restMso);
 435
 436         /*---------- service -----------*/
 437
 438         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 439         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 440             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 441
 442         //mock mso to answer COMPLETE for service instance create
 443         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 444             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 445
 446         /*---------- vnf -----------*/
 447
 448         //mock mso to answer 200 of create vnf instance request
 449         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 450             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 451
 452         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 453         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 454             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 455
 456         try {
 457             reset(commandUtils);
 458             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true);
 459             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false);
 460         } catch (AsdcCatalogException e) {
 461
 462         }
 463
 464         /*---------- vf Module without volume group name (base) -----------*/
 465
 466         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 467         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 468             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 469         //mock mso to answer for vf module orchestration request
 470         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 471             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 472
 473         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 474             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 475         //mock mso to answer for volume group orchestration request
 476         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 477             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 478
 479         /*---------- vfModule -----------*/
 480
 481         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 482         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 483             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 484
 485         //mock mso to answer for vf module orchestration request
 486         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 487             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 488
 489         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 490         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 491         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 492         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 493         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 494         verify(restMso, times(5)).GetForObject(any(), any());
 495     }
 496
 497     @Test
 498     public void testALaCarteLifeCycle2Networks() {
 499
 500         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 501         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 502         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 503         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 504
 505         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 506         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 507         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 508         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 509         //TODO use them later for different networks
 510         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 511         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 512
 513
 514         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 515         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 516
 517         reset(restMso);
 518
 519         /*---------- service -----------*/
 520
 521         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 522         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 523             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 524
 525         //mock mso to answer COMPLETE for service instance create
 526         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 527             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 528
 529         final MutableInt secondNetworkCode = new MutableInt(500);
 530         final MutableInt inProgressCount = new MutableInt(0);
 531
 532         /*---------- network 1-----------*/
 533
 534         //mock mso to answer 200 of first create network instance request
 535         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 536             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 537             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 538             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 539
 540         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 541         //Once COMPLETE, second network creation will return 200
 542         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 543             thenAnswer(x->{
 544                 String status;
 545                 if (inProgressCount.getValue()<10) {
 546                     status = IN_PROGRESS_STR;
 547                 } else {
 548                     secondNetworkCode.setValue(200);
 549                     status = COMPLETE_STR;
 550                 }
 551                 inProgressCount.add(1);
 552                 return asyncRequestStatusResponseAsRestObject(status);
 553             });
 554
 555         /*---------- network 2-----------*/
 556
 557         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 558         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 559             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 560             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 561             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 562
 563 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 564 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 565         //mock mso to answer COMPLETE for network creation status,
 566
 567         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 568             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 569
 570
 571         /*---------- Create request and process it -----------*/
 572         //push alacarte with 2 networks, verify STATUS pending
 573         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 574         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 575         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 576         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 577
 578         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 579
 580         //validate the mso request id is the right one
 581         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 582         ServiceInfo serviceInfo = serviceInfoList.get(0);
 583         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 584
 585         /*---------- verify -----------*/
 586         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 587         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 588         //get status
 589         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 590         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 591         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 592     }
 593
 594     @Test
 595     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 596         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 597         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 598         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 599             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 600
 601         List<UUID> uuids = pushMacroBulk();
 602         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 603
 604         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 605         Mockito.reset(aaiClient); // must forget the "thenThrow"
 606         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 607         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 608
 609     }
 610
 611     @Test
 612     public void testAaiResponseNameUsedTillMaxRetries() {
 613         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 614         //simulate MSO to return good result, for making sure we failed because of AAI error
 615         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 616         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 617             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 618
 619         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 620         List<UUID> uuids = pushMacroBulk();
 621         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 622     }
 623
 624     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 625         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 626     }
 627
 628     //return the pulled job (and not the pushed job)
 629     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 630         Optional<Job> job = pullJob(topic, pullingAssertion);
 631
 632         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 633
 634         try {
 635             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 636
 637             if (pullingAssertion) {
 638                 //assert another pulling on same topic return no result (before push back)
 639                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 640             }
 641
 642         } finally {
 643             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 644         }
 645         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 646         return job.get();
 647     }
 648
 649     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 650         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 651         optionalJob.ifPresent(job->{
 652             Job nextJob = jobWorker.executeJobAndGetNext(job);
 653             jobsBrokerService.pushBack(nextJob);
 654         });
 655     }
 656
 657     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 658         JobType expectedNextJobType) {
 659         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 660         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 661         return job;
 662     }
 663
 664     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 665         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 666         return retryWithAssertionsLimit(retries, () -> {
 667             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 668             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 669             if (internalState != null) {
 670                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 671             }
 672             return job;
 673         });
 674     }
 675
 676     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 677         java.util.Stack<AssertionError> history = new Stack<>();
 678
 679         do {
 680             try {
 681                 return supplier.get();
 682             } catch (AssertionError assertionError) {
 683                 history.push(assertionError);
 684             }
 685         } while (history.size() < retries);
 686
 687         // No success:
 688         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 689             .map(Throwable::getMessage)
 690             .map(s -> s.replace('\n', ' '))
 691             .map(s -> s.replaceAll("\\s{2,}"," "))
 692             .distinct()
 693             .collect(joining("\n   ", "\n   ", "")), history.peek());
 694     }
 695
 696     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 697         if (pullingAssertion) {
 698             //assert pulling on inverse topic return no result
 699             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 700         }
 701
 702         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 703         assertTrue("no job fetched", job.isPresent());
 704
 705         if (pullingAssertion) {
 706             //assert another pulling on same topic return no result
 707             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 708         }
 709
 710         return job;
 711     }
 712
 713     private JobStatus inverseTopic(JobStatus topic) {
 714         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 715     }
 716
 717
 718     @Test
 719     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 720         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 721         //assert that when get ProcessingException from restMso, status remain the same
 722         List<UUID> uuids = pushMacroBulk();
 723         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 724     }
 725
 726     @Test
 727     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 728         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 729         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 730             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 731
 732         //assert that when get ProcessingException from restMso, status remain the same
 733         List<UUID> uuids = pushMacroBulk();
 734         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 735
 736
 737         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 738         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 739             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 740         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 741
 742         //the job get COMPLETE response this job is copmpleted and then also other jobs
 743         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 744             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 745         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 746
 747         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 748         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 749         assertThat(vidStatuses, is(expectedStatusesForVid));
 750     }
 751
 752     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 753         pushMacroBulk();
 754         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 755     }
 756
 757     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 758         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 759         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 760         return job;
 761     }
 762
 763     @Test
 764     public void test2BulksLifeCyclesAreIndependent() {
 765
 766         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 767         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 768         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 769         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 770         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 771         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 772
 773
 774         //create first bulk and make one job in progress
 775         List<UUID> firstBulksIDs = pushMacroBulk();
 776         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 777             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 778         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 779             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 780         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 781
 782         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 783         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 784             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 785         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 786             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 787         List<UUID> secondBulksIDs = pushMacroBulk();
 788         processJobsCountTimes(30);
 789         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 790         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 791         assertThat(statusCount.get(PENDING), is(4L));
 792
 793         //return failed to first job
 794         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 795         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 796         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 797             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 798         processJobsCountTimes(30);
 799         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 800             asyncInstantiationBL.getAllServicesInfo()
 801                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 802         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 803         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 804         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 805         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 806
 807         //return completed to all other jobs
 808         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 809             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 810         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 811             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 812         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 813             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 814
 815         processJobsCountTimes(30);
 816         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 817         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 818         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 819         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 820         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 821         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 822     }
 823
 824     protected Map<JobStatus, Long> getJobStatusesCount() {
 825         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 826     }
 827
 828     @Test
 829     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 830
 831         /*
 832         new feature: skip service (existing impl) and skip group (new impl)
 833         service+group aren't touched, 2 groups ARE created
 834
 835         [v]  success if all GROUPs success
 836
 837         Next test variation should:
 838         [ ]  error if all GROUPs error
 839         [ ]  completed with error if 1 GROUP error
 840
 841
 842         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 843         [v]    verify STATUS pending
 844         [v]  + pull+execute  (should NOT post to MSO)
 845         [v]    verify STATUS in progress; TYPE watching
 846                ...
 847
 848         [v]  + pull+execute job#1
 849         [v]    verify job#1 STATUS in progress; TYPE watching
 850
 851         [v]  + pull+execute job#6 (should post to MSO)
 852         [v]    verify job#6 STATUS resource in progress
 853         [v]  + pull+execute job#1
 854         [v]    verify job#1 STATUS in progress; TYPE watching
 855         [v]  + pull+execute job#6 (should get from MSO)
 856         [v]    verify job#6 STATUS completed
 857         [v]  + pull+execute job#1
 858         [v]    verify job#1 STATUS in progress; TYPE watching
 859
 860         [v]  + pull+execute job#7 (should post to MSO)
 861         [v]    verify job#7 STATUS resource in progress
 862         [v]  + pull+execute job#1
 863         [v]    verify job#1 STATUS in progress; TYPE watching
 864         [v]  + pull+execute job#7 (should get from MSO)
 865         [v]    verify job#7 STATUS completed
 866         [v]  + pull+execute job#1
 867         [v]    verify job#1 STATUS completed
 868
 869          */
 870
 871         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 872         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 873         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 874         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 875
 876         // Utility method
 877         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 878             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 879         };
 880
 881         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 882         UUID uuid = pushALaCarteUpdateWithGroups();
 883         singleServicesAndAssertStatus(PENDING, uuid);
 884
 885         // take from pending, put in-progress -> 3 delete-child were born
 886         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 887         verifyQueueSizes(ImmutableMap.of(
 888             IN_PROGRESS, 1
 889         ));
 890
 891         // take job #1 from phase delete to phase create -> 3 create-child were born
 892         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 893         verifyQueueSizes(ImmutableMap.of(
 894             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 895         ));
 896
 897         // prepare MSO mock
 898         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 899             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 900             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 901             .thenReturn(null);
 902         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 903             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 904
 905         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 906         // note there's no concrete mechanism that makes the first child be
 907         // the "action=None" case, but that's what happens, and following lines
 908         // relies on that fact.
 909         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 910
 911         // take each of next two children from creating to in-progress, then to completed
 912         // verify job #1 is watching, and MSO is getting requests
 913         Stream.of(1, 2).forEach(i -> {
 914             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 915             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 916             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 917
 918             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 919             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 920             verify(restMso, times(i)).GetForObject(any(), any());
 921         });
 922
 923         // job #1 is done as all children are done
 924         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 925         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 926     }
 927
 928     @DataProvider
 929     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 930         return new Object[][]{
 931             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 932             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 933                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 934             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 935                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 936             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 937                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 938             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 939                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 940             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 941                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 942             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 943                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 944         };
 945     }
 946
 947     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 948     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 949     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 950     public void vnfGropingIntegrationTest(
 951         String desc,
 952         RestObject<RequestReferencesContainer> createGroupResponse,
 953         RestObject<RequestReferencesContainer> deleteGroupResponse,
 954         RestObject<AsyncRequestStatus> createStatusResponse,
 955         RestObject<AsyncRequestStatus> deleteStatusResponse,
 956         JobStatus expectedJobStatus,
 957         int getStatusCounter) {
 958
 959         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 960             "/serviceInstantiation/v7/instanceGroups",
 961             createGroupResponse,
 962             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 963             deleteGroupResponse,
 964             createStatusResponse,
 965             deleteStatusResponse,
 966             expectedJobStatus,
 967             getStatusCounter);
 968
 969         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 970         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 971         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 972
 973         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 974             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 975             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 976         }
 977
 978         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 979             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 980             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 981         }
 982     }
 983
 984     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 985     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 986     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 987     public void vnfsIntegrationTest(
 988         String desc,
 989         RestObject<RequestReferencesContainer> createVnfResponse,
 990         RestObject<RequestReferencesContainer> deleteVnfResponse,
 991         RestObject<AsyncRequestStatus> createStatusResponse,
 992         RestObject<AsyncRequestStatus> deleteStatusResponse,
 993         JobStatus expectedJobStatus,
 994         int getStatusCounter) {
 995
 996         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
 997             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
 998             createVnfResponse,
 999             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
1000             deleteVnfResponse,
1001             createStatusResponse,
1002             deleteStatusResponse,
1003             expectedJobStatus,
1004             getStatusCounter);
1005     }
1006
1007     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1008     public void vfModulesIntegrationTest(
1009         String desc,
1010         RestObject<RequestReferencesContainer> createVfModuleResponse,
1011         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
1012         RestObject<AsyncRequestStatus> createStatusResponse,
1013         RestObject<AsyncRequestStatus> deleteStatusResponse,
1014         JobStatus expectedJobStatus,
1015         int getStatusCounter) throws IOException, AsdcCatalogException {
1016
1017         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1018         reset(commandUtils);
1019         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "f8360508-3f17-4414-a2ed-6bc71161e8db")).thenReturn(true);
1020         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "25284168-24bb-4698-8cb4-3f509146eca5")).thenReturn(false);
1021
1022         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
1023             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
1024             createVfModuleResponse,
1025             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
1026             deleteVfModuleResponse,
1027             createStatusResponse,
1028             deleteStatusResponse,
1029             expectedJobStatus,
1030             getStatusCounter);
1031     }
1032
1033     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
1034     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
1035     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1036     public void networksIntegrationTest(
1037         String desc,
1038         RestObject<RequestReferencesContainer> createNetworkResponse,
1039         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1040         RestObject<AsyncRequestStatus> createStatusResponse,
1041         RestObject<AsyncRequestStatus> deleteStatusResponse,
1042         JobStatus expectedJobStatus,
1043         int getStatusCounter) {
1044
1045         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1046             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1047             createNetworkResponse,
1048             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1049             deleteNetworkResponse,
1050             createStatusResponse,
1051             deleteStatusResponse,
1052             expectedJobStatus,
1053             getStatusCounter);
1054     }
1055
1056     private UUID createAndDeleteIntegrationTest(String payload,
1057         String createPath,
1058         RestObject<RequestReferencesContainer> createResponse,
1059         String deletePath,
1060         RestObject<RequestReferencesContainer> deleteResponse,
1061         RestObject<AsyncRequestStatus> createStatusResponse,
1062         RestObject<AsyncRequestStatus> deleteStatusResponse,
1063         JobStatus expectedJobStatus,
1064         int getStatusCounter) {
1065         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1066             readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1067             .get(0);
1068
1069         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1070         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1071         if  (createStatusResponse!=null) {
1072             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1073         }
1074         if  (deleteStatusResponse!=null) {
1075             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1076         }
1077
1078         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1079
1080         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1081         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1082         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1083
1084         return jobUUID;
1085     }
1086
1087     @Test
1088     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() {
1089         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1090         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1091
1092         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1093             .thenReturn(createResponse);
1094         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1095             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1096         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1097
1098         JsonNode expectedJson = readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1099         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1100         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1101         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1102     }
1103
1104     private void assertErrorForResource(BaseResource resource,
1105         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1106         RestObject<AsyncRequestStatus> statusResponse) {
1107         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1108         assertThat(auditStatus, is(notNullValue()));
1109         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1110         if (statusResponse == null) {
1111             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1112             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1113             assertThat(auditStatus.getRequestId(), is(nullValue()));
1114         } else {
1115             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1116         }
1117     }
1118
1119     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1120         processJobsCountTimes(times);
1121         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1122     }
1123
1124     private void processJobsCountTimes(int times) {
1125         for (int i = 0; i < times; i++) {
1126             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1127         }
1128     }
1129
1130     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1131         processJobsCountTimes(times);
1132         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1133     }
1134
1135
1136     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1137         final Collection<Job> peek = jobsBrokerService.peek();
1138         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1139         assertThat(jobTypes, jsonEquals(expected));
1140     }
1141
1142     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1143         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1144         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1145
1146         return serviceInfoList;
1147     }
1148
1149     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1150         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1151         assertEquals(1, serviceInfoList.size());
1152         ServiceInfo serviceInfo = serviceInfoList.get(0);
1153         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1154         assertThat(serviceInfo.getJobId(), is(jobUUID));
1155         return serviceInfo;
1156     }
1157
1158     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1159         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1160         assertEquals(1, serviceInfoList.size());
1161         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1162     }
1163
1164     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1165         serviceInfoList.forEach(si->{
1166             if (si.getJobId().equals(jobUUID)) {
1167                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1168             }
1169             else {
1170                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1171             }
1172         });
1173     }
1174
1175     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1176         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1177             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1178         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1179     }
1180
1181     private String randomUuid() {
1182         return UUID.randomUUID().toString();
1183     }
1184
1185     @Test
1186     public void whenResumeService_thanExpectedResumeRequestSent() {
1187         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1188         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1189         String resumeRequestId = randomUuid();
1190         String userId = TestUtils.generateRandomAlphaNumeric(6);
1191
1192         //prepare mocks for get all requests for instance id
1193         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1194         when(restMso.GetForObject(
1195             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1196             eq(AsyncRequestStatusList.class)))
1197             .thenReturn(getRequestByIdResponse);
1198
1199         //prepare mocks resume request
1200         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1201             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1202
1203         //prepare mocks for get resume status
1204         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1205             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1206                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1207                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1208
1209
1210         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1211         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1212         verify(restMso).GetForObject(
1213             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1214             eq(AsyncRequestStatusList.class));
1215         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1216         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1217     }
1218
1219     @Test
1220     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() {
1221         //there is no mocks for restMSO which means restMso return bad response...
1222         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1223         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1224     }
1225
1226     @NotNull
1227     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() {
1228         AsyncRequestStatusList asyncRequestStatusList = readJsonResourceFileAsObject(
1229             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1230             AsyncRequestStatusList.class);
1231         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1232         getRequestByIdResponse.set(asyncRequestStatusList);
1233         getRequestByIdResponse.setStatusCode(200);
1234         return getRequestByIdResponse;
1235     }
1236
1237     private ServiceInstantiation generateResumeMacroPayload() {
1238         return readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1239     }
1240
1241     @Test
1242     public void whenUpgradingVfModule_thenExpectedReplaceRequestSent() throws AsdcCatalogException {
1243         String currentServiceInstanceId = "6196ab1f-2349-4b32-9b6c-cffeb0ccc79c";
1244         String currentVnfInstanceId = "d520268f-7489-4662-be59-f81495b3a069";
1245         String currentVfModuleInstanceId = "b0732bed-3ddf-43cc-b193-7f18db84e476";
1246
1247         assertTestPayloadFitsExpectedIds(upgradeVfModulePayload(), currentServiceInstanceId, currentVnfInstanceId, currentVfModuleInstanceId);
1248
1249         String replaceRequestId = randomUuid();
1250         String userId = "az2016";
1251
1252         String modelInvariantId = "b3a1a119-dede-4ed0-b077-2a617fa519a3";
1253         String newestModelUuid = "d9a5b318-187e-476d-97f7-a15687a927a9";
1254
1255         String expectedMsoReplacePath = "/serviceInstantiation/v7/serviceInstances/"
1256             + currentServiceInstanceId + "/vnfs/" + currentVnfInstanceId + "/vfModules/" + currentVfModuleInstanceId + "/replace";
1257
1258         when(commandUtils.getNewestModelUuid(eq(modelInvariantId))).thenReturn(newestModelUuid);
1259         when(commandUtils.getServiceModel(eq(newestModelUuid))).thenReturn(newestServiceModel());
1260
1261         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(expectedMsoReplacePath), eq(Optional.of(userId))))
1262             .thenReturn(createResponse(202, currentVfModuleInstanceId, replaceRequestId));
1263
1264         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1265             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1266                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1267                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1268
1269         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1270         enableAddCloudOwnerOnMsoRequest();
1271
1272
1273         UUID jobUUID = asyncInstantiationBL.pushBulkJob(upgradeVfModulePayload(), userId).get(0);
1274         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1275
1276
1277         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1278         verify(restMso, times(1)).restCall(
1279             eq(HttpMethod.POST),
1280             eq(RequestReferencesContainer.class),
1281             requestCaptor.capture(),
1282             eq(expectedMsoReplacePath),
1283             eq(Optional.of(userId))
1284         );
1285
1286         JsonNode expectedPayloadToMso = readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__payload_to_mso.json", JsonNode.class);
1287         assertThat(requestCaptor.getValue(), jsonEquals(expectedPayloadToMso).when(IGNORING_ARRAY_ORDER));
1288     }
1289
1290     private void assertTestPayloadFitsExpectedIds(ServiceInstantiation upgradeVfModulePayload, String serviceInstanceId,
1291         String vnfInstanceId, String vfModuleInstanceId) {
1292         /*
1293         Just verifies the test and the input-file are using the same set of instance IDs
1294          */
1295         assertThat(upgradeVfModulePayload, jsonPartEquals("instanceId", serviceInstanceId));
1296         assertThat(upgradeVfModulePayload, jsonNodePresent(
1297             "vnfs"
1298                 + "." + vnfInstanceId
1299                 + ".vfModules"
1300                 + ".xbitestmodulereplace0\\.\\.XbiTestModuleReplace\\.\\.base_ocg\\.\\.module-0"
1301                 + "." + vfModuleInstanceId));
1302     }
1303
1304     private ServiceModel newestServiceModel() {
1305         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__target_newest_service_model.json", ServiceModel.class);
1306     }
1307
1308     private ServiceInstantiation upgradeVfModulePayload() {
1309         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__fe_input_cypress.json", ServiceInstantiation.class);
1310     }
1311
1312 }