vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import static java.util.stream.Collectors.counting;
  24 import static java.util.stream.Collectors.groupingBy;
  25 import static java.util.stream.Collectors.joining;
  26 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  27 import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals;
  28 import static net.javacrumbs.jsonunit.JsonMatchers.jsonNodePresent;
  29 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  30 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartMatches;
  31 import static net.javacrumbs.jsonunit.core.Option.IGNORING_ARRAY_ORDER;
  32 import static org.hamcrest.CoreMatchers.equalTo;
  33 import static org.hamcrest.CoreMatchers.is;
  34 import static org.hamcrest.CoreMatchers.notNullValue;
  35 import static org.hamcrest.CoreMatchers.nullValue;
  36 import static org.hamcrest.MatcherAssert.assertThat;
  37 import static org.hamcrest.Matchers.hasProperty;
  38 import static org.hamcrest.Matchers.hasSize;
  39 import static org.hamcrest.core.Every.everyItem;
  40 import static org.mockito.ArgumentMatchers.any;
  41 import static org.mockito.ArgumentMatchers.argThat;
  42 import static org.mockito.ArgumentMatchers.endsWith;
  43 import static org.mockito.ArgumentMatchers.eq;
  44 import static org.mockito.ArgumentMatchers.isNull;
  45 import static org.mockito.Mockito.reset;
  46 import static org.mockito.Mockito.times;
  47 import static org.mockito.Mockito.verify;
  48 import static org.mockito.Mockito.when;
  49 import static org.onap.vid.job.Job.JobStatus.COMPLETED;
  50 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_ERRORS;
  51 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_NO_ACTION;
  52 import static org.onap.vid.job.Job.JobStatus.FAILED;
  53 import static org.onap.vid.job.Job.JobStatus.IN_PROGRESS;
  54 import static org.onap.vid.job.Job.JobStatus.PAUSE;
  55 import static org.onap.vid.job.Job.JobStatus.PENDING;
  56 import static org.onap.vid.job.Job.JobStatus.PENDING_RESOURCE;
  57 import static org.onap.vid.job.Job.JobStatus.RESOURCE_IN_PROGRESS;
  58 import static org.onap.vid.job.Job.JobStatus.STOPPED;
  59 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  60 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  61 import static org.onap.vid.testUtils.TestUtils.readJsonResourceFileAsObject;
  62 import static org.testng.Assert.assertNull;
  63 import static org.testng.AssertJUnit.assertEquals;
  64 import static org.testng.AssertJUnit.assertFalse;
  65 import static org.testng.AssertJUnit.assertTrue;
  66
  67 import com.fasterxml.jackson.databind.JsonNode;
  68 import com.google.common.collect.ImmutableList;
  69 import com.google.common.collect.ImmutableMap;
  70 import java.lang.reflect.Method;
  71 import java.util.Collection;
  72 import java.util.Collections;
  73 import java.util.List;
  74 import java.util.Map;
  75 import java.util.Optional;
  76 import java.util.Stack;
  77 import java.util.UUID;
  78 import java.util.function.BiConsumer;
  79 import java.util.function.Supplier;
  80 import java.util.stream.Collectors;
  81 import java.util.stream.IntStream;
  82 import java.util.stream.Stream;
  83 import javax.inject.Inject;
  84 import javax.ws.rs.ProcessingException;
  85 import org.apache.commons.lang3.StringUtils;
  86 import org.apache.commons.lang3.mutable.MutableInt;
  87 import org.jetbrains.annotations.NotNull;
  88 import org.mockito.ArgumentCaptor;
  89 import org.mockito.Mockito;
  90 import org.mockito.hamcrest.MockitoHamcrest;
  91 import org.onap.portalsdk.core.service.DataAccessService;
  92 import org.onap.portalsdk.core.util.SystemProperties;
  93 import org.onap.vid.asdc.AsdcCatalogException;
  94 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  95 import org.onap.vid.config.DataSourceConfig;
  96 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  97 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  98 import org.onap.vid.job.Job;
  99 import org.onap.vid.job.Job.JobStatus;
 100 import org.onap.vid.job.JobType;
 101 import org.onap.vid.job.JobsBrokerService;
 102 import org.onap.vid.job.command.CommandUtils;
 103 import org.onap.vid.job.command.InternalState;
 104 import org.onap.vid.model.Action;
 105 import org.onap.vid.model.JobAuditStatus;
 106 import org.onap.vid.model.NameCounter;
 107 import org.onap.vid.model.RequestReferencesContainer;
 108 import org.onap.vid.model.ServiceInfo;
 109 import org.onap.vid.model.ServiceModel;
 110 import org.onap.vid.model.serviceInstantiation.BaseResource;
 111 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
 112 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
 113 import org.onap.vid.mso.RestMsoImplementation;
 114 import org.onap.vid.mso.RestObject;
 115 import org.onap.vid.mso.model.RequestReferences;
 116 import org.onap.vid.mso.rest.AsyncRequestStatus;
 117 import org.onap.vid.mso.rest.AsyncRequestStatusList;
 118 import org.onap.vid.properties.Features;
 119 import org.onap.vid.services.AsyncInstantiationBaseTest;
 120 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
 121 import org.onap.vid.services.AuditService;
 122 import org.onap.vid.services.InstantiationTemplatesService;
 123 import org.onap.vid.services.VersionService;
 124 import org.onap.vid.testUtils.TestUtils;
 125 import org.onap.vid.utils.DaoUtils;
 126 import org.springframework.http.HttpMethod;
 127 import org.springframework.test.context.ContextConfiguration;
 128 import org.testng.annotations.BeforeClass;
 129 import org.testng.annotations.BeforeMethod;
 130 import org.testng.annotations.DataProvider;
 131 import org.testng.annotations.Test;
 132 import org.togglz.core.manager.FeatureManager;
 133
 134 //it's more like integration test than UT
 135 //But it's very hard to test in API test so I use UT
 136 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 137 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 138
 139     private static final String FAILED_STR = "FAILED";
 140     private static final String COMPLETE_STR = "COMPLETE";
 141     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 142     private static final String REQUESTED = "REQUESTED";
 143     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 144     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 145     private static String USER_ID =  "123";
 146     public static String REQUEST_ID = UUID.randomUUID().toString();
 147     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 148
 149     @Inject
 150     private VersionService versionService;
 151
 152     @Inject
 153     private JobsBrokerService jobsBrokerService;
 154
 155     @Inject
 156     private JobWorker jobWorker;
 157
 158     @Inject
 159     private FeatureManager featureManager;
 160
 161     @Inject
 162     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 163
 164     @Inject
 165     private AuditService auditService;
 166
 167     @Inject
 168     private RestMsoImplementation restMso;
 169
 170     @Inject
 171     private DataAccessService dataAccessService;
 172
 173     @Inject
 174     private CommandUtils commandUtils;
 175
 176     @Inject
 177     private InstantiationTemplatesService instantiationTemplates;
 178
 179     @BeforeClass
 180     void initServicesInfoService() {
 181         createInstanceParamsMaps();
 182         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 183     }
 184
 185     @BeforeMethod
 186     void clearDb() {
 187         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 188         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 189         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 190     }
 191
 192     @BeforeMethod
 193     void defineMocks() {
 194         Mockito.reset(restMso);
 195         Mockito.reset(aaiClient);
 196         Mockito.reset(commandUtils);
 197         mockAaiClientAnyNameFree();
 198
 199         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 200         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 201         when(featureManager.isActive(Features.FLAG_2006_VFMODULE_TAKES_TENANT_AND_REGION_FROM_VNF)).thenReturn(true);
 202     }
 203
 204     @Test
 205     public void whenPushNewBulk_thenAllServicesAreInPending() {
 206
 207         pushMacroBulk();
 208         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 209         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 210     }
 211
 212     private List<UUID> pushMacroBulk() {
 213         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 214             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 215             3, true,PROJECT_NAME, true);
 216         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 217     }
 218
 219     private UUID pushALaCarteWithVnf() {
 220         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 221         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 222         assertThat(uuids, hasSize(1));
 223         return uuids.get(0);
 224     }
 225
 226     private UUID pushALaCarteUpdateWithGroups() {
 227         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 228         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 229         assertThat(uuids, hasSize(1));
 230         return uuids.get(0);
 231     }
 232
 233     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 234         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 235     }
 236
 237     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 238         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 239     }
 240
 241     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 242         RequestReferences requestReferences = new RequestReferences();
 243         requestReferences.setRequestId(requestId);
 244         requestReferences.setInstanceId(instanceId);
 245         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 246         restObject.set(new RequestReferencesContainer(requestReferences));
 247         restObject.setStatusCode(statusCode);
 248         restObject.setRaw(RAW_DATA_FROM_MSO);
 249         return restObject;
 250     }
 251
 252     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 253         return Stream.of(jobStatuses).map(
 254             Enum::toString).collect(ImmutableList.toImmutableList());
 255     }
 256
 257     /*
 258     Make sure service state is in progress once request has sent to MSO
 259     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 260     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 261     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 262     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 263     */
 264     @Test
 265     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 266
 267         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 268         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 269         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 270         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 271
 272         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 273             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 274
 275         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 276             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 277             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 278             statusesToStrings(PENDING, STOPPED)
 279         );
 280
 281         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 282             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 283             ImmutableList.of(REQUESTED, FAILED_STR),
 284             ImmutableList.of()
 285         );
 286
 287         List<UUID> uuids = pushMacroBulk();
 288         UUID firstJobUuid = uuids.get(0);
 289         UUID secondJobUuid = uuids.get(1);
 290         //assert that when get ProcessingException from restMso, status remain the same
 291         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 292         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 293
 294         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 295         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 296         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 297
 298         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 299         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 300         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 301
 302         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 303         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 304         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 305
 306         //when get job COMPLETE from MSO, service status become COMPLETED
 307         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 308         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 309         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 310
 311
 312         //for use later in the test
 313         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 314             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 315         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 316
 317         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 318         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 319             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 320         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 321         processJobsCountTimes(10);
 322
 323         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 324         listServicesAndAssertStatus(expectedJobStatusMap);
 325
 326
 327         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 328         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 329         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 330         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 331         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 332             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 333         ));
 334
 335         listServicesAndAssertStatus(expectedJobStatusMap);
 336         IntStream.range(0, uuids.size()).forEach(i -> {
 337             UUID uuid = uuids.get(i);
 338             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 339             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 340         });
 341
 342         //assert no more jobs to pull
 343         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 344         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 345     }
 346
 347
 348     @DataProvider
 349     public static Object[][] AlaCarteStatuses(Method test) {
 350         return new Object[][]{
 351             {COMPLETE_STR, JobStatus.COMPLETED},
 352             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 353         };
 354     }
 355
 356     /*
 357     Make sure service state is in progress once request has sent to MSO
 358     Make sure service state is watching until state changes to complemented
 359     Make sure service state is watching until vnf state changes to completed
 360     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 361     status Creating
 362      */
 363     @Test(dataProvider = "AlaCarteStatuses")
 364     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 365         /*
 366             [v]  + push alacarte with 1 vnf
 367             [v]    verify STATUS pending
 368             [v]  + pull+execute  (should post to MSO)
 369             [v]    verify STATUS in progress
 370             [v]  + pull+execute  (should GET completed from MSO)
 371             [v]    verify STATUS in progress; TYPE watching
 372             [v]    verify job#2 *new* VNF job STATUS creating
 373             [v]  + pull+execute job#2 (should post to MSO)
 374             [v]    verify job#2 STATUS resource in progress
 375             [v]    verify job#1 STATUS in progress
 376             [v]  + pull+execute job#2 (should GET completed from MSO)
 377             [v]    verify job#2 STATUS completed
 378             [v]  + pull+execute job#1
 379             [v]    verify job#1 STATUS completed
 380
 381            * not looking on audit (yet)
 382         */
 383         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false); // this makes the test pass without mocking the vfModules
 384         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 385         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 386         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 387
 388
 389         //push alacarte with 1 vnf, verify STATUS pending
 390         UUID uuid = pushALaCarteWithVnf();
 391         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 392         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 393         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 394             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 395         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 396         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 397             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 398         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 399         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 400             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 401         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 402             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 403
 404         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 405         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 406         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 407         verify(restMso, times(2)).GetForObject(any(), any());
 408
 409     }
 410
 411     /*
 412     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 413
 414     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 415     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 416
 417     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 418     And union these tests to single one.
 419      */
 420
 421     @Test
 422     public void testALaCarteLifeCycle1Vnf2VfModules() {
 423
 424
 425         String msoVnfStatus = COMPLETE_STR;
 426         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 427         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 428         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 429         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 430         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 431         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 432         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 433         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 434
 435
 436         //push alacarte with 1 vnf, verify STATUS pending
 437         UUID uuid = pushALaCarteWithVnf();
 438         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 439
 440         /*---------- service -----------*/
 441
 442         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 443         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 444             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 445
 446         //mock mso to answer COMPLETE for service instance create
 447         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 448             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 449
 450         /*---------- vnf -----------*/
 451
 452         //mock mso to answer 200 of create vnf instance request
 453         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 454             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 455
 456         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 457         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 458             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 459
 460         reset(commandUtils);
 461         when(commandUtils.isVfModuleBaseModule(eq(SERVICE_MODEL_VERSION_ID),
 462             argThat(it -> it.getModelCustomizationId().equals(VF_MODULE_0_MODEL_CUSTOMIZATION_NAME)))).thenReturn(true);
 463         when(commandUtils.isVfModuleBaseModule(eq(SERVICE_MODEL_VERSION_ID),
 464             argThat(it -> it.getModelCustomizationId().equals(VF_MODULE_1_MODEL_CUSTOMIZATION_NAME)))).thenReturn(false);
 465
 466         /*---------- vf Module without volume group name (base) -----------*/
 467
 468         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 469         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 470             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 471         //mock mso to answer for vf module orchestration request
 472         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 473             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 474
 475         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 476             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 477         //mock mso to answer for volume group orchestration request
 478         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 479             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 480
 481         /*---------- vfModule -----------*/
 482
 483         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 484         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 485             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 486
 487         //mock mso to answer for vf module orchestration request
 488         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 489             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 490
 491         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 492         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 493         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 494         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 495         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 496         verify(restMso, times(5)).GetForObject(any(), any());
 497     }
 498
 499     @Test
 500     public void testALaCarteLifeCycle2Networks() {
 501
 502         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 503         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 504         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 505         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 506
 507         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 508         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 509         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 510         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 511         //TODO use them later for different networks
 512         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 513         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 514
 515
 516         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 517         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 518
 519         /*---------- service -----------*/
 520
 521         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 522         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 523             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 524
 525         //mock mso to answer COMPLETE for service instance create
 526         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 527             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 528
 529         final MutableInt secondNetworkCode = new MutableInt(500);
 530         final MutableInt inProgressCount = new MutableInt(0);
 531
 532         /*---------- network 1-----------*/
 533
 534         //mock mso to answer 200 of first create network instance request
 535         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 536             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 537             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 538             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 539
 540         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 541         //Once COMPLETE, second network creation will return 200
 542         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 543             thenAnswer(x->{
 544                 String status;
 545                 if (inProgressCount.getValue()<10) {
 546                     status = IN_PROGRESS_STR;
 547                 } else {
 548                     secondNetworkCode.setValue(200);
 549                     status = COMPLETE_STR;
 550                 }
 551                 inProgressCount.add(1);
 552                 return asyncRequestStatusResponseAsRestObject(status);
 553             });
 554
 555         /*---------- network 2-----------*/
 556
 557         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 558         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 559             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 560             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 561             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 562
 563 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 564 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 565         //mock mso to answer COMPLETE for network creation status,
 566
 567         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 568             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 569
 570
 571         /*---------- Create request and process it -----------*/
 572         //push alacarte with 2 networks, verify STATUS pending
 573         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 574         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 575         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 576         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 577
 578         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 579
 580         //validate the mso request id is the right one
 581         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 582         ServiceInfo serviceInfo = serviceInfoList.get(0);
 583         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 584
 585         /*---------- verify -----------*/
 586         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 587         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 588         //get status
 589         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 590         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 591         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 592     }
 593
 594     @Test
 595     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 596         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 597         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 598         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 599             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 600
 601         List<UUID> uuids = pushMacroBulk();
 602         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 603
 604         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 605         Mockito.reset(aaiClient); // must forget the "thenThrow"
 606         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 607         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 608
 609     }
 610
 611     @Test
 612     public void testAaiResponseNameUsedTillMaxRetries() {
 613         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 614         //simulate MSO to return good result, for making sure we failed because of AAI error
 615         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 616         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 617             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 618
 619         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 620         List<UUID> uuids = pushMacroBulk();
 621         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 622     }
 623
 624     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 625         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 626     }
 627
 628     //return the pulled job (and not the pushed job)
 629     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 630         Optional<Job> job = pullJob(topic, pullingAssertion);
 631
 632         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 633
 634         try {
 635             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 636
 637             if (pullingAssertion) {
 638                 //assert another pulling on same topic return no result (before push back)
 639                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 640             }
 641
 642         } finally {
 643             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 644         }
 645         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 646         return job.get();
 647     }
 648
 649     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 650         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 651         optionalJob.ifPresent(job->{
 652             Job nextJob = jobWorker.executeJobAndGetNext(job);
 653             jobsBrokerService.pushBack(nextJob);
 654         });
 655     }
 656
 657     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 658         JobType expectedNextJobType) {
 659         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 660         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 661         return job;
 662     }
 663
 664     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 665         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 666         return retryWithAssertionsLimit(retries, () -> {
 667             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 668             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 669             if (internalState != null) {
 670                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 671             }
 672             return job;
 673         });
 674     }
 675
 676     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 677         java.util.Stack<AssertionError> history = new Stack<>();
 678
 679         do {
 680             try {
 681                 return supplier.get();
 682             } catch (AssertionError assertionError) {
 683                 history.push(assertionError);
 684             }
 685         } while (history.size() < retries);
 686
 687         // No success:
 688         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 689             .map(Throwable::getMessage)
 690             .map(s -> s.replace('\n', ' '))
 691             .map(s -> s.replaceAll("\\s{2,}"," "))
 692             .distinct()
 693             .collect(joining("\n   ", "\n   ", "")), history.peek());
 694     }
 695
 696     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 697         if (pullingAssertion) {
 698             //assert pulling on inverse topic return no result
 699             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 700         }
 701
 702         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 703         assertTrue("no job fetched", job.isPresent());
 704
 705         if (pullingAssertion) {
 706             //assert another pulling on same topic return no result
 707             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 708         }
 709
 710         return job;
 711     }
 712
 713     private JobStatus inverseTopic(JobStatus topic) {
 714         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 715     }
 716
 717
 718     @Test
 719     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 720         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 721         //assert that when get ProcessingException from restMso, status remain the same
 722         List<UUID> uuids = pushMacroBulk();
 723         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 724     }
 725
 726     @Test
 727     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 728         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 729         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 730             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 731
 732         //assert that when get ProcessingException from restMso, status remain the same
 733         List<UUID> uuids = pushMacroBulk();
 734         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 735
 736
 737         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 738         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 739             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 740         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 741
 742         //the job get COMPLETE response this job is copmpleted and then also other jobs
 743         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 744             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 745         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 746
 747         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 748         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 749         assertThat(vidStatuses, is(expectedStatusesForVid));
 750     }
 751
 752     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 753         pushMacroBulk();
 754         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 755     }
 756
 757     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 758         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 759         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 760         return job;
 761     }
 762
 763     @Test
 764     public void test2BulksLifeCyclesAreIndependent() {
 765
 766         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 767         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 768         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 769         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 770         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 771         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 772
 773
 774         //create first bulk and make one job in progress
 775         List<UUID> firstBulksIDs = pushMacroBulk();
 776         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 777             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 778         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 779             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 780         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 781
 782         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 783         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 784             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 785         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 786             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 787         List<UUID> secondBulksIDs = pushMacroBulk();
 788         processJobsCountTimes(30);
 789         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 790         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 791         assertThat(statusCount.get(PENDING), is(4L));
 792
 793         //return failed to first job
 794         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 795         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 796         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 797             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 798         processJobsCountTimes(30);
 799         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 800             asyncInstantiationBL.getAllServicesInfo()
 801                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 802         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 803         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 804         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 805         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 806
 807         //return completed to all other jobs
 808         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 809             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 810         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 811             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 812         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 813             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 814
 815         processJobsCountTimes(30);
 816         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 817         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 818         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 819         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 820         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 821         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 822     }
 823
 824     protected Map<JobStatus, Long> getJobStatusesCount() {
 825         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 826     }
 827
 828     @Test
 829     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 830
 831         /*
 832         new feature: skip service (existing impl) and skip group (new impl)
 833         service+group aren't touched, 2 groups ARE created
 834
 835         [v]  success if all GROUPs success
 836
 837         Next test variation should:
 838         [ ]  error if all GROUPs error
 839         [ ]  completed with error if 1 GROUP error
 840
 841
 842         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 843         [v]    verify STATUS pending
 844         [v]  + pull+execute  (should NOT post to MSO)
 845         [v]    verify STATUS in progress; TYPE watching
 846                ...
 847
 848         [v]  + pull+execute job#1
 849         [v]    verify job#1 STATUS in progress; TYPE watching
 850
 851         [v]  + pull+execute job#6 (should post to MSO)
 852         [v]    verify job#6 STATUS resource in progress
 853         [v]  + pull+execute job#1
 854         [v]    verify job#1 STATUS in progress; TYPE watching
 855         [v]  + pull+execute job#6 (should get from MSO)
 856         [v]    verify job#6 STATUS completed
 857         [v]  + pull+execute job#1
 858         [v]    verify job#1 STATUS in progress; TYPE watching
 859
 860         [v]  + pull+execute job#7 (should post to MSO)
 861         [v]    verify job#7 STATUS resource in progress
 862         [v]  + pull+execute job#1
 863         [v]    verify job#1 STATUS in progress; TYPE watching
 864         [v]  + pull+execute job#7 (should get from MSO)
 865         [v]    verify job#7 STATUS completed
 866         [v]  + pull+execute job#1
 867         [v]    verify job#1 STATUS completed
 868
 869          */
 870
 871         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 872         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 873         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 874         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 875
 876         // Utility method
 877         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 878             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 879         };
 880
 881         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 882         UUID uuid = pushALaCarteUpdateWithGroups();
 883         singleServicesAndAssertStatus(PENDING, uuid);
 884
 885         // take from pending, put in-progress -> 3 delete-child were born
 886         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 887         verifyQueueSizes(ImmutableMap.of(
 888             IN_PROGRESS, 1
 889         ));
 890
 891         // take job #1 from phase delete to phase create -> 3 create-child were born
 892         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 893         verifyQueueSizes(ImmutableMap.of(
 894             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 895         ));
 896
 897         // prepare MSO mock
 898         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 899             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 900             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 901             .thenReturn(null);
 902         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 903             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 904
 905         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 906         // note there's no concrete mechanism that makes the first child be
 907         // the "action=None" case, but that's what happens, and following lines
 908         // relies on that fact.
 909         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 910
 911         // take each of next two children from creating to in-progress, then to completed
 912         // verify job #1 is watching, and MSO is getting requests
 913         Stream.of(1, 2).forEach(i -> {
 914             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 915             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 916             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 917
 918             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 919             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 920             verify(restMso, times(i)).GetForObject(any(), any());
 921         });
 922
 923         // job #1 is done as all children are done
 924         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 925         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 926     }
 927
 928     @DataProvider
 929     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 930         return new Object[][]{
 931             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 932             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 933                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 934             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 935                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 936             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 937                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 938             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 939                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 940             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 941                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 942             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 943                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 944         };
 945     }
 946
 947     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 948     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 949     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 950     public void vnfGropingIntegrationTest(
 951         String desc,
 952         RestObject<RequestReferencesContainer> createGroupResponse,
 953         RestObject<RequestReferencesContainer> deleteGroupResponse,
 954         RestObject<AsyncRequestStatus> createStatusResponse,
 955         RestObject<AsyncRequestStatus> deleteStatusResponse,
 956         JobStatus expectedJobStatus,
 957         int getStatusCounter) {
 958
 959         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 960             "/serviceInstantiation/v7/instanceGroups",
 961             createGroupResponse,
 962             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 963             deleteGroupResponse,
 964             createStatusResponse,
 965             deleteStatusResponse,
 966             expectedJobStatus,
 967             getStatusCounter);
 968
 969         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 970         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 971         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 972
 973         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 974             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 975             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 976         }
 977
 978         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 979             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 980             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 981         }
 982     }
 983
 984     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 985     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 986     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 987     public void vnfsIntegrationTest(
 988         String desc,
 989         RestObject<RequestReferencesContainer> createVnfResponse,
 990         RestObject<RequestReferencesContainer> deleteVnfResponse,
 991         RestObject<AsyncRequestStatus> createStatusResponse,
 992         RestObject<AsyncRequestStatus> deleteStatusResponse,
 993         JobStatus expectedJobStatus,
 994         int getStatusCounter) {
 995
 996         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
 997             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
 998             createVnfResponse,
 999             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
1000             deleteVnfResponse,
1001             createStatusResponse,
1002             deleteStatusResponse,
1003             expectedJobStatus,
1004             getStatusCounter);
1005     }
1006
1007     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1008     public void vfModulesIntegrationTest(
1009         String desc,
1010         RestObject<RequestReferencesContainer> createVfModuleResponse,
1011         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
1012         RestObject<AsyncRequestStatus> createStatusResponse,
1013         RestObject<AsyncRequestStatus> deleteStatusResponse,
1014         JobStatus expectedJobStatus,
1015         int getStatusCounter) {
1016
1017         when(commandUtils.isVfModuleBaseModule(eq("6b528779-44a3-4472-bdff-9cd15ec93450"),
1018             argThat(it -> it.getModelCustomizationName().equals("2017488PasqualeVpe..PASQUALE_base_vPE_BV..module-0")))).thenReturn(true);
1019         when(commandUtils.isVfModuleBaseModule(eq("6b528779-44a3-4472-bdff-9cd15ec93450"),
1020             argThat(it -> it.getModelCustomizationName().equals("2017488PasqualeVpe..PASQUALE_vRE_BV..module-1")))).thenReturn(false);
1021
1022         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
1023             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
1024             createVfModuleResponse,
1025             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
1026             deleteVfModuleResponse,
1027             createStatusResponse,
1028             deleteStatusResponse,
1029             expectedJobStatus,
1030             getStatusCounter);
1031     }
1032
1033     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
1034     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
1035     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1036     public void networksIntegrationTest(
1037         String desc,
1038         RestObject<RequestReferencesContainer> createNetworkResponse,
1039         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1040         RestObject<AsyncRequestStatus> createStatusResponse,
1041         RestObject<AsyncRequestStatus> deleteStatusResponse,
1042         JobStatus expectedJobStatus,
1043         int getStatusCounter) {
1044
1045         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1046             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1047             createNetworkResponse,
1048             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1049             deleteNetworkResponse,
1050             createStatusResponse,
1051             deleteStatusResponse,
1052             expectedJobStatus,
1053             getStatusCounter);
1054     }
1055
1056     private UUID createAndDeleteIntegrationTest(String payload,
1057         String createPath,
1058         RestObject<RequestReferencesContainer> createResponse,
1059         String deletePath,
1060         RestObject<RequestReferencesContainer> deleteResponse,
1061         RestObject<AsyncRequestStatus> createStatusResponse,
1062         RestObject<AsyncRequestStatus> deleteStatusResponse,
1063         JobStatus expectedJobStatus,
1064         int getStatusCounter) {
1065         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1066             readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1067             .get(0);
1068
1069         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1070         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1071         if  (createStatusResponse!=null) {
1072             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1073         }
1074         if  (deleteStatusResponse!=null) {
1075             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1076         }
1077
1078         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1079
1080         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1081         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1082         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1083
1084         return jobUUID;
1085     }
1086
1087     @Test
1088     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() {
1089         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1090         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1091
1092         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1093             .thenReturn(createResponse);
1094         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1095             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1096         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1097
1098         JsonNode expectedJson = readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1099         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1100         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1101         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1102     }
1103
1104     private void assertErrorForResource(BaseResource resource,
1105         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1106         RestObject<AsyncRequestStatus> statusResponse) {
1107         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1108         assertThat(auditStatus, is(notNullValue()));
1109         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1110         if (statusResponse == null) {
1111             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1112             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1113             assertThat(auditStatus.getRequestId(), is(nullValue()));
1114         } else {
1115             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1116         }
1117     }
1118
1119     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1120         processJobsCountTimes(times);
1121         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1122     }
1123
1124     private void processJobsCountTimes(int times) {
1125         for (int i = 0; i < times; i++) {
1126             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1127         }
1128     }
1129
1130     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1131         processJobsCountTimes(times);
1132         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1133     }
1134
1135
1136     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1137         final Collection<Job> peek = jobsBrokerService.peek();
1138         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1139         assertThat(jobTypes, jsonEquals(expected));
1140     }
1141
1142     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1143         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1144         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1145
1146         return serviceInfoList;
1147     }
1148
1149     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1150         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1151         assertEquals(1, serviceInfoList.size());
1152         ServiceInfo serviceInfo = serviceInfoList.get(0);
1153         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1154         assertThat(serviceInfo.getJobId(), is(jobUUID));
1155         return serviceInfo;
1156     }
1157
1158     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1159         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1160         assertEquals(1, serviceInfoList.size());
1161         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1162     }
1163
1164     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1165         serviceInfoList.forEach(si->{
1166             if (si.getJobId().equals(jobUUID)) {
1167                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1168             }
1169             else {
1170                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1171             }
1172         });
1173     }
1174
1175     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1176         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1177             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1178         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1179     }
1180
1181     private String randomUuid() {
1182         return UUID.randomUUID().toString();
1183     }
1184
1185     @Test
1186     public void whenResumeService_thanExpectedResumeRequestSent() {
1187         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1188         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1189         String resumeRequestId = randomUuid();
1190         String userId = TestUtils.generateRandomAlphaNumeric(6);
1191
1192         //prepare mocks for get all requests for instance id
1193         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1194         when(restMso.GetForObject(
1195             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1196             eq(AsyncRequestStatusList.class)))
1197             .thenReturn(getRequestByIdResponse);
1198
1199         //prepare mocks resume request
1200         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1201             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1202
1203         //prepare mocks for get resume status
1204         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1205             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1206                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1207                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1208
1209
1210         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1211         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1212         verify(restMso).GetForObject(
1213             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1214             eq(AsyncRequestStatusList.class));
1215         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1216         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1217     }
1218
1219     @Test
1220     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() {
1221         //there is no mocks for restMSO which means restMso return bad response...
1222         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1223         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1224     }
1225
1226     @NotNull
1227     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() {
1228         AsyncRequestStatusList asyncRequestStatusList = readJsonResourceFileAsObject(
1229             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1230             AsyncRequestStatusList.class);
1231         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1232         getRequestByIdResponse.set(asyncRequestStatusList);
1233         getRequestByIdResponse.setStatusCode(200);
1234         return getRequestByIdResponse;
1235     }
1236
1237     private ServiceInstantiation generateResumeMacroPayload() {
1238         return readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1239     }
1240
1241     @Test
1242     public void whenUpgradingVfModule_thenExpectedReplaceRequestSent() throws AsdcCatalogException {
1243         String currentServiceInstanceId = "6196ab1f-2349-4b32-9b6c-cffeb0ccc79c";
1244         String currentVnfInstanceId = "d520268f-7489-4662-be59-f81495b3a069";
1245         String currentVfModuleInstanceId = "b0732bed-3ddf-43cc-b193-7f18db84e476";
1246
1247         assertTestPayloadFitsExpectedIds(upgradeVfModulePayload(), currentServiceInstanceId, currentVnfInstanceId, currentVfModuleInstanceId);
1248
1249         String replaceRequestId = randomUuid();
1250         String userId = "az2016";
1251
1252         String modelInvariantId = "b3a1a119-dede-4ed0-b077-2a617fa519a3";
1253         String newestModelUuid = "d9a5b318-187e-476d-97f7-a15687a927a9";
1254
1255         String expectedMsoReplacePath = "/serviceInstantiation/v7/serviceInstances/"
1256             + currentServiceInstanceId + "/vnfs/" + currentVnfInstanceId + "/vfModules/" + currentVfModuleInstanceId + "/replace";
1257
1258         when(commandUtils.getNewestModelUuid(eq(modelInvariantId))).thenReturn(newestModelUuid);
1259         when(commandUtils.getServiceModel(eq(newestModelUuid))).thenReturn(newestServiceModel());
1260
1261         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(expectedMsoReplacePath), eq(Optional.of(userId))))
1262             .thenReturn(createResponse(202, currentVfModuleInstanceId, replaceRequestId));
1263
1264         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1265             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1266                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1267                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1268
1269         enableAddCloudOwnerOnMsoRequest();
1270
1271
1272         UUID jobUUID = asyncInstantiationBL.pushBulkJob(upgradeVfModulePayload(), userId).get(0);
1273         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1274
1275
1276         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1277         verify(restMso, times(1)).restCall(
1278             eq(HttpMethod.POST),
1279             eq(RequestReferencesContainer.class),
1280             requestCaptor.capture(),
1281             eq(expectedMsoReplacePath),
1282             eq(Optional.of(userId))
1283         );
1284
1285         JsonNode expectedPayloadToMso = readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__payload_to_mso.json", JsonNode.class);
1286         assertThat(requestCaptor.getValue(), jsonEquals(expectedPayloadToMso).when(IGNORING_ARRAY_ORDER));
1287     }
1288
1289     private void assertTestPayloadFitsExpectedIds(ServiceInstantiation upgradeVfModulePayload, String serviceInstanceId,
1290         String vnfInstanceId, String vfModuleInstanceId) {
1291         /*
1292         Just verifies the test and the input-file are using the same set of instance IDs
1293          */
1294         assertThat(upgradeVfModulePayload, jsonPartEquals("instanceId", serviceInstanceId));
1295         assertThat(upgradeVfModulePayload, jsonNodePresent(
1296             "vnfs"
1297                 + "." + vnfInstanceId
1298                 + ".vfModules"
1299                 + ".xbitestmodulereplace0\\.\\.XbiTestModuleReplace\\.\\.base_ocg\\.\\.module-0"
1300                 + "." + vfModuleInstanceId));
1301     }
1302
1303     private ServiceModel newestServiceModel() {
1304         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__target_newest_service_model.json", ServiceModel.class);
1305     }
1306
1307     private ServiceInstantiation upgradeVfModulePayload() {
1308         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__fe_input_cypress.json", ServiceInstantiation.class);
1309     }
1310
1311     @Test
1312     public void deployService_failIt_retryDeploy_getRetryAsTemplate_makeSureFalsyIsFailedInTemplate() {
1313
1314         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
1315
1316         //push alacarte with 1 vnf, verify STATUS pending
1317         UUID uuid = pushALaCarteWithVnf();
1318         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
1319
1320         //mock mso to answer 200 of create service instance request, verify STATUS in progress
1321         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
1322             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
1323
1324         //mock mso to answer FAILED for service instance create
1325         final RestObject<AsyncRequestStatus> failedResponse = asyncRequestStatusResponseAsRestObject(FAILED_STR);
1326         final String failureDescription = "Some deep failure";
1327         failedResponse.get().request.requestStatus.setStatusMessage(failureDescription);
1328         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
1329             thenReturn(failedResponse);
1330
1331         //Wait till job failed
1332         processJobsCountTimesAndAssertStatus(uuid, 3, FAILED);
1333
1334         //make sure retry request jas isFailed = true, and status message is with failureDescription
1335         ServiceInstantiation retryRequest = asyncInstantiationBL.getBulkForRetry(uuid);
1336         assertTrue(retryRequest.getIsFailed());
1337         assertEquals(failureDescription, retryRequest.getStatusMessage());
1338
1339         //deploy retry job and it's template
1340         UUID retryUuid = asyncInstantiationBL.pushBulkJob(retryRequest, USER_ID).get(0);
1341         ServiceInstantiation templateOfRetry = instantiationTemplates.getJobRequestAsTemplate(retryUuid);
1342
1343         //make sure the template request has isFailed = false, and no status message
1344         assertFalse(templateOfRetry.getIsFailed());
1345         assertNull(templateOfRetry.getStatusMessage());
1346     }
1347
1348 }