vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import static java.util.stream.Collectors.counting;
  24 import static java.util.stream.Collectors.groupingBy;
  25 import static java.util.stream.Collectors.joining;
  26 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  27 import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals;
  28 import static net.javacrumbs.jsonunit.JsonMatchers.jsonNodePresent;
  29 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  30 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartMatches;
  31 import static net.javacrumbs.jsonunit.core.Option.IGNORING_ARRAY_ORDER;
  32 import static org.hamcrest.CoreMatchers.equalTo;
  33 import static org.hamcrest.CoreMatchers.is;
  34 import static org.hamcrest.CoreMatchers.notNullValue;
  35 import static org.hamcrest.CoreMatchers.nullValue;
  36 import static org.hamcrest.MatcherAssert.assertThat;
  37 import static org.hamcrest.Matchers.hasProperty;
  38 import static org.hamcrest.Matchers.hasSize;
  39 import static org.hamcrest.core.Every.everyItem;
  40 import static org.mockito.ArgumentMatchers.any;
  41 import static org.mockito.ArgumentMatchers.argThat;
  42 import static org.mockito.ArgumentMatchers.endsWith;
  43 import static org.mockito.ArgumentMatchers.eq;
  44 import static org.mockito.ArgumentMatchers.isNull;
  45 import static org.mockito.Mockito.reset;
  46 import static org.mockito.Mockito.times;
  47 import static org.mockito.Mockito.verify;
  48 import static org.mockito.Mockito.when;
  49 import static org.onap.vid.job.Job.JobStatus.COMPLETED;
  50 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_ERRORS;
  51 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_NO_ACTION;
  52 import static org.onap.vid.job.Job.JobStatus.FAILED;
  53 import static org.onap.vid.job.Job.JobStatus.IN_PROGRESS;
  54 import static org.onap.vid.job.Job.JobStatus.PAUSE;
  55 import static org.onap.vid.job.Job.JobStatus.PENDING;
  56 import static org.onap.vid.job.Job.JobStatus.PENDING_RESOURCE;
  57 import static org.onap.vid.job.Job.JobStatus.RESOURCE_IN_PROGRESS;
  58 import static org.onap.vid.job.Job.JobStatus.STOPPED;
  59 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  60 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  61 import static org.onap.vid.testUtils.TestUtils.readJsonResourceFileAsObject;
  62 import static org.testng.Assert.assertNull;
  63 import static org.testng.AssertJUnit.assertEquals;
  64 import static org.testng.AssertJUnit.assertFalse;
  65 import static org.testng.AssertJUnit.assertTrue;
  66
  67 import com.fasterxml.jackson.databind.JsonNode;
  68 import com.google.common.collect.ImmutableList;
  69 import com.google.common.collect.ImmutableMap;
  70 import java.io.IOException;
  71 import java.lang.reflect.Method;
  72 import java.util.Collection;
  73 import java.util.Collections;
  74 import java.util.List;
  75 import java.util.Map;
  76 import java.util.Optional;
  77 import java.util.Stack;
  78 import java.util.UUID;
  79 import java.util.function.BiConsumer;
  80 import java.util.function.Supplier;
  81 import java.util.stream.Collectors;
  82 import java.util.stream.IntStream;
  83 import java.util.stream.Stream;
  84 import javax.inject.Inject;
  85 import javax.ws.rs.ProcessingException;
  86 import org.apache.commons.lang3.StringUtils;
  87 import org.apache.commons.lang3.mutable.MutableInt;
  88 import org.jetbrains.annotations.NotNull;
  89 import org.mockito.ArgumentCaptor;
  90 import org.mockito.Mockito;
  91 import org.mockito.hamcrest.MockitoHamcrest;
  92 import org.onap.portalsdk.core.service.DataAccessService;
  93 import org.onap.portalsdk.core.util.SystemProperties;
  94 import org.onap.vid.asdc.AsdcCatalogException;
  95 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  96 import org.onap.vid.config.DataSourceConfig;
  97 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  98 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  99 import org.onap.vid.job.Job;
 100 import org.onap.vid.job.Job.JobStatus;
 101 import org.onap.vid.job.JobType;
 102 import org.onap.vid.job.JobsBrokerService;
 103 import org.onap.vid.job.command.CommandUtils;
 104 import org.onap.vid.job.command.InternalState;
 105 import org.onap.vid.model.Action;
 106 import org.onap.vid.model.JobAuditStatus;
 107 import org.onap.vid.model.NameCounter;
 108 import org.onap.vid.model.RequestReferencesContainer;
 109 import org.onap.vid.model.ServiceInfo;
 110 import org.onap.vid.model.ServiceModel;
 111 import org.onap.vid.model.serviceInstantiation.BaseResource;
 112 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
 113 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
 114 import org.onap.vid.mso.RestMsoImplementation;
 115 import org.onap.vid.mso.RestObject;
 116 import org.onap.vid.mso.model.RequestReferences;
 117 import org.onap.vid.mso.rest.AsyncRequestStatus;
 118 import org.onap.vid.mso.rest.AsyncRequestStatusList;
 119 import org.onap.vid.properties.Features;
 120 import org.onap.vid.services.AsyncInstantiationBaseTest;
 121 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
 122 import org.onap.vid.services.AuditService;
 123 import org.onap.vid.services.InstantiationTemplatesService;
 124 import org.onap.vid.services.VersionService;
 125 import org.onap.vid.testUtils.TestUtils;
 126 import org.onap.vid.utils.DaoUtils;
 127 import org.springframework.http.HttpMethod;
 128 import org.springframework.test.context.ContextConfiguration;
 129 import org.testng.annotations.BeforeClass;
 130 import org.testng.annotations.BeforeMethod;
 131 import org.testng.annotations.DataProvider;
 132 import org.testng.annotations.Test;
 133 import org.togglz.core.manager.FeatureManager;
 134
 135 //it's more like integration test than UT
 136 //But it's very hard to test in API test so I use UT
 137 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 138 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 139
 140     private static final String FAILED_STR = "FAILED";
 141     private static final String COMPLETE_STR = "COMPLETE";
 142     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 143     private static final String REQUESTED = "REQUESTED";
 144     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 145     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 146     private static String USER_ID =  "123";
 147     public static String REQUEST_ID = UUID.randomUUID().toString();
 148     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 149
 150     @Inject
 151     private VersionService versionService;
 152
 153     @Inject
 154     private JobsBrokerService jobsBrokerService;
 155
 156     @Inject
 157     private JobWorker jobWorker;
 158
 159     @Inject
 160     private FeatureManager featureManager;
 161
 162     @Inject
 163     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 164
 165     @Inject
 166     private AuditService auditService;
 167
 168     @Inject
 169     private RestMsoImplementation restMso;
 170
 171     @Inject
 172     private DataAccessService dataAccessService;
 173
 174     @Inject
 175     private CommandUtils commandUtils;
 176
 177     @Inject
 178     private InstantiationTemplatesService instantiationTemplates;
 179
 180     @BeforeClass
 181     void initServicesInfoService() {
 182         createInstanceParamsMaps();
 183         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 184     }
 185
 186     @BeforeMethod
 187     void clearDb() {
 188         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 189         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 190         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 191     }
 192
 193     @BeforeMethod
 194     void defineMocks() {
 195         Mockito.reset(restMso);
 196         Mockito.reset(aaiClient);
 197         mockAaiClientAnyNameFree();
 198     }
 199
 200     @Test
 201     public void whenPushNewBulk_thenAllServicesAreInPending() {
 202
 203         pushMacroBulk();
 204         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 205         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 206     }
 207
 208     private List<UUID> pushMacroBulk() {
 209         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 210             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 211             3, true,PROJECT_NAME, true);
 212         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 213     }
 214
 215     private UUID pushALaCarteWithVnf() {
 216         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 217         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 218         assertThat(uuids, hasSize(1));
 219         return uuids.get(0);
 220     }
 221
 222     private UUID pushALaCarteUpdateWithGroups() {
 223         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 224         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 225         assertThat(uuids, hasSize(1));
 226         return uuids.get(0);
 227     }
 228
 229     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 230         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 231     }
 232
 233     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 234         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 235     }
 236
 237     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 238         RequestReferences requestReferences = new RequestReferences();
 239         requestReferences.setRequestId(requestId);
 240         requestReferences.setInstanceId(instanceId);
 241         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 242         restObject.set(new RequestReferencesContainer(requestReferences));
 243         restObject.setStatusCode(statusCode);
 244         restObject.setRaw(RAW_DATA_FROM_MSO);
 245         return restObject;
 246     }
 247
 248     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 249         return Stream.of(jobStatuses).map(
 250             Enum::toString).collect(ImmutableList.toImmutableList());
 251     }
 252
 253     /*
 254     Make sure service state is in progress once request has sent to MSO
 255     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 256     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 257     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 258     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 259     */
 260     @Test
 261     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 262
 263         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 264         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 265         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 266         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 267
 268         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 269             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 270
 271         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 272             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 273             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 274             statusesToStrings(PENDING, STOPPED)
 275         );
 276
 277         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 278             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 279             ImmutableList.of(REQUESTED, FAILED_STR),
 280             ImmutableList.of()
 281         );
 282
 283         List<UUID> uuids = pushMacroBulk();
 284         UUID firstJobUuid = uuids.get(0);
 285         UUID secondJobUuid = uuids.get(1);
 286         //assert that when get ProcessingException from restMso, status remain the same
 287         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 288         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 289
 290         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 291         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 292         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 293
 294         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 295         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 296         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 297
 298         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 299         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 300         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 301
 302         //when get job COMPLETE from MSO, service status become COMPLETED
 303         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 304         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 305         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 306
 307
 308         //for use later in the test
 309         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 310             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 311         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 312
 313         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 314         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 315             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 316         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 317         processJobsCountTimes(10);
 318
 319         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 320         listServicesAndAssertStatus(expectedJobStatusMap);
 321
 322
 323         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 324         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 325         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 326         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 327         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 328             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 329         ));
 330
 331         listServicesAndAssertStatus(expectedJobStatusMap);
 332         IntStream.range(0, uuids.size()).forEach(i -> {
 333             UUID uuid = uuids.get(i);
 334             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 335             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 336         });
 337
 338         //assert no more jobs to pull
 339         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 340         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 341     }
 342
 343
 344     @DataProvider
 345     public static Object[][] AlaCarteStatuses(Method test) {
 346         return new Object[][]{
 347             {COMPLETE_STR, JobStatus.COMPLETED},
 348             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 349         };
 350     }
 351
 352     /*
 353     Make sure service state is in progress once request has sent to MSO
 354     Make sure service state is watching until state changes to complemented
 355     Make sure service state is watching until vnf state changes to completed
 356     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 357     status Creating
 358      */
 359     @Test(dataProvider = "AlaCarteStatuses")
 360     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 361         /*
 362             [v]  + push alacarte with 1 vnf
 363             [v]    verify STATUS pending
 364             [v]  + pull+execute  (should post to MSO)
 365             [v]    verify STATUS in progress
 366             [v]  + pull+execute  (should GET completed from MSO)
 367             [v]    verify STATUS in progress; TYPE watching
 368             [v]    verify job#2 *new* VNF job STATUS creating
 369             [v]  + pull+execute job#2 (should post to MSO)
 370             [v]    verify job#2 STATUS resource in progress
 371             [v]    verify job#1 STATUS in progress
 372             [v]  + pull+execute job#2 (should GET completed from MSO)
 373             [v]    verify job#2 STATUS completed
 374             [v]  + pull+execute job#1
 375             [v]    verify job#1 STATUS completed
 376
 377            * not looking on audit (yet)
 378         */
 379         reset(restMso);
 380         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 381         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false);
 382         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 383         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 384         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 385
 386
 387         //push alacarte with 1 vnf, verify STATUS pending
 388         UUID uuid = pushALaCarteWithVnf();
 389         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 390         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 391         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 392             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 393         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 394         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 395             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 396         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 397         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 398             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 399         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 400             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 401
 402         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 403         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 404         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 405         verify(restMso, times(2)).GetForObject(any(), any());
 406
 407     }
 408
 409     /*
 410     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 411
 412     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 413     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 414
 415     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 416     And union these tests to single one.
 417      */
 418
 419     @Test
 420     public void testALaCarteLifeCycle1Vnf2VfModules() {
 421
 422
 423         String msoVnfStatus = COMPLETE_STR;
 424         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 425         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 426         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 427         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 428         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 429         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 430         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 431         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 432         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 433         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 434
 435
 436         //push alacarte with 1 vnf, verify STATUS pending
 437         UUID uuid = pushALaCarteWithVnf();
 438         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 439         reset(restMso);
 440
 441         /*---------- service -----------*/
 442
 443         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 444         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 445             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 446
 447         //mock mso to answer COMPLETE for service instance create
 448         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 449             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 450
 451         /*---------- vnf -----------*/
 452
 453         //mock mso to answer 200 of create vnf instance request
 454         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 455             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 456
 457         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 458         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 459             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 460
 461         try {
 462             reset(commandUtils);
 463             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true);
 464             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false);
 465         } catch (AsdcCatalogException e) {
 466
 467         }
 468
 469         /*---------- vf Module without volume group name (base) -----------*/
 470
 471         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 472         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 473             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 474         //mock mso to answer for vf module orchestration request
 475         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 476             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 477
 478         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 479             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 480         //mock mso to answer for volume group orchestration request
 481         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 482             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 483
 484         /*---------- vfModule -----------*/
 485
 486         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 487         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 488             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 489
 490         //mock mso to answer for vf module orchestration request
 491         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 492             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 493
 494         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 495         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 496         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 497         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 498         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 499         verify(restMso, times(5)).GetForObject(any(), any());
 500     }
 501
 502     @Test
 503     public void testALaCarteLifeCycle2Networks() {
 504
 505         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 506         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 507         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 508         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 509
 510         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 511         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 512         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 513         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 514         //TODO use them later for different networks
 515         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 516         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 517
 518
 519         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 520         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 521
 522         reset(restMso);
 523
 524         /*---------- service -----------*/
 525
 526         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 527         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 528             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 529
 530         //mock mso to answer COMPLETE for service instance create
 531         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 532             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 533
 534         final MutableInt secondNetworkCode = new MutableInt(500);
 535         final MutableInt inProgressCount = new MutableInt(0);
 536
 537         /*---------- network 1-----------*/
 538
 539         //mock mso to answer 200 of first create network instance request
 540         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 541             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 542             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 543             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 544
 545         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 546         //Once COMPLETE, second network creation will return 200
 547         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 548             thenAnswer(x->{
 549                 String status;
 550                 if (inProgressCount.getValue()<10) {
 551                     status = IN_PROGRESS_STR;
 552                 } else {
 553                     secondNetworkCode.setValue(200);
 554                     status = COMPLETE_STR;
 555                 }
 556                 inProgressCount.add(1);
 557                 return asyncRequestStatusResponseAsRestObject(status);
 558             });
 559
 560         /*---------- network 2-----------*/
 561
 562         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 563         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 564             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 565             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 566             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 567
 568 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 569 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 570         //mock mso to answer COMPLETE for network creation status,
 571
 572         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 573             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 574
 575
 576         /*---------- Create request and process it -----------*/
 577         //push alacarte with 2 networks, verify STATUS pending
 578         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 579         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 580         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 581         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 582
 583         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 584
 585         //validate the mso request id is the right one
 586         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 587         ServiceInfo serviceInfo = serviceInfoList.get(0);
 588         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 589
 590         /*---------- verify -----------*/
 591         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 592         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 593         //get status
 594         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 595         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 596         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 597     }
 598
 599     @Test
 600     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 601         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 602         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 603         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 604             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 605
 606         List<UUID> uuids = pushMacroBulk();
 607         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 608
 609         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 610         Mockito.reset(aaiClient); // must forget the "thenThrow"
 611         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 612         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 613
 614     }
 615
 616     @Test
 617     public void testAaiResponseNameUsedTillMaxRetries() {
 618         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 619         //simulate MSO to return good result, for making sure we failed because of AAI error
 620         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 621         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 622             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 623
 624         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 625         List<UUID> uuids = pushMacroBulk();
 626         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 627     }
 628
 629     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 630         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 631     }
 632
 633     //return the pulled job (and not the pushed job)
 634     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 635         Optional<Job> job = pullJob(topic, pullingAssertion);
 636
 637         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 638
 639         try {
 640             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 641
 642             if (pullingAssertion) {
 643                 //assert another pulling on same topic return no result (before push back)
 644                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 645             }
 646
 647         } finally {
 648             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 649         }
 650         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 651         return job.get();
 652     }
 653
 654     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 655         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 656         optionalJob.ifPresent(job->{
 657             Job nextJob = jobWorker.executeJobAndGetNext(job);
 658             jobsBrokerService.pushBack(nextJob);
 659         });
 660     }
 661
 662     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 663         JobType expectedNextJobType) {
 664         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 665         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 666         return job;
 667     }
 668
 669     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 670         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 671         return retryWithAssertionsLimit(retries, () -> {
 672             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 673             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 674             if (internalState != null) {
 675                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 676             }
 677             return job;
 678         });
 679     }
 680
 681     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 682         java.util.Stack<AssertionError> history = new Stack<>();
 683
 684         do {
 685             try {
 686                 return supplier.get();
 687             } catch (AssertionError assertionError) {
 688                 history.push(assertionError);
 689             }
 690         } while (history.size() < retries);
 691
 692         // No success:
 693         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 694             .map(Throwable::getMessage)
 695             .map(s -> s.replace('\n', ' '))
 696             .map(s -> s.replaceAll("\\s{2,}"," "))
 697             .distinct()
 698             .collect(joining("\n   ", "\n   ", "")), history.peek());
 699     }
 700
 701     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 702         if (pullingAssertion) {
 703             //assert pulling on inverse topic return no result
 704             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 705         }
 706
 707         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 708         assertTrue("no job fetched", job.isPresent());
 709
 710         if (pullingAssertion) {
 711             //assert another pulling on same topic return no result
 712             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 713         }
 714
 715         return job;
 716     }
 717
 718     private JobStatus inverseTopic(JobStatus topic) {
 719         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 720     }
 721
 722
 723     @Test
 724     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 725         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 726         //assert that when get ProcessingException from restMso, status remain the same
 727         List<UUID> uuids = pushMacroBulk();
 728         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 729     }
 730
 731     @Test
 732     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 733         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 734         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 735             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 736
 737         //assert that when get ProcessingException from restMso, status remain the same
 738         List<UUID> uuids = pushMacroBulk();
 739         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 740
 741
 742         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 743         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 744             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 745         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 746
 747         //the job get COMPLETE response this job is copmpleted and then also other jobs
 748         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 749             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 750         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 751
 752         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 753         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 754         assertThat(vidStatuses, is(expectedStatusesForVid));
 755     }
 756
 757     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 758         pushMacroBulk();
 759         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 760     }
 761
 762     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 763         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 764         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 765         return job;
 766     }
 767
 768     @Test
 769     public void test2BulksLifeCyclesAreIndependent() {
 770
 771         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 772         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 773         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 774         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 775         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 776         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 777
 778
 779         //create first bulk and make one job in progress
 780         List<UUID> firstBulksIDs = pushMacroBulk();
 781         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 782             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 783         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 784             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 785         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 786
 787         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 788         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 789             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 790         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 791             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 792         List<UUID> secondBulksIDs = pushMacroBulk();
 793         processJobsCountTimes(30);
 794         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 795         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 796         assertThat(statusCount.get(PENDING), is(4L));
 797
 798         //return failed to first job
 799         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 800         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 801         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 802             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 803         processJobsCountTimes(30);
 804         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 805             asyncInstantiationBL.getAllServicesInfo()
 806                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 807         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 808         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 809         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 810         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 811
 812         //return completed to all other jobs
 813         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 814             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 815         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 816             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 817         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 818             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 819
 820         processJobsCountTimes(30);
 821         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 822         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 823         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 824         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 825         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 826         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 827     }
 828
 829     protected Map<JobStatus, Long> getJobStatusesCount() {
 830         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 831     }
 832
 833     @Test
 834     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 835
 836         /*
 837         new feature: skip service (existing impl) and skip group (new impl)
 838         service+group aren't touched, 2 groups ARE created
 839
 840         [v]  success if all GROUPs success
 841
 842         Next test variation should:
 843         [ ]  error if all GROUPs error
 844         [ ]  completed with error if 1 GROUP error
 845
 846
 847         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 848         [v]    verify STATUS pending
 849         [v]  + pull+execute  (should NOT post to MSO)
 850         [v]    verify STATUS in progress; TYPE watching
 851                ...
 852
 853         [v]  + pull+execute job#1
 854         [v]    verify job#1 STATUS in progress; TYPE watching
 855
 856         [v]  + pull+execute job#6 (should post to MSO)
 857         [v]    verify job#6 STATUS resource in progress
 858         [v]  + pull+execute job#1
 859         [v]    verify job#1 STATUS in progress; TYPE watching
 860         [v]  + pull+execute job#6 (should get from MSO)
 861         [v]    verify job#6 STATUS completed
 862         [v]  + pull+execute job#1
 863         [v]    verify job#1 STATUS in progress; TYPE watching
 864
 865         [v]  + pull+execute job#7 (should post to MSO)
 866         [v]    verify job#7 STATUS resource in progress
 867         [v]  + pull+execute job#1
 868         [v]    verify job#1 STATUS in progress; TYPE watching
 869         [v]  + pull+execute job#7 (should get from MSO)
 870         [v]    verify job#7 STATUS completed
 871         [v]  + pull+execute job#1
 872         [v]    verify job#1 STATUS completed
 873
 874          */
 875
 876         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 877         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 878         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 879         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 880
 881         // Utility method
 882         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 883             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 884         };
 885
 886         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 887         UUID uuid = pushALaCarteUpdateWithGroups();
 888         singleServicesAndAssertStatus(PENDING, uuid);
 889
 890         // take from pending, put in-progress -> 3 delete-child were born
 891         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 892         verifyQueueSizes(ImmutableMap.of(
 893             IN_PROGRESS, 1
 894         ));
 895
 896         // take job #1 from phase delete to phase create -> 3 create-child were born
 897         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 898         verifyQueueSizes(ImmutableMap.of(
 899             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 900         ));
 901
 902         // prepare MSO mock
 903         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 904             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 905             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 906             .thenReturn(null);
 907         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 908             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 909
 910         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 911         // note there's no concrete mechanism that makes the first child be
 912         // the "action=None" case, but that's what happens, and following lines
 913         // relies on that fact.
 914         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 915
 916         // take each of next two children from creating to in-progress, then to completed
 917         // verify job #1 is watching, and MSO is getting requests
 918         Stream.of(1, 2).forEach(i -> {
 919             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 920             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 921             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 922
 923             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 924             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 925             verify(restMso, times(i)).GetForObject(any(), any());
 926         });
 927
 928         // job #1 is done as all children are done
 929         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 930         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 931     }
 932
 933     @DataProvider
 934     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 935         return new Object[][]{
 936             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 937             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 938                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 939             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 940                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 941             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 942                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 943             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 944                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 945             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 946                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 947             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 948                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 949         };
 950     }
 951
 952     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 953     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 954     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 955     public void vnfGropingIntegrationTest(
 956         String desc,
 957         RestObject<RequestReferencesContainer> createGroupResponse,
 958         RestObject<RequestReferencesContainer> deleteGroupResponse,
 959         RestObject<AsyncRequestStatus> createStatusResponse,
 960         RestObject<AsyncRequestStatus> deleteStatusResponse,
 961         JobStatus expectedJobStatus,
 962         int getStatusCounter) {
 963
 964         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 965             "/serviceInstantiation/v7/instanceGroups",
 966             createGroupResponse,
 967             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 968             deleteGroupResponse,
 969             createStatusResponse,
 970             deleteStatusResponse,
 971             expectedJobStatus,
 972             getStatusCounter);
 973
 974         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 975         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 976         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 977
 978         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 979             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 980             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 981         }
 982
 983         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 984             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 985             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 986         }
 987     }
 988
 989     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 990     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 991     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 992     public void vnfsIntegrationTest(
 993         String desc,
 994         RestObject<RequestReferencesContainer> createVnfResponse,
 995         RestObject<RequestReferencesContainer> deleteVnfResponse,
 996         RestObject<AsyncRequestStatus> createStatusResponse,
 997         RestObject<AsyncRequestStatus> deleteStatusResponse,
 998         JobStatus expectedJobStatus,
 999         int getStatusCounter) {
1000
1001         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
1002             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
1003             createVnfResponse,
1004             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
1005             deleteVnfResponse,
1006             createStatusResponse,
1007             deleteStatusResponse,
1008             expectedJobStatus,
1009             getStatusCounter);
1010     }
1011
1012     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1013     public void vfModulesIntegrationTest(
1014         String desc,
1015         RestObject<RequestReferencesContainer> createVfModuleResponse,
1016         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
1017         RestObject<AsyncRequestStatus> createStatusResponse,
1018         RestObject<AsyncRequestStatus> deleteStatusResponse,
1019         JobStatus expectedJobStatus,
1020         int getStatusCounter) throws IOException, AsdcCatalogException {
1021
1022         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1023         reset(commandUtils);
1024         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "f8360508-3f17-4414-a2ed-6bc71161e8db")).thenReturn(true);
1025         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "25284168-24bb-4698-8cb4-3f509146eca5")).thenReturn(false);
1026
1027         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
1028             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
1029             createVfModuleResponse,
1030             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
1031             deleteVfModuleResponse,
1032             createStatusResponse,
1033             deleteStatusResponse,
1034             expectedJobStatus,
1035             getStatusCounter);
1036     }
1037
1038     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
1039     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
1040     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1041     public void networksIntegrationTest(
1042         String desc,
1043         RestObject<RequestReferencesContainer> createNetworkResponse,
1044         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1045         RestObject<AsyncRequestStatus> createStatusResponse,
1046         RestObject<AsyncRequestStatus> deleteStatusResponse,
1047         JobStatus expectedJobStatus,
1048         int getStatusCounter) {
1049
1050         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1051             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1052             createNetworkResponse,
1053             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1054             deleteNetworkResponse,
1055             createStatusResponse,
1056             deleteStatusResponse,
1057             expectedJobStatus,
1058             getStatusCounter);
1059     }
1060
1061     private UUID createAndDeleteIntegrationTest(String payload,
1062         String createPath,
1063         RestObject<RequestReferencesContainer> createResponse,
1064         String deletePath,
1065         RestObject<RequestReferencesContainer> deleteResponse,
1066         RestObject<AsyncRequestStatus> createStatusResponse,
1067         RestObject<AsyncRequestStatus> deleteStatusResponse,
1068         JobStatus expectedJobStatus,
1069         int getStatusCounter) {
1070         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1071             readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1072             .get(0);
1073
1074         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1075         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1076         if  (createStatusResponse!=null) {
1077             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1078         }
1079         if  (deleteStatusResponse!=null) {
1080             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1081         }
1082
1083         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1084
1085         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1086         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1087         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1088
1089         return jobUUID;
1090     }
1091
1092     @Test
1093     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() {
1094         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1095         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1096
1097         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1098             .thenReturn(createResponse);
1099         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1100             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1101         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1102
1103         JsonNode expectedJson = readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1104         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1105         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1106         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1107     }
1108
1109     private void assertErrorForResource(BaseResource resource,
1110         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1111         RestObject<AsyncRequestStatus> statusResponse) {
1112         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1113         assertThat(auditStatus, is(notNullValue()));
1114         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1115         if (statusResponse == null) {
1116             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1117             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1118             assertThat(auditStatus.getRequestId(), is(nullValue()));
1119         } else {
1120             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1121         }
1122     }
1123
1124     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1125         processJobsCountTimes(times);
1126         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1127     }
1128
1129     private void processJobsCountTimes(int times) {
1130         for (int i = 0; i < times; i++) {
1131             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1132         }
1133     }
1134
1135     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1136         processJobsCountTimes(times);
1137         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1138     }
1139
1140
1141     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1142         final Collection<Job> peek = jobsBrokerService.peek();
1143         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1144         assertThat(jobTypes, jsonEquals(expected));
1145     }
1146
1147     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1148         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1149         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1150
1151         return serviceInfoList;
1152     }
1153
1154     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1155         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1156         assertEquals(1, serviceInfoList.size());
1157         ServiceInfo serviceInfo = serviceInfoList.get(0);
1158         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1159         assertThat(serviceInfo.getJobId(), is(jobUUID));
1160         return serviceInfo;
1161     }
1162
1163     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1164         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1165         assertEquals(1, serviceInfoList.size());
1166         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1167     }
1168
1169     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1170         serviceInfoList.forEach(si->{
1171             if (si.getJobId().equals(jobUUID)) {
1172                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1173             }
1174             else {
1175                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1176             }
1177         });
1178     }
1179
1180     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1181         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1182             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1183         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1184     }
1185
1186     private String randomUuid() {
1187         return UUID.randomUUID().toString();
1188     }
1189
1190     @Test
1191     public void whenResumeService_thanExpectedResumeRequestSent() {
1192         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1193         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1194         String resumeRequestId = randomUuid();
1195         String userId = TestUtils.generateRandomAlphaNumeric(6);
1196
1197         //prepare mocks for get all requests for instance id
1198         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1199         when(restMso.GetForObject(
1200             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1201             eq(AsyncRequestStatusList.class)))
1202             .thenReturn(getRequestByIdResponse);
1203
1204         //prepare mocks resume request
1205         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1206             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1207
1208         //prepare mocks for get resume status
1209         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1210             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1211                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1212                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1213
1214
1215         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1216         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1217         verify(restMso).GetForObject(
1218             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1219             eq(AsyncRequestStatusList.class));
1220         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1221         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1222     }
1223
1224     @Test
1225     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() {
1226         //there is no mocks for restMSO which means restMso return bad response...
1227         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1228         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1229     }
1230
1231     @NotNull
1232     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() {
1233         AsyncRequestStatusList asyncRequestStatusList = readJsonResourceFileAsObject(
1234             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1235             AsyncRequestStatusList.class);
1236         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1237         getRequestByIdResponse.set(asyncRequestStatusList);
1238         getRequestByIdResponse.setStatusCode(200);
1239         return getRequestByIdResponse;
1240     }
1241
1242     private ServiceInstantiation generateResumeMacroPayload() {
1243         return readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1244     }
1245
1246     @Test
1247     public void whenUpgradingVfModule_thenExpectedReplaceRequestSent() throws AsdcCatalogException {
1248         String currentServiceInstanceId = "6196ab1f-2349-4b32-9b6c-cffeb0ccc79c";
1249         String currentVnfInstanceId = "d520268f-7489-4662-be59-f81495b3a069";
1250         String currentVfModuleInstanceId = "b0732bed-3ddf-43cc-b193-7f18db84e476";
1251
1252         assertTestPayloadFitsExpectedIds(upgradeVfModulePayload(), currentServiceInstanceId, currentVnfInstanceId, currentVfModuleInstanceId);
1253
1254         String replaceRequestId = randomUuid();
1255         String userId = "az2016";
1256
1257         String modelInvariantId = "b3a1a119-dede-4ed0-b077-2a617fa519a3";
1258         String newestModelUuid = "d9a5b318-187e-476d-97f7-a15687a927a9";
1259
1260         String expectedMsoReplacePath = "/serviceInstantiation/v7/serviceInstances/"
1261             + currentServiceInstanceId + "/vnfs/" + currentVnfInstanceId + "/vfModules/" + currentVfModuleInstanceId + "/replace";
1262
1263         when(commandUtils.getNewestModelUuid(eq(modelInvariantId))).thenReturn(newestModelUuid);
1264         when(commandUtils.getServiceModel(eq(newestModelUuid))).thenReturn(newestServiceModel());
1265
1266         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(expectedMsoReplacePath), eq(Optional.of(userId))))
1267             .thenReturn(createResponse(202, currentVfModuleInstanceId, replaceRequestId));
1268
1269         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1270             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1271                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1272                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1273
1274         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1275         enableAddCloudOwnerOnMsoRequest();
1276
1277
1278         UUID jobUUID = asyncInstantiationBL.pushBulkJob(upgradeVfModulePayload(), userId).get(0);
1279         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1280
1281
1282         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1283         verify(restMso, times(1)).restCall(
1284             eq(HttpMethod.POST),
1285             eq(RequestReferencesContainer.class),
1286             requestCaptor.capture(),
1287             eq(expectedMsoReplacePath),
1288             eq(Optional.of(userId))
1289         );
1290
1291         JsonNode expectedPayloadToMso = readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__payload_to_mso.json", JsonNode.class);
1292         assertThat(requestCaptor.getValue(), jsonEquals(expectedPayloadToMso).when(IGNORING_ARRAY_ORDER));
1293     }
1294
1295     private void assertTestPayloadFitsExpectedIds(ServiceInstantiation upgradeVfModulePayload, String serviceInstanceId,
1296         String vnfInstanceId, String vfModuleInstanceId) {
1297         /*
1298         Just verifies the test and the input-file are using the same set of instance IDs
1299          */
1300         assertThat(upgradeVfModulePayload, jsonPartEquals("instanceId", serviceInstanceId));
1301         assertThat(upgradeVfModulePayload, jsonNodePresent(
1302             "vnfs"
1303                 + "." + vnfInstanceId
1304                 + ".vfModules"
1305                 + ".xbitestmodulereplace0\\.\\.XbiTestModuleReplace\\.\\.base_ocg\\.\\.module-0"
1306                 + "." + vfModuleInstanceId));
1307     }
1308
1309     private ServiceModel newestServiceModel() {
1310         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__target_newest_service_model.json", ServiceModel.class);
1311     }
1312
1313     private ServiceInstantiation upgradeVfModulePayload() {
1314         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__fe_input_cypress.json", ServiceInstantiation.class);
1315     }
1316
1317     @Test
1318     public void deployService_failIt_retryDeploy_getRetryAsTemplate_makeSureFalsyIsFailedInTemplate() {
1319
1320         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
1321         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
1322         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
1323
1324         //push alacarte with 1 vnf, verify STATUS pending
1325         UUID uuid = pushALaCarteWithVnf();
1326         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
1327
1328         reset(restMso);
1329         //mock mso to answer 200 of create service instance request, verify STATUS in progress
1330         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
1331             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
1332
1333         //mock mso to answer FAILED for service instance create
1334         final RestObject<AsyncRequestStatus> failedResponse = asyncRequestStatusResponseAsRestObject(FAILED_STR);
1335         final String failureDescription = "Some deep failure";
1336         failedResponse.get().request.requestStatus.setStatusMessage(failureDescription);
1337         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
1338             thenReturn(failedResponse);
1339
1340         //Wait till job failed
1341         processJobsCountTimesAndAssertStatus(uuid, 3, FAILED);
1342
1343         //make sure retry request jas isFailed = true, and status message is with failureDescription
1344         ServiceInstantiation retryRequest = asyncInstantiationBL.getBulkForRetry(uuid);
1345         assertTrue(retryRequest.getIsFailed());
1346         assertEquals(failureDescription, retryRequest.getStatusMessage());
1347
1348         //deploy retry job and it's template
1349         UUID retryUuid = asyncInstantiationBL.pushBulkJob(retryRequest, USER_ID).get(0);
1350         ServiceInstantiation templateOfRetry = instantiationTemplates.getJobRequestAsTemplate(retryUuid);
1351
1352         //make sure the template request has isFailed = false, and no status message
1353         assertFalse(templateOfRetry.getIsFailed());
1354         assertNull(templateOfRetry.getStatusMessage());
1355     }
1356
1357 }