vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import static java.util.stream.Collectors.counting;
  24 import static java.util.stream.Collectors.groupingBy;
  25 import static java.util.stream.Collectors.joining;
  26 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  27 import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals;
  28 import static net.javacrumbs.jsonunit.JsonMatchers.jsonNodePresent;
  29 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  30 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartMatches;
  31 import static net.javacrumbs.jsonunit.core.Option.IGNORING_ARRAY_ORDER;
  32 import static org.hamcrest.CoreMatchers.equalTo;
  33 import static org.hamcrest.CoreMatchers.is;
  34 import static org.hamcrest.CoreMatchers.notNullValue;
  35 import static org.hamcrest.CoreMatchers.nullValue;
  36 import static org.hamcrest.MatcherAssert.assertThat;
  37 import static org.hamcrest.Matchers.hasProperty;
  38 import static org.hamcrest.Matchers.hasSize;
  39 import static org.hamcrest.core.Every.everyItem;
  40 import static org.mockito.ArgumentMatchers.any;
  41 import static org.mockito.ArgumentMatchers.argThat;
  42 import static org.mockito.ArgumentMatchers.endsWith;
  43 import static org.mockito.ArgumentMatchers.eq;
  44 import static org.mockito.ArgumentMatchers.isNull;
  45 import static org.mockito.Mockito.reset;
  46 import static org.mockito.Mockito.times;
  47 import static org.mockito.Mockito.verify;
  48 import static org.mockito.Mockito.when;
  49 import static org.onap.vid.job.Job.JobStatus.COMPLETED;
  50 import static org.onap.vid.job.Job.JobStatus.COMPLETED_AND_PAUSED;
  51 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_ERRORS;
  52 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_NO_ACTION;
  53 import static org.onap.vid.job.Job.JobStatus.FAILED;
  54 import static org.onap.vid.job.Job.JobStatus.IN_PROGRESS;
  55 import static org.onap.vid.job.Job.JobStatus.PAUSE;
  56 import static org.onap.vid.job.Job.JobStatus.PENDING;
  57 import static org.onap.vid.job.Job.JobStatus.PENDING_RESOURCE;
  58 import static org.onap.vid.job.Job.JobStatus.RESOURCE_IN_PROGRESS;
  59 import static org.onap.vid.job.Job.JobStatus.STOPPED;
  60 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  61 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  62 import static org.onap.vid.testUtils.TestUtils.readJsonResourceFileAsObject;
  63 import static org.testng.Assert.assertNull;
  64 import static org.testng.AssertJUnit.assertEquals;
  65 import static org.testng.AssertJUnit.assertFalse;
  66 import static org.testng.AssertJUnit.assertTrue;
  67
  68 import com.fasterxml.jackson.databind.JsonNode;
  69 import com.google.common.collect.ImmutableList;
  70 import com.google.common.collect.ImmutableMap;
  71 import java.lang.reflect.Method;
  72 import java.util.Collection;
  73 import java.util.Collections;
  74 import java.util.List;
  75 import java.util.Map;
  76 import java.util.Optional;
  77 import java.util.Stack;
  78 import java.util.UUID;
  79 import java.util.function.BiConsumer;
  80 import java.util.function.Supplier;
  81 import java.util.stream.Collectors;
  82 import java.util.stream.IntStream;
  83 import java.util.stream.Stream;
  84 import javax.inject.Inject;
  85 import javax.ws.rs.ProcessingException;
  86 import org.apache.commons.lang3.StringUtils;
  87 import org.apache.commons.lang3.mutable.MutableInt;
  88 import org.jetbrains.annotations.NotNull;
  89 import org.mockito.ArgumentCaptor;
  90 import org.mockito.Mockito;
  91 import org.mockito.hamcrest.MockitoHamcrest;
  92 import org.onap.portalsdk.core.service.DataAccessService;
  93 import org.onap.portalsdk.core.util.SystemProperties;
  94 import org.onap.vid.asdc.AsdcCatalogException;
  95 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  96 import org.onap.vid.config.DataSourceConfig;
  97 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  98 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  99 import org.onap.vid.job.Job;
 100 import org.onap.vid.job.Job.JobStatus;
 101 import org.onap.vid.job.JobType;
 102 import org.onap.vid.job.JobsBrokerService;
 103 import org.onap.vid.job.command.CommandUtils;
 104 import org.onap.vid.job.command.InternalState;
 105 import org.onap.vid.model.Action;
 106 import org.onap.vid.model.JobAuditStatus;
 107 import org.onap.vid.model.NameCounter;
 108 import org.onap.vid.model.RequestReferencesContainer;
 109 import org.onap.vid.model.ServiceInfo;
 110 import org.onap.vid.model.ServiceModel;
 111 import org.onap.vid.model.serviceInstantiation.BaseResource;
 112 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
 113 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
 114 import org.onap.vid.mso.RestMsoImplementation;
 115 import org.onap.vid.mso.RestObject;
 116 import org.onap.vid.mso.model.RequestReferences;
 117 import org.onap.vid.mso.rest.AsyncRequestStatus;
 118 import org.onap.vid.mso.rest.AsyncRequestStatusList;
 119 import org.onap.vid.properties.Features;
 120 import org.onap.vid.services.AsyncInstantiationBaseTest;
 121 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
 122 import org.onap.vid.services.AuditService;
 123 import org.onap.vid.services.InstantiationTemplatesService;
 124 import org.onap.vid.services.VersionService;
 125 import org.onap.vid.testUtils.TestUtils;
 126 import org.onap.vid.utils.DaoUtils;
 127 import org.springframework.http.HttpMethod;
 128 import org.springframework.test.context.ContextConfiguration;
 129 import org.testng.annotations.BeforeClass;
 130 import org.testng.annotations.BeforeMethod;
 131 import org.testng.annotations.DataProvider;
 132 import org.testng.annotations.Test;
 133 import org.togglz.core.manager.FeatureManager;
 134
 135 //it's more like integration test than UT
 136 //But it's very hard to test in API test so I use UT
 137 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 138 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 139
 140     private static final String FAILED_STR = "FAILED";
 141     private static final String COMPLETE_STR = "COMPLETE";
 142     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 143     private static final String REQUESTED = "REQUESTED";
 144     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 145     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 146     private static String USER_ID =  "123";
 147     public static String REQUEST_ID = UUID.randomUUID().toString();
 148     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 149
 150     @Inject
 151     private VersionService versionService;
 152
 153     @Inject
 154     private JobsBrokerService jobsBrokerService;
 155
 156     @Inject
 157     private JobWorker jobWorker;
 158
 159     @Inject
 160     private FeatureManager featureManager;
 161
 162     @Inject
 163     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 164
 165     @Inject
 166     private AuditService auditService;
 167
 168     @Inject
 169     private RestMsoImplementation restMso;
 170
 171     @Inject
 172     private DataAccessService dataAccessService;
 173
 174     @Inject
 175     private CommandUtils commandUtils;
 176
 177     @Inject
 178     private InstantiationTemplatesService instantiationTemplates;
 179
 180     @BeforeClass
 181     void initServicesInfoService() {
 182         createInstanceParamsMaps();
 183         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 184     }
 185
 186     @BeforeMethod
 187     void clearDb() {
 188         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 189         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 190         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 191     }
 192
 193     @BeforeMethod
 194     void defineMocks() {
 195         Mockito.reset(restMso);
 196         Mockito.reset(aaiClient);
 197         Mockito.reset(commandUtils);
 198         mockAaiClientAnyNameFree();
 199
 200         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 201         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 202         when(featureManager.isActive(Features.FLAG_2006_VFMODULE_TAKES_TENANT_AND_REGION_FROM_VNF)).thenReturn(true);
 203     }
 204
 205     @Test
 206     public void whenPushNewBulk_thenAllServicesAreInPending() {
 207
 208         pushMacroBulk();
 209         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 210         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 211     }
 212
 213     private List<UUID> pushMacroBulk() {
 214         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 215             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 216             3, true,PROJECT_NAME, true);
 217         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 218     }
 219
 220     private UUID pushALaCarteWithVnf() {
 221         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 222         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 223         assertThat(uuids, hasSize(1));
 224         return uuids.get(0);
 225     }
 226
 227     private UUID pushALaCarteUpdateWithGroups() {
 228         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 229         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 230         assertThat(uuids, hasSize(1));
 231         return uuids.get(0);
 232     }
 233
 234     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 235         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 236     }
 237
 238     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 239         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 240     }
 241
 242     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 243         RequestReferences requestReferences = new RequestReferences();
 244         requestReferences.setRequestId(requestId);
 245         requestReferences.setInstanceId(instanceId);
 246         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 247         restObject.set(new RequestReferencesContainer(requestReferences));
 248         restObject.setStatusCode(statusCode);
 249         restObject.setRaw(RAW_DATA_FROM_MSO);
 250         return restObject;
 251     }
 252
 253     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 254         return Stream.of(jobStatuses).map(
 255             Enum::toString).collect(ImmutableList.toImmutableList());
 256     }
 257
 258     /*
 259     Make sure service state is in progress once request has sent to MSO
 260     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 261     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 262     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 263     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 264     */
 265     @Test
 266     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 267
 268         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 269         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 270         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 271         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 272
 273         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 274             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 275
 276         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 277             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 278             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 279             statusesToStrings(PENDING, STOPPED)
 280         );
 281
 282         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 283             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 284             ImmutableList.of(REQUESTED, FAILED_STR),
 285             ImmutableList.of()
 286         );
 287
 288         List<UUID> uuids = pushMacroBulk();
 289         UUID firstJobUuid = uuids.get(0);
 290         UUID secondJobUuid = uuids.get(1);
 291         //assert that when get ProcessingException from restMso, status remain the same
 292         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 293         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 294
 295         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 296         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 297         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 298
 299         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 300         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 301         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 302
 303         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 304         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 305         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 306
 307         //when get job COMPLETE from MSO, service status become COMPLETED
 308         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 309         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 310         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 311
 312
 313         //for use later in the test
 314         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 315             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 316         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 317
 318         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 319         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 320             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 321         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 322         processJobsCountTimes(10);
 323
 324         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 325         listServicesAndAssertStatus(expectedJobStatusMap);
 326
 327
 328         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 329         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 330         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 331         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 332         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 333             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 334         ));
 335
 336         listServicesAndAssertStatus(expectedJobStatusMap);
 337         IntStream.range(0, uuids.size()).forEach(i -> {
 338             UUID uuid = uuids.get(i);
 339             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 340             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 341         });
 342
 343         //assert no more jobs to pull
 344         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 345         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 346     }
 347
 348
 349     @DataProvider
 350     public static Object[][] AlaCarteStatuses(Method test) {
 351         return new Object[][]{
 352             {COMPLETE_STR, JobStatus.COMPLETED},
 353             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 354         };
 355     }
 356
 357     /*
 358     Make sure service state is in progress once request has sent to MSO
 359     Make sure service state is watching until state changes to complemented
 360     Make sure service state is watching until vnf state changes to completed
 361     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 362     status Creating
 363      */
 364     @Test(dataProvider = "AlaCarteStatuses")
 365     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 366         /*
 367             [v]  + push alacarte with 1 vnf
 368             [v]    verify STATUS pending
 369             [v]  + pull+execute  (should post to MSO)
 370             [v]    verify STATUS in progress
 371             [v]  + pull+execute  (should GET completed from MSO)
 372             [v]    verify STATUS in progress; TYPE watching
 373             [v]    verify job#2 *new* VNF job STATUS creating
 374             [v]  + pull+execute job#2 (should post to MSO)
 375             [v]    verify job#2 STATUS resource in progress
 376             [v]    verify job#1 STATUS in progress
 377             [v]  + pull+execute job#2 (should GET completed from MSO)
 378             [v]    verify job#2 STATUS completed
 379             [v]  + pull+execute job#1
 380             [v]    verify job#1 STATUS completed
 381
 382            * not looking on audit (yet)
 383         */
 384         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false); // this makes the test pass without mocking the vfModules
 385         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 386         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 387         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 388
 389
 390         //push alacarte with 1 vnf, verify STATUS pending
 391         UUID uuid = pushALaCarteWithVnf();
 392         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 393         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 394         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 395             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 396         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 397         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 398             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 399         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 400         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 401             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 402         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 403             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 404
 405         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 406         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 407         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 408         verify(restMso, times(2)).GetForObject(any(), any());
 409
 410     }
 411
 412     /*
 413     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 414
 415     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 416     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 417
 418     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 419     And union these tests to single one.
 420      */
 421
 422     @Test
 423     public void testALaCarteLifeCycle1Vnf2VfModules() {
 424
 425
 426         String msoVnfStatus = COMPLETE_STR;
 427         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 428         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 429         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 430         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 431         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 432         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 433         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 434         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 435
 436
 437         //push alacarte with 1 vnf, verify STATUS pending
 438         UUID uuid = pushALaCarteWithVnf();
 439         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 440
 441         /*---------- service -----------*/
 442
 443         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 444         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 445             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 446
 447         //mock mso to answer COMPLETE for service instance create
 448         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 449             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 450
 451         /*---------- vnf -----------*/
 452
 453         //mock mso to answer 200 of create vnf instance request
 454         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 455             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 456
 457         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 458         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 459             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 460
 461         reset(commandUtils);
 462         when(commandUtils.isVfModuleBaseModule(eq(SERVICE_MODEL_VERSION_ID),
 463             argThat(it -> it.getModelCustomizationId().equals(VF_MODULE_0_MODEL_CUSTOMIZATION_NAME)))).thenReturn(true);
 464         when(commandUtils.isVfModuleBaseModule(eq(SERVICE_MODEL_VERSION_ID),
 465             argThat(it -> it.getModelCustomizationId().equals(VF_MODULE_1_MODEL_CUSTOMIZATION_NAME)))).thenReturn(false);
 466
 467         /*---------- vf Module without volume group name (base) -----------*/
 468
 469         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 470         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 471             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 472         //mock mso to answer for vf module orchestration request
 473         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 474             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 475
 476         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 477             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 478         //mock mso to answer for volume group orchestration request
 479         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 480             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 481
 482         /*---------- vfModule -----------*/
 483
 484         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 485         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 486             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 487
 488         //mock mso to answer for vf module orchestration request
 489         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 490             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 491
 492         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 493         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 494         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 495         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 496         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 497         verify(restMso, times(5)).GetForObject(any(), any());
 498     }
 499
 500     @Test
 501     public void testALaCarteLifeCycle2Networks() {
 502
 503         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 504         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 505         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 506         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 507
 508         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 509         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 510         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 511         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 512         //TODO use them later for different networks
 513         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 514         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 515
 516
 517         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 518         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 519
 520         /*---------- service -----------*/
 521
 522         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 523         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 524             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 525
 526         //mock mso to answer COMPLETE for service instance create
 527         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 528             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 529
 530         final MutableInt secondNetworkCode = new MutableInt(500);
 531         final MutableInt inProgressCount = new MutableInt(0);
 532
 533         /*---------- network 1-----------*/
 534
 535         //mock mso to answer 200 of first create network instance request
 536         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 537             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 538             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 539             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 540
 541         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 542         //Once COMPLETE, second network creation will return 200
 543         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 544             thenAnswer(x->{
 545                 String status;
 546                 if (inProgressCount.getValue()<10) {
 547                     status = IN_PROGRESS_STR;
 548                 } else {
 549                     secondNetworkCode.setValue(200);
 550                     status = COMPLETE_STR;
 551                 }
 552                 inProgressCount.add(1);
 553                 return asyncRequestStatusResponseAsRestObject(status);
 554             });
 555
 556         /*---------- network 2-----------*/
 557
 558         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 559         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 560             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 561             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 562             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 563
 564 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 565 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 566         //mock mso to answer COMPLETE for network creation status,
 567
 568         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 569             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 570
 571
 572         /*---------- Create request and process it -----------*/
 573         //push alacarte with 2 networks, verify STATUS pending
 574         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 575         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 576         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 577         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 578
 579         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 580
 581         //validate the mso request id is the right one
 582         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 583         ServiceInfo serviceInfo = serviceInfoList.get(0);
 584         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 585
 586         /*---------- verify -----------*/
 587         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 588         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 589         //get status
 590         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 591         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 592         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 593     }
 594
 595     @Test
 596     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 597         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 598         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 599         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 600             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 601
 602         List<UUID> uuids = pushMacroBulk();
 603         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 604
 605         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 606         Mockito.reset(aaiClient); // must forget the "thenThrow"
 607         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 608         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 609
 610     }
 611
 612     @Test
 613     public void testAaiResponseNameUsedTillMaxRetries() {
 614         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 615         //simulate MSO to return good result, for making sure we failed because of AAI error
 616         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 617         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 618             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 619
 620         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 621         List<UUID> uuids = pushMacroBulk();
 622         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 623     }
 624
 625     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 626         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 627     }
 628
 629     //return the pulled job (and not the pushed job)
 630     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 631         Optional<Job> job = pullJob(topic, pullingAssertion);
 632
 633         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 634
 635         try {
 636             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 637
 638             if (pullingAssertion) {
 639                 //assert another pulling on same topic return no result (before push back)
 640                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 641             }
 642
 643         } finally {
 644             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 645         }
 646         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 647         return job.get();
 648     }
 649
 650     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 651         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 652         optionalJob.ifPresent(job->{
 653             Job nextJob = jobWorker.executeJobAndGetNext(job);
 654             jobsBrokerService.pushBack(nextJob);
 655         });
 656     }
 657
 658     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 659         JobType expectedNextJobType) {
 660         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 661         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 662         return job;
 663     }
 664
 665     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 666         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 667         return retryWithAssertionsLimit(retries, () -> {
 668             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 669             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 670             if (internalState != null) {
 671                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 672             }
 673             return job;
 674         });
 675     }
 676
 677     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 678         java.util.Stack<AssertionError> history = new Stack<>();
 679
 680         do {
 681             try {
 682                 return supplier.get();
 683             } catch (AssertionError assertionError) {
 684                 history.push(assertionError);
 685             }
 686         } while (history.size() < retries);
 687
 688         // No success:
 689         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 690             .map(Throwable::getMessage)
 691             .map(s -> s.replace('\n', ' '))
 692             .map(s -> s.replaceAll("\\s{2,}"," "))
 693             .distinct()
 694             .collect(joining("\n   ", "\n   ", "")), history.peek());
 695     }
 696
 697     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 698         if (pullingAssertion) {
 699             //assert pulling on inverse topic return no result
 700             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 701         }
 702
 703         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 704         assertTrue("no job fetched", job.isPresent());
 705
 706         if (pullingAssertion) {
 707             //assert another pulling on same topic return no result
 708             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 709         }
 710
 711         return job;
 712     }
 713
 714     private JobStatus inverseTopic(JobStatus topic) {
 715         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 716     }
 717
 718
 719     @Test
 720     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 721         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 722         //assert that when get ProcessingException from restMso, status remain the same
 723         List<UUID> uuids = pushMacroBulk();
 724         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 725     }
 726
 727     @Test
 728     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 729         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 730         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 731             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 732
 733         //assert that when get ProcessingException from restMso, status remain the same
 734         List<UUID> uuids = pushMacroBulk();
 735         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 736
 737
 738         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 739         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 740             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 741         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 742
 743         //the job get COMPLETE response this job is copmpleted and then also other jobs
 744         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 745             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 746         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 747
 748         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 749         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 750         assertThat(vidStatuses, is(expectedStatusesForVid));
 751     }
 752
 753     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 754         pushMacroBulk();
 755         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 756     }
 757
 758     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 759         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 760         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 761         return job;
 762     }
 763
 764     @Test
 765     public void test2BulksLifeCyclesAreIndependent() {
 766
 767         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 768         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 769         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 770         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 771         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 772         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 773
 774
 775         //create first bulk and make one job in progress
 776         List<UUID> firstBulksIDs = pushMacroBulk();
 777         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 778             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 779         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 780             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 781         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 782
 783         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 784         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 785             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 786         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 787             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 788         List<UUID> secondBulksIDs = pushMacroBulk();
 789         processJobsCountTimes(30);
 790         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 791         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 792         assertThat(statusCount.get(PENDING), is(4L));
 793
 794         //return failed to first job
 795         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 796         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 797         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 798             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 799         processJobsCountTimes(30);
 800         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 801             asyncInstantiationBL.getAllServicesInfo()
 802                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 803         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 804         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 805         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 806         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 807
 808         //return completed to all other jobs
 809         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 810             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 811         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 812             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 813         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 814             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 815
 816         processJobsCountTimes(30);
 817         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 818         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 819         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 820         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 821         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 822         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 823     }
 824
 825     protected Map<JobStatus, Long> getJobStatusesCount() {
 826         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 827     }
 828
 829     @Test
 830     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 831
 832         /*
 833         new feature: skip service (existing impl) and skip group (new impl)
 834         service+group aren't touched, 2 groups ARE created
 835
 836         [v]  success if all GROUPs success
 837
 838         Next test variation should:
 839         [ ]  error if all GROUPs error
 840         [ ]  completed with error if 1 GROUP error
 841
 842
 843         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 844         [v]    verify STATUS pending
 845         [v]  + pull+execute  (should NOT post to MSO)
 846         [v]    verify STATUS in progress; TYPE watching
 847                ...
 848
 849         [v]  + pull+execute job#1
 850         [v]    verify job#1 STATUS in progress; TYPE watching
 851
 852         [v]  + pull+execute job#6 (should post to MSO)
 853         [v]    verify job#6 STATUS resource in progress
 854         [v]  + pull+execute job#1
 855         [v]    verify job#1 STATUS in progress; TYPE watching
 856         [v]  + pull+execute job#6 (should get from MSO)
 857         [v]    verify job#6 STATUS completed
 858         [v]  + pull+execute job#1
 859         [v]    verify job#1 STATUS in progress; TYPE watching
 860
 861         [v]  + pull+execute job#7 (should post to MSO)
 862         [v]    verify job#7 STATUS resource in progress
 863         [v]  + pull+execute job#1
 864         [v]    verify job#1 STATUS in progress; TYPE watching
 865         [v]  + pull+execute job#7 (should get from MSO)
 866         [v]    verify job#7 STATUS completed
 867         [v]  + pull+execute job#1
 868         [v]    verify job#1 STATUS completed
 869
 870          */
 871
 872         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 873         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 874         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 875         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 876
 877         // Utility method
 878         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 879             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 880         };
 881
 882         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 883         UUID uuid = pushALaCarteUpdateWithGroups();
 884         singleServicesAndAssertStatus(PENDING, uuid);
 885
 886         // take from pending, put in-progress -> 3 delete-child were born
 887         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 888         verifyQueueSizes(ImmutableMap.of(
 889             IN_PROGRESS, 1
 890         ));
 891
 892         // take job #1 from phase delete to phase create -> 3 create-child were born
 893         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 894         verifyQueueSizes(ImmutableMap.of(
 895             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 896         ));
 897
 898         // prepare MSO mock
 899         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 900             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 901             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 902             .thenReturn(null);
 903         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 904             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 905
 906         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 907         // note there's no concrete mechanism that makes the first child be
 908         // the "action=None" case, but that's what happens, and following lines
 909         // relies on that fact.
 910         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 911
 912         // take each of next two children from creating to in-progress, then to completed
 913         // verify job #1 is watching, and MSO is getting requests
 914         Stream.of(1, 2).forEach(i -> {
 915             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 916             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 917             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 918
 919             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 920             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 921             verify(restMso, times(i)).GetForObject(any(), any());
 922         });
 923
 924         // job #1 is done as all children are done
 925         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 926         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 927     }
 928
 929     @DataProvider
 930     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 931         return new Object[][]{
 932             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 933             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 934                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 935             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 936                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 937             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 938                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 939             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 940                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 941             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 942                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 943             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 944                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 945         };
 946     }
 947
 948     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 949     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 950     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 951     public void vnfGropingIntegrationTest(
 952         String desc,
 953         RestObject<RequestReferencesContainer> createGroupResponse,
 954         RestObject<RequestReferencesContainer> deleteGroupResponse,
 955         RestObject<AsyncRequestStatus> createStatusResponse,
 956         RestObject<AsyncRequestStatus> deleteStatusResponse,
 957         JobStatus expectedJobStatus,
 958         int getStatusCounter) {
 959
 960         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 961             "/serviceInstantiation/v7/instanceGroups",
 962             createGroupResponse,
 963             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 964             deleteGroupResponse,
 965             createStatusResponse,
 966             deleteStatusResponse,
 967             expectedJobStatus,
 968             getStatusCounter);
 969
 970         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 971         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 972         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 973
 974         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 975             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 976             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 977         }
 978
 979         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 980             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 981             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 982         }
 983     }
 984
 985     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 986     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 987     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 988     public void vnfsIntegrationTest(
 989         String desc,
 990         RestObject<RequestReferencesContainer> createVnfResponse,
 991         RestObject<RequestReferencesContainer> deleteVnfResponse,
 992         RestObject<AsyncRequestStatus> createStatusResponse,
 993         RestObject<AsyncRequestStatus> deleteStatusResponse,
 994         JobStatus expectedJobStatus,
 995         int getStatusCounter) {
 996
 997         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
 998             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
 999             createVnfResponse,
1000             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
1001             deleteVnfResponse,
1002             createStatusResponse,
1003             deleteStatusResponse,
1004             expectedJobStatus,
1005             getStatusCounter);
1006     }
1007
1008     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1009     public void vfModulesIntegrationTest(
1010         String desc,
1011         RestObject<RequestReferencesContainer> createVfModuleResponse,
1012         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
1013         RestObject<AsyncRequestStatus> createStatusResponse,
1014         RestObject<AsyncRequestStatus> deleteStatusResponse,
1015         JobStatus expectedJobStatus,
1016         int getStatusCounter) {
1017
1018         when(commandUtils.isVfModuleBaseModule(eq("6b528779-44a3-4472-bdff-9cd15ec93450"),
1019             argThat(it -> it.getModelCustomizationName().equals("2017488PasqualeVpe..PASQUALE_base_vPE_BV..module-0")))).thenReturn(true);
1020         when(commandUtils.isVfModuleBaseModule(eq("6b528779-44a3-4472-bdff-9cd15ec93450"),
1021             argThat(it -> it.getModelCustomizationName().equals("2017488PasqualeVpe..PASQUALE_vRE_BV..module-1")))).thenReturn(false);
1022
1023         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
1024             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
1025             createVfModuleResponse,
1026             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
1027             deleteVfModuleResponse,
1028             createStatusResponse,
1029             deleteStatusResponse,
1030             expectedJobStatus,
1031             getStatusCounter);
1032     }
1033
1034     @DataProvider
1035     public static Object[][] pauseInstantiation(Method test) {
1036         return new Object[][]{
1037             {true, 2, COMPLETED_AND_PAUSED},
1038             {false, 3, COMPLETED}
1039         };
1040     }
1041
1042     @Test (dataProvider = "pauseInstantiation")
1043     public void viewEdit_existingVnfCreate3VfModulesPauseAfterTheSecond(boolean flag, int expectedNumberOfInvocation,
1044         JobStatus expectedJobStatus) {
1045         RestObject<RequestReferencesContainer> createVfModuleResponse = createResponseRandomIds(202);
1046         RestObject<AsyncRequestStatus> createStatusResponse = asyncRequestStatusResponseAsRestObject(COMPLETE_STR);
1047
1048         when(featureManager.isActive(Features.FLAG_2006_PAUSE_VFMODULE_INSTANTIATION_CREATION)).thenReturn(flag);
1049
1050         when(commandUtils.isVfModuleBaseModule(eq("6b528779-44a3-4472-bdff-9cd15ec93450"),
1051             argThat(it -> it.getModelCustomizationName().equals("2017488PasqualeVpe..PASQUALE_base_vPE_BV..module-0")))).thenReturn(true);
1052
1053         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1054             readJsonResourceFileAsObject("/payload_jsons/vfModuleCreate2AndPauseRequest.json", ServiceInstantiation.class), "userId")
1055             .get(0);
1056
1057         String createPath= "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules";
1058
1059         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createVfModuleResponse);
1060         when(restMso.GetForObject(endsWith(createVfModuleResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1061
1062
1063         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1064
1065         verify(restMso, times(expectedNumberOfInvocation)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1066     }
1067
1068     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
1069     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
1070     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1071     public void networksIntegrationTest(
1072         String desc,
1073         RestObject<RequestReferencesContainer> createNetworkResponse,
1074         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1075         RestObject<AsyncRequestStatus> createStatusResponse,
1076         RestObject<AsyncRequestStatus> deleteStatusResponse,
1077         JobStatus expectedJobStatus,
1078         int getStatusCounter) {
1079
1080         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1081             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1082             createNetworkResponse,
1083             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1084             deleteNetworkResponse,
1085             createStatusResponse,
1086             deleteStatusResponse,
1087             expectedJobStatus,
1088             getStatusCounter);
1089     }
1090
1091     private UUID createAndDeleteIntegrationTest(String payload,
1092         String createPath,
1093         RestObject<RequestReferencesContainer> createResponse,
1094         String deletePath,
1095         RestObject<RequestReferencesContainer> deleteResponse,
1096         RestObject<AsyncRequestStatus> createStatusResponse,
1097         RestObject<AsyncRequestStatus> deleteStatusResponse,
1098         JobStatus expectedJobStatus,
1099         int getStatusCounter) {
1100         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1101             readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1102             .get(0);
1103
1104         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1105         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1106         if  (createStatusResponse!=null) {
1107             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1108         }
1109         if  (deleteStatusResponse!=null) {
1110             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1111         }
1112
1113         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1114
1115         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1116         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1117         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1118
1119         return jobUUID;
1120     }
1121
1122     @Test
1123     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() {
1124         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1125         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1126
1127         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1128             .thenReturn(createResponse);
1129         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1130             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1131         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1132
1133         JsonNode expectedJson = readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1134         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1135         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1136         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1137     }
1138
1139     private void assertErrorForResource(BaseResource resource,
1140         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1141         RestObject<AsyncRequestStatus> statusResponse) {
1142         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1143         assertThat(auditStatus, is(notNullValue()));
1144         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1145         if (statusResponse == null) {
1146             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1147             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1148             assertThat(auditStatus.getRequestId(), is(nullValue()));
1149         } else {
1150             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1151         }
1152     }
1153
1154     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1155         processJobsCountTimes(times);
1156         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1157     }
1158
1159     private void processJobsCountTimes(int times) {
1160         for (int i = 0; i < times; i++) {
1161             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1162         }
1163     }
1164
1165     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1166         processJobsCountTimes(times);
1167         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1168     }
1169
1170
1171     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1172         final Collection<Job> peek = jobsBrokerService.peek();
1173         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1174         assertThat(jobTypes, jsonEquals(expected));
1175     }
1176
1177     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1178         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1179         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1180
1181         return serviceInfoList;
1182     }
1183
1184     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1185         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1186         assertEquals(1, serviceInfoList.size());
1187         ServiceInfo serviceInfo = serviceInfoList.get(0);
1188         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1189         assertThat(serviceInfo.getJobId(), is(jobUUID));
1190         return serviceInfo;
1191     }
1192
1193     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1194         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1195         assertEquals(1, serviceInfoList.size());
1196         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1197     }
1198
1199     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1200         serviceInfoList.forEach(si->{
1201             if (si.getJobId().equals(jobUUID)) {
1202                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1203             }
1204             else {
1205                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1206             }
1207         });
1208     }
1209
1210     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1211         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1212             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1213         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1214     }
1215
1216     private String randomUuid() {
1217         return UUID.randomUUID().toString();
1218     }
1219
1220     @Test
1221     public void whenResumeService_thanExpectedResumeRequestSent() {
1222         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1223         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1224         String resumeRequestId = randomUuid();
1225         String userId = TestUtils.generateRandomAlphaNumeric(6);
1226
1227         //prepare mocks for get all requests for instance id
1228         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1229         when(restMso.GetForObject(
1230             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1231             eq(AsyncRequestStatusList.class)))
1232             .thenReturn(getRequestByIdResponse);
1233
1234         //prepare mocks resume request
1235         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1236             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1237
1238         //prepare mocks for get resume status
1239         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1240             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1241                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1242                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1243
1244
1245         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1246         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1247         verify(restMso).GetForObject(
1248             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1249             eq(AsyncRequestStatusList.class));
1250         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1251         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1252     }
1253
1254     @Test
1255     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() {
1256         //there is no mocks for restMSO which means restMso return bad response...
1257         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1258         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1259     }
1260
1261     @NotNull
1262     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() {
1263         AsyncRequestStatusList asyncRequestStatusList = readJsonResourceFileAsObject(
1264             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1265             AsyncRequestStatusList.class);
1266         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1267         getRequestByIdResponse.set(asyncRequestStatusList);
1268         getRequestByIdResponse.setStatusCode(200);
1269         return getRequestByIdResponse;
1270     }
1271
1272     private ServiceInstantiation generateResumeMacroPayload() {
1273         return readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1274     }
1275
1276     @Test
1277     public void whenUpgradingVfModule_thenExpectedReplaceRequestSent() throws AsdcCatalogException {
1278         String currentServiceInstanceId = "6196ab1f-2349-4b32-9b6c-cffeb0ccc79c";
1279         String currentVnfInstanceId = "d520268f-7489-4662-be59-f81495b3a069";
1280         String currentVfModuleInstanceId = "b0732bed-3ddf-43cc-b193-7f18db84e476";
1281
1282         assertTestPayloadFitsExpectedIds(upgradeVfModulePayload(), currentServiceInstanceId, currentVnfInstanceId, currentVfModuleInstanceId);
1283
1284         String replaceRequestId = randomUuid();
1285         String userId = "az2016";
1286
1287         String modelInvariantId = "b3a1a119-dede-4ed0-b077-2a617fa519a3";
1288         String newestModelUuid = "d9a5b318-187e-476d-97f7-a15687a927a9";
1289
1290         String expectedMsoReplacePath = "/serviceInstantiation/v7/serviceInstances/"
1291             + currentServiceInstanceId + "/vnfs/" + currentVnfInstanceId + "/vfModules/" + currentVfModuleInstanceId + "/replace";
1292
1293         when(commandUtils.getNewestModelUuid(eq(modelInvariantId))).thenReturn(newestModelUuid);
1294         when(commandUtils.getServiceModel(eq(newestModelUuid))).thenReturn(newestServiceModel());
1295
1296         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(expectedMsoReplacePath), eq(Optional.of(userId))))
1297             .thenReturn(createResponse(202, currentVfModuleInstanceId, replaceRequestId));
1298
1299         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1300             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1301                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1302                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1303
1304         enableAddCloudOwnerOnMsoRequest();
1305
1306
1307         UUID jobUUID = asyncInstantiationBL.pushBulkJob(upgradeVfModulePayload(), userId).get(0);
1308         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1309
1310
1311         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1312         verify(restMso, times(1)).restCall(
1313             eq(HttpMethod.POST),
1314             eq(RequestReferencesContainer.class),
1315             requestCaptor.capture(),
1316             eq(expectedMsoReplacePath),
1317             eq(Optional.of(userId))
1318         );
1319
1320         JsonNode expectedPayloadToMso = readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__payload_to_mso.json", JsonNode.class);
1321         assertThat(requestCaptor.getValue(), jsonEquals(expectedPayloadToMso).when(IGNORING_ARRAY_ORDER));
1322     }
1323
1324     @Test
1325     public void viewEdit_oneNetworkExistsAddAnotherNetwork() {
1326         String currentServiceInstanceId = "ce2821fc-3b28-4759-9613-1e514d7563c0";
1327         String addNetworkRequestId = randomUuid();
1328         String userId = "az2016";
1329
1330
1331         String expectedMsoAddNetworkPath =
1332             "/serviceInstantiation/v7/serviceInstances/" + currentServiceInstanceId + "/networks";
1333
1334         when(restMso.restCall(eq(HttpMethod.POST),eq(RequestReferencesContainer.class),any(), eq(expectedMsoAddNetworkPath), any()))
1335             .thenReturn(createResponse(202, currentServiceInstanceId, addNetworkRequestId));
1336
1337         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + addNetworkRequestId),eq(AsyncRequestStatus.class)))
1338             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1339
1340         List<UUID> uuids= asyncInstantiationBL.pushBulkJob(addNetworkBulkPayload(), userId);
1341         assertThat(uuids, hasSize(1));
1342         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED);
1343
1344         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(expectedMsoAddNetworkPath),any());
1345         verify(restMso, times(1)).GetForObject(any(),any());
1346     }
1347
1348     @Test
1349     public void whenDeletingVfModule_thenExpectedDeleteRequestSent()
1350     {
1351         String currentServiceInstanceId = "6196ab1f-2349-4b32-9b6c-cffeb0ccc79c";
1352         String currentVnfInstanceId = "d520268f-7489-4662-be59-f81495b3a069";
1353         String currentVfModuleInstanceId = "b0732bed-3ddf-43cc-b193-7f18db84e476";
1354
1355         assertTestPayloadFitsExpectedIds(deleteVfModuleBulkPayload(), currentServiceInstanceId, currentVnfInstanceId, currentVfModuleInstanceId);
1356
1357         String deleteRequestId = randomUuid();
1358         String userId = "az2016";
1359
1360
1361         String expectedMsoDeletePath = "/serviceInstantiation/v7/serviceInstances/"
1362                 + currentServiceInstanceId + "/vnfs/" + currentVnfInstanceId + "/vfModules/" + currentVfModuleInstanceId;
1363
1364         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(expectedMsoDeletePath), eq(Optional.of(userId))))
1365                 .thenReturn(createResponse(202, currentVfModuleInstanceId, deleteRequestId));
1366
1367         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + deleteRequestId), eq(AsyncRequestStatus.class)))
1368                 .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1369                         asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1370                         asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1371
1372         enableAddCloudOwnerOnMsoRequest();
1373
1374
1375         UUID jobUUID = asyncInstantiationBL.pushBulkJob(deleteVfModuleBulkPayload(), userId).get(0);
1376         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1377
1378
1379         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1380         verify(restMso, times(1)).restCall(
1381                 eq(HttpMethod.DELETE),
1382                 eq(RequestReferencesContainer.class),
1383                 requestCaptor.capture(),
1384                 eq(expectedMsoDeletePath),
1385                 eq(Optional.of(userId))
1386         );
1387
1388         assertThat(requestCaptor.getValue(), jsonEquals(getDeleteVfModulePayloadToMso()));
1389     }
1390
1391
1392     private ServiceInstantiation deleteVfModuleBulkPayload() {
1393         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/delete_1_vfmodule_expected_bulk.json", ServiceInstantiation.class);
1394     }
1395
1396     private ServiceInstantiation addNetworkBulkPayload() {
1397         return readJsonResourceFileAsObject("/payload_jsons/Network/one_network_exists_add_another_network_expected_bulk.json", ServiceInstantiation.class);
1398     }
1399
1400     private String getDeleteVfModulePayloadToMso() {
1401         return "{ " +
1402                 "  \"requestDetails\": { " +
1403                 "    \"requestInfo\": { " +
1404                 "      \"source\": \"VID\", " +
1405                 "      \"requestorId\": \"az2016\" " +
1406                 "    }, " +
1407                 "    \"modelInfo\": { " +
1408                 "      \"modelType\": \"vfModule\", " +
1409                 "      \"modelName\": \"XbiTestModuleReplace..base_ocg..module-0\", " +
1410                 "      \"modelVersionId\": \"04b21d26-9780-4956-8329-b22b049329f4\", " +
1411                 "      \"modelVersion\": \"1.0\", " +
1412                 "      \"modelInvariantId\": \"d887658e-2a89-4baf-83e2-b189601a1a7c\", " +
1413                 "      \"modelCustomizationName\": \"XbiTestModuleReplace..base_ocg..module-0\", " +
1414                 "      \"modelCustomizationId\": \"3f1f0fcb-8a88-4612-a794-3912613ed9e8\" " +
1415                 "    }, " +
1416                 "    \"cloudConfiguration\": { " +
1417                 "      \"lcpCloudRegionId\": \"olson5a\", " +
1418                 "      \"cloudOwner\": \"irma-aic\", " +
1419                 "      \"tenantId\": \"7ff7b1a4fe954f71ab79d3160ec3eb08\" " +
1420                 "    } " +
1421                 "  } " +
1422                 "}";
1423     }
1424
1425     private void assertTestPayloadFitsExpectedIds(ServiceInstantiation upgradeVfModulePayload, String serviceInstanceId,
1426         String vnfInstanceId, String vfModuleInstanceId) {
1427         /*
1428         Just verifies the test and the input-file are using the same set of instance IDs
1429          */
1430         assertThat(upgradeVfModulePayload, jsonPartEquals("instanceId", serviceInstanceId));
1431         assertThat(upgradeVfModulePayload, jsonNodePresent(
1432             "vnfs"
1433                 + "." + vnfInstanceId
1434                 + ".vfModules"
1435                 + ".xbitestmodulereplace0\\.\\.XbiTestModuleReplace\\.\\.base_ocg\\.\\.module-0"
1436                 + "." + vfModuleInstanceId));
1437     }
1438
1439     private ServiceModel newestServiceModel() {
1440         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__target_newest_service_model.json", ServiceModel.class);
1441     }
1442
1443     private ServiceInstantiation upgradeVfModulePayload() {
1444         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__fe_input_cypress.json", ServiceInstantiation.class);
1445     }
1446
1447     @Test
1448     public void deployService_failIt_retryDeploy_getRetryAsTemplate_makeSureFalsyIsFailedInTemplate() {
1449
1450         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
1451
1452         //push alacarte with 1 vnf, verify STATUS pending
1453         UUID uuid = pushALaCarteWithVnf();
1454         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
1455
1456         //mock mso to answer 200 of create service instance request, verify STATUS in progress
1457         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
1458             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
1459
1460         //mock mso to answer FAILED for service instance create
1461         final RestObject<AsyncRequestStatus> failedResponse = asyncRequestStatusResponseAsRestObject(FAILED_STR);
1462         final String failureDescription = "Some deep failure";
1463         failedResponse.get().request.requestStatus.setStatusMessage(failureDescription);
1464         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
1465             thenReturn(failedResponse);
1466
1467         //Wait till job failed
1468         processJobsCountTimesAndAssertStatus(uuid, 3, FAILED);
1469
1470         //make sure retry request jas isFailed = true, and status message is with failureDescription
1471         ServiceInstantiation retryRequest = asyncInstantiationBL.getBulkForRetry(uuid);
1472         assertTrue(retryRequest.getIsFailed());
1473         assertEquals(failureDescription, retryRequest.getStatusMessage());
1474
1475         //deploy retry job and it's template
1476         UUID retryUuid = asyncInstantiationBL.pushBulkJob(retryRequest, USER_ID).get(0);
1477         ServiceInstantiation templateOfRetry = instantiationTemplates.getJobRequestAsTemplate(retryUuid);
1478
1479         //make sure the template request has isFailed = false, and no status message
1480         assertFalse(templateOfRetry.getIsFailed());
1481         assertNull(templateOfRetry.getStatusMessage());
1482     }
1483
1484     @Test
1485     public void oneVnfExistsAddAnotherVnf(){
1486         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
1487         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
1488
1489         ServiceInstantiation serviceInstantiation = readJsonResourceFileAsObject("/payload_jsons/vnf/one_vnf_exists_add_another_vnf_expected_bulk.json",
1490             ServiceInstantiation.class);
1491         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
1492         assertThat(uuids, hasSize(1));
1493
1494         //mock mso to answer 200 of create vnf instance request
1495         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("e6cc1c4f-05f7-49bc-8e86-ac2eb92baaaa/vnfs"), any())).thenReturn(
1496             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
1497
1498         //mock mso to answer msoVnfStatus (COMPLETE) for vnf creation status,
1499         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
1500             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1501
1502         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED);
1503         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith("e6cc1c4f-05f7-49bc-8e86-ac2eb92baaaa/vnfs"), any());
1504         verify(restMso, times(1)).GetForObject(any(), any());
1505     }
1506
1507 }