vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import static java.util.stream.Collectors.counting;
  24 import static java.util.stream.Collectors.groupingBy;
  25 import static java.util.stream.Collectors.joining;
  26 import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals;
  27 import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals;
  28 import static net.javacrumbs.jsonunit.JsonMatchers.jsonNodePresent;
  29 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  30 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartMatches;
  31 import static net.javacrumbs.jsonunit.core.Option.IGNORING_ARRAY_ORDER;
  32 import static org.hamcrest.CoreMatchers.equalTo;
  33 import static org.hamcrest.CoreMatchers.is;
  34 import static org.hamcrest.CoreMatchers.notNullValue;
  35 import static org.hamcrest.CoreMatchers.nullValue;
  36 import static org.hamcrest.MatcherAssert.assertThat;
  37 import static org.hamcrest.Matchers.hasProperty;
  38 import static org.hamcrest.Matchers.hasSize;
  39 import static org.hamcrest.core.Every.everyItem;
  40 import static org.mockito.ArgumentMatchers.any;
  41 import static org.mockito.ArgumentMatchers.argThat;
  42 import static org.mockito.ArgumentMatchers.endsWith;
  43 import static org.mockito.ArgumentMatchers.eq;
  44 import static org.mockito.ArgumentMatchers.isNull;
  45 import static org.mockito.Mockito.reset;
  46 import static org.mockito.Mockito.times;
  47 import static org.mockito.Mockito.verify;
  48 import static org.mockito.Mockito.when;
  49 import static org.onap.vid.job.Job.JobStatus.COMPLETED;
  50 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_ERRORS;
  51 import static org.onap.vid.job.Job.JobStatus.COMPLETED_WITH_NO_ACTION;
  52 import static org.onap.vid.job.Job.JobStatus.FAILED;
  53 import static org.onap.vid.job.Job.JobStatus.IN_PROGRESS;
  54 import static org.onap.vid.job.Job.JobStatus.PAUSE;
  55 import static org.onap.vid.job.Job.JobStatus.PENDING;
  56 import static org.onap.vid.job.Job.JobStatus.PENDING_RESOURCE;
  57 import static org.onap.vid.job.Job.JobStatus.RESOURCE_IN_PROGRESS;
  58 import static org.onap.vid.job.Job.JobStatus.STOPPED;
  59 import static org.onap.vid.job.impl.JobSchedulerInitializer.WORKERS_TOPICS;
  60 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  61 import static org.onap.vid.testUtils.TestUtils.readJsonResourceFileAsObject;
  62 import static org.testng.Assert.assertNull;
  63 import static org.testng.AssertJUnit.assertEquals;
  64 import static org.testng.AssertJUnit.assertFalse;
  65 import static org.testng.AssertJUnit.assertTrue;
  66
  67 import com.fasterxml.jackson.databind.JsonNode;
  68 import com.google.common.collect.ImmutableList;
  69 import com.google.common.collect.ImmutableMap;
  70 import java.io.IOException;
  71 import java.lang.reflect.Method;
  72 import java.util.Collection;
  73 import java.util.Collections;
  74 import java.util.List;
  75 import java.util.Map;
  76 import java.util.Optional;
  77 import java.util.Stack;
  78 import java.util.UUID;
  79 import java.util.function.BiConsumer;
  80 import java.util.function.Supplier;
  81 import java.util.stream.Collectors;
  82 import java.util.stream.IntStream;
  83 import java.util.stream.Stream;
  84 import javax.inject.Inject;
  85 import javax.ws.rs.ProcessingException;
  86 import org.apache.commons.lang3.StringUtils;
  87 import org.apache.commons.lang3.mutable.MutableInt;
  88 import org.jetbrains.annotations.NotNull;
  89 import org.mockito.ArgumentCaptor;
  90 import org.mockito.Mockito;
  91 import org.mockito.hamcrest.MockitoHamcrest;
  92 import org.onap.portalsdk.core.service.DataAccessService;
  93 import org.onap.portalsdk.core.util.SystemProperties;
  94 import org.onap.vid.asdc.AsdcCatalogException;
  95 import org.onap.vid.changeManagement.RequestDetailsWrapper;
  96 import org.onap.vid.config.DataSourceConfig;
  97 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  98 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  99 import org.onap.vid.job.Job;
 100 import org.onap.vid.job.Job.JobStatus;
 101 import org.onap.vid.job.JobType;
 102 import org.onap.vid.job.JobsBrokerService;
 103 import org.onap.vid.job.command.CommandUtils;
 104 import org.onap.vid.job.command.InternalState;
 105 import org.onap.vid.model.Action;
 106 import org.onap.vid.model.JobAuditStatus;
 107 import org.onap.vid.model.NameCounter;
 108 import org.onap.vid.model.RequestReferencesContainer;
 109 import org.onap.vid.model.ServiceInfo;
 110 import org.onap.vid.model.ServiceModel;
 111 import org.onap.vid.model.serviceInstantiation.BaseResource;
 112 import org.onap.vid.model.serviceInstantiation.InstanceGroup;
 113 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
 114 import org.onap.vid.mso.RestMsoImplementation;
 115 import org.onap.vid.mso.RestObject;
 116 import org.onap.vid.mso.model.RequestReferences;
 117 import org.onap.vid.mso.rest.AsyncRequestStatus;
 118 import org.onap.vid.mso.rest.AsyncRequestStatusList;
 119 import org.onap.vid.properties.Features;
 120 import org.onap.vid.services.AsyncInstantiationBaseTest;
 121 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
 122 import org.onap.vid.services.AuditService;
 123 import org.onap.vid.services.InstantiationTemplatesService;
 124 import org.onap.vid.services.VersionService;
 125 import org.onap.vid.testUtils.TestUtils;
 126 import org.onap.vid.utils.DaoUtils;
 127 import org.springframework.http.HttpMethod;
 128 import org.springframework.test.context.ContextConfiguration;
 129 import org.testng.annotations.BeforeClass;
 130 import org.testng.annotations.BeforeMethod;
 131 import org.testng.annotations.DataProvider;
 132 import org.testng.annotations.Test;
 133 import org.togglz.core.manager.FeatureManager;
 134
 135 //it's more like integration test than UT
 136 //But it's very hard to test in API test so I use UT
 137 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
 138 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
 139
 140     private static final String FAILED_STR = "FAILED";
 141     private static final String COMPLETE_STR = "COMPLETE";
 142     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
 143     private static final String REQUESTED = "REQUESTED";
 144     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
 145     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
 146     private static String USER_ID =  "123";
 147     public static String REQUEST_ID = UUID.randomUUID().toString();
 148     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 149
 150     @Inject
 151     private VersionService versionService;
 152
 153     @Inject
 154     private JobsBrokerService jobsBrokerService;
 155
 156     @Inject
 157     private JobWorker jobWorker;
 158
 159     @Inject
 160     private FeatureManager featureManager;
 161
 162     @Inject
 163     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 164
 165     @Inject
 166     private AuditService auditService;
 167
 168     @Inject
 169     private RestMsoImplementation restMso;
 170
 171     @Inject
 172     private DataAccessService dataAccessService;
 173
 174     @Inject
 175     private CommandUtils commandUtils;
 176
 177     @Inject
 178     private InstantiationTemplatesService instantiationTemplates;
 179
 180     @BeforeClass
 181     void initServicesInfoService() {
 182         createInstanceParamsMaps();
 183         when(versionService.retrieveBuildNumber()).thenReturn("fakeBuild");
 184     }
 185
 186     @BeforeMethod
 187     void clearDb() {
 188         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 189         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 190         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 191     }
 192
 193     @BeforeMethod
 194     void defineMocks() {
 195         Mockito.reset(restMso);
 196         Mockito.reset(aaiClient);
 197         Mockito.reset(commandUtils);
 198         mockAaiClientAnyNameFree();
 199
 200         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 201         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 202         when(featureManager.isActive(Features.FLAG_2006_VFMODULE_TAKES_TENANT_AND_REGION_FROM_VNF)).thenReturn(true);
 203     }
 204
 205     @Test
 206     public void whenPushNewBulk_thenAllServicesAreInPending() {
 207
 208         pushMacroBulk();
 209         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 210         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 211     }
 212
 213     private List<UUID> pushMacroBulk() {
 214         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 215             createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 216             3, true,PROJECT_NAME, true);
 217         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 218     }
 219
 220     private UUID pushALaCarteWithVnf() {
 221         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 222         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 223         assertThat(uuids, hasSize(1));
 224         return uuids.get(0);
 225     }
 226
 227     private UUID pushALaCarteUpdateWithGroups() {
 228         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 229         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 230         assertThat(uuids, hasSize(1));
 231         return uuids.get(0);
 232     }
 233
 234     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 235         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 236     }
 237
 238     public static RestObject<RequestReferencesContainer> createResponseRandomIds(int statusCode) {
 239         return createResponse(statusCode, UUID.randomUUID().toString(), UUID.randomUUID().toString());
 240     }
 241
 242     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 243         RequestReferences requestReferences = new RequestReferences();
 244         requestReferences.setRequestId(requestId);
 245         requestReferences.setInstanceId(instanceId);
 246         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 247         restObject.set(new RequestReferencesContainer(requestReferences));
 248         restObject.setStatusCode(statusCode);
 249         restObject.setRaw(RAW_DATA_FROM_MSO);
 250         return restObject;
 251     }
 252
 253     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 254         return Stream.of(jobStatuses).map(
 255             Enum::toString).collect(ImmutableList.toImmutableList());
 256     }
 257
 258     /*
 259     Make sure service state is in progress once request has sent to MSO
 260     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 261     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 262     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 263     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 264     */
 265     @Test
 266     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 267
 268         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 269         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 270         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 271         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 272
 273         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 274             .thenReturn(createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 275
 276         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 277             statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 278             statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 279             statusesToStrings(PENDING, STOPPED)
 280         );
 281
 282         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 283             ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 284             ImmutableList.of(REQUESTED, FAILED_STR),
 285             ImmutableList.of()
 286         );
 287
 288         List<UUID> uuids = pushMacroBulk();
 289         UUID firstJobUuid = uuids.get(0);
 290         UUID secondJobUuid = uuids.get(1);
 291         //assert that when get ProcessingException from restMso, status remain the same
 292         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 293         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 294
 295         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 296         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 297         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 298
 299         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 300         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 301         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 302
 303         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 304         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 305         processJobsCountTimesAndAssertStatus(firstJobUuid, 10, IN_PROGRESS, PENDING);
 306
 307         //when get job COMPLETE from MSO, service status become COMPLETED
 308         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 309         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 310         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, firstJobUuid);
 311
 312
 313         //for use later in the test
 314         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 315             Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 316         expectedJobStatusMap.put(firstJobUuid, COMPLETED);
 317
 318         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 319         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 320             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 321         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 322         processJobsCountTimes(10);
 323
 324         expectedJobStatusMap.put(secondJobUuid, JobStatus.IN_PROGRESS);
 325         listServicesAndAssertStatus(expectedJobStatusMap);
 326
 327
 328         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 329         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 330         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 331         expectedJobStatusMap.put(secondJobUuid, JobStatus.FAILED);
 332         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 333             e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 334         ));
 335
 336         listServicesAndAssertStatus(expectedJobStatusMap);
 337         IntStream.range(0, uuids.size()).forEach(i -> {
 338             UUID uuid = uuids.get(i);
 339             List<String> vidStatuses = auditService.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 340             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 341         });
 342
 343         //assert no more jobs to pull
 344         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 345         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 346     }
 347
 348
 349     @DataProvider
 350     public static Object[][] AlaCarteStatuses(Method test) {
 351         return new Object[][]{
 352             {COMPLETE_STR, JobStatus.COMPLETED},
 353             {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS},
 354         };
 355     }
 356
 357     /*
 358     Make sure service state is in progress once request has sent to MSO
 359     Make sure service state is watching until state changes to complemented
 360     Make sure service state is watching until vnf state changes to completed
 361     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 362     status Creating
 363      */
 364     @Test(dataProvider = "AlaCarteStatuses")
 365     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus) {
 366         /*
 367             [v]  + push alacarte with 1 vnf
 368             [v]    verify STATUS pending
 369             [v]  + pull+execute  (should post to MSO)
 370             [v]    verify STATUS in progress
 371             [v]  + pull+execute  (should GET completed from MSO)
 372             [v]    verify STATUS in progress; TYPE watching
 373             [v]    verify job#2 *new* VNF job STATUS creating
 374             [v]  + pull+execute job#2 (should post to MSO)
 375             [v]    verify job#2 STATUS resource in progress
 376             [v]    verify job#1 STATUS in progress
 377             [v]  + pull+execute job#2 (should GET completed from MSO)
 378             [v]    verify job#2 STATUS completed
 379             [v]  + pull+execute job#1
 380             [v]    verify job#1 STATUS completed
 381
 382            * not looking on audit (yet)
 383         */
 384         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false); // this makes the test pass without mocking the vfModules
 385         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 386         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 387         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 388
 389
 390         //push alacarte with 1 vnf, verify STATUS pending
 391         UUID uuid = pushALaCarteWithVnf();
 392         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 393         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 394         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 395             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 396         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 397         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 398             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 399         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 400         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 401             createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 402         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 403             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 404
 405         processJobsCountTimesAndAssertStatus(uuid, 100, expectedServiceStatus);
 406         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 407         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 408         verify(restMso, times(2)).GetForObject(any(), any());
 409
 410     }
 411
 412     /*
 413     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 414
 415     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 416     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 417
 418     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 419     And union these tests to single one.
 420      */
 421
 422     @Test
 423     public void testALaCarteLifeCycle1Vnf2VfModules() {
 424
 425
 426         String msoVnfStatus = COMPLETE_STR;
 427         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 428         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 429         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 430         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 431         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 432         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 433         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 434         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 435
 436
 437         //push alacarte with 1 vnf, verify STATUS pending
 438         UUID uuid = pushALaCarteWithVnf();
 439         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 440
 441         /*---------- service -----------*/
 442
 443         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 444         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 445             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 446
 447         //mock mso to answer COMPLETE for service instance create
 448         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 449             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 450
 451         /*---------- vnf -----------*/
 452
 453         //mock mso to answer 200 of create vnf instance request
 454         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any())).thenReturn(
 455             createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 456
 457         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 458         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 459             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 460
 461         try {
 462             reset(commandUtils);
 463             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true);
 464             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false);
 465         } catch (AsdcCatalogException e) {
 466
 467         }
 468
 469         /*---------- vf Module without volume group name (base) -----------*/
 470
 471         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 472         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 473             createResponse(200, UUID.randomUUID().toString(), VG_REQUEST_ID));
 474         //mock mso to answer for vf module orchestration request
 475         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 476             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 477
 478         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any())).thenReturn(
 479             createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 480         //mock mso to answer for volume group orchestration request
 481         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 482             asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 483
 484         /*---------- vfModule -----------*/
 485
 486         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 487         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any())).thenReturn(
 488             createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 489
 490         //mock mso to answer for vf module orchestration request
 491         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 492             asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 493
 494         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 495         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 496         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), any());
 497         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), any());
 498         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), any());
 499         verify(restMso, times(5)).GetForObject(any(), any());
 500     }
 501
 502     @Test
 503     public void testALaCarteLifeCycle2Networks() {
 504
 505         //Create Service with 2 networks, and make sure they created in sequence (and not in parallel)
 506         //Config MSO to response 200 only to first network creation. And answer 500 for second one.
 507         //Then MSO return in_progress some times (like 10 times), and then return COMPLETE.
 508         //Only when MSO return COMPLETE for first network, config MSO to return 200 for second network creation
 509
 510         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 511         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 512         final String NETWORK_REQUEST_ID1 = UUID.randomUUID().toString();
 513         final String NETWORK_INSTANCE_ID1 = UUID.randomUUID().toString();
 514         //TODO use them later for different networks
 515         final String NETWORK_REQUEST_ID2 = UUID.randomUUID().toString();
 516         final String NETWORK_INSTANCE_ID2 = UUID.randomUUID().toString();
 517
 518
 519         NetworkDetails networkDetails1 = new NetworkDetails("LukaDoncic", "1");
 520         NetworkDetails networkDetails2 = new NetworkDetails("KevinDurant", "2");
 521
 522         /*---------- service -----------*/
 523
 524         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 525         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
 526             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 527
 528         //mock mso to answer COMPLETE for service instance create
 529         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 530             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 531
 532         final MutableInt secondNetworkCode = new MutableInt(500);
 533         final MutableInt inProgressCount = new MutableInt(0);
 534
 535         /*---------- network 1-----------*/
 536
 537         //mock mso to answer 200 of first create network instance request
 538         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 539             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails1.name))) ,
 540             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 541             createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 542
 543         //mock mso to answer IN_PROGRESS 10 times, and only then COMPLETE for first network
 544         //Once COMPLETE, second network creation will return 200
 545         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID1), eq(AsyncRequestStatus.class))).
 546             thenAnswer(x->{
 547                 String status;
 548                 if (inProgressCount.getValue()<10) {
 549                     status = IN_PROGRESS_STR;
 550                 } else {
 551                     secondNetworkCode.setValue(200);
 552                     status = COMPLETE_STR;
 553                 }
 554                 inProgressCount.add(1);
 555                 return asyncRequestStatusResponseAsRestObject(status);
 556             });
 557
 558         /*---------- network 2-----------*/
 559
 560         //mock MSO to return status code of secondNetworkCode (500 and 200 after first one COMPLETED)
 561         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class),
 562             MockitoHamcrest.argThat(jsonPartMatches("requestDetails.requestInfo.instanceName", equalTo(networkDetails2.name))) ,
 563             endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenAnswer(x->
 564             createResponse(secondNetworkCode.intValue(), NETWORK_INSTANCE_ID2, NETWORK_REQUEST_ID2));
 565
 566 //        when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any() , endsWith(SERVICE_INSTANCE_ID + "/networks"), any())).thenReturn(
 567 //                createResponse(200, NETWORK_INSTANCE_ID1, NETWORK_REQUEST_ID1));
 568         //mock mso to answer COMPLETE for network creation status,
 569
 570         when(restMso.GetForObject(endsWith(NETWORK_REQUEST_ID2), eq(AsyncRequestStatus.class))).
 571             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 572
 573
 574         /*---------- Create request and process it -----------*/
 575         //push alacarte with 2 networks, verify STATUS pending
 576         when(featureManager.isActive(Features.FLAG_EXP_CREATE_RESOURCES_IN_PARALLEL)).thenReturn(false);
 577         ServiceInstantiation serviceInstantiation = generateALaCarteWithNetworksPayload(ImmutableList.of(networkDetails1, networkDetails2));
 578         UUID uuid = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID).get(0);
 579         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 580
 581         processJobsCountTimesAndAssertStatus(uuid, 200, COMPLETED);
 582
 583         //validate the mso request id is the right one
 584         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 585         ServiceInfo serviceInfo = serviceInfoList.get(0);
 586         assertThat(serviceInfo.getMsoRequestId(), is(UUID.fromString(SERVICE_REQUEST_ID)));
 587
 588         /*---------- verify -----------*/
 589         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq("/serviceInstantiation/v7/serviceInstances"), any());
 590         verify(restMso, times(2)).restCall(eq(HttpMethod.POST), any(), any(), endsWith(SERVICE_INSTANCE_ID + "/networks"), any());
 591         //get status
 592         verify(restMso, times(1)).GetForObject(endsWith(SERVICE_REQUEST_ID), any());
 593         verify(restMso, times(11)).GetForObject(endsWith(NETWORK_REQUEST_ID1), any());
 594         verify(restMso, times(1)).GetForObject(endsWith(NETWORK_REQUEST_ID2), any());
 595     }
 596
 597     @Test
 598     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 599         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 600         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 601         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 602             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 603
 604         List<UUID> uuids = pushMacroBulk();
 605         processJobsCountTimesAndAssertStatus(uuids.get(0), 5, IN_PROGRESS, PENDING);  //JOB shall become IN_PROGRESS but service info is still pending
 606
 607         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 608         Mockito.reset(aaiClient); // must forget the "thenThrow"
 609         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 610         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, COMPLETED, COMPLETED);
 611
 612     }
 613
 614     @Test
 615     public void testAaiResponseNameUsedTillMaxRetries() {
 616         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 617         //simulate MSO to return good result, for making sure we failed because of AAI error
 618         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 619         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 620             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 621
 622         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 623         List<UUID> uuids = pushMacroBulk();
 624         processJobsCountTimesAndAssertStatus(uuids.get(0), 20, FAILED, STOPPED);
 625     }
 626
 627     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 628         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 629     }
 630
 631     //return the pulled job (and not the pushed job)
 632     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 633         Optional<Job> job = pullJob(topic, pullingAssertion);
 634
 635         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 636
 637         try {
 638             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 639
 640             if (pullingAssertion) {
 641                 //assert another pulling on same topic return no result (before push back)
 642                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 643             }
 644
 645         } finally {
 646             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 647         }
 648         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 649         return job.get();
 650     }
 651
 652     private void simplePullJobProcessAndPushBack(JobStatus topic) {
 653         Optional<Job> optionalJob =  jobsBrokerService.pull(topic, randomUuid());
 654         optionalJob.ifPresent(job->{
 655             Job nextJob = jobWorker.executeJobAndGetNext(job);
 656             jobsBrokerService.pushBack(nextJob);
 657         });
 658     }
 659
 660     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 661         JobType expectedNextJobType) {
 662         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 663         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 664         return job;
 665     }
 666
 667     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 668         JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 669         return retryWithAssertionsLimit(retries, () -> {
 670             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 671             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 672             if (internalState != null) {
 673                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 674             }
 675             return job;
 676         });
 677     }
 678
 679     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 680         java.util.Stack<AssertionError> history = new Stack<>();
 681
 682         do {
 683             try {
 684                 return supplier.get();
 685             } catch (AssertionError assertionError) {
 686                 history.push(assertionError);
 687             }
 688         } while (history.size() < retries);
 689
 690         // No success:
 691         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 692             .map(Throwable::getMessage)
 693             .map(s -> s.replace('\n', ' '))
 694             .map(s -> s.replaceAll("\\s{2,}"," "))
 695             .distinct()
 696             .collect(joining("\n   ", "\n   ", "")), history.peek());
 697     }
 698
 699     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 700         if (pullingAssertion) {
 701             //assert pulling on inverse topic return no result
 702             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 703         }
 704
 705         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 706         assertTrue("no job fetched", job.isPresent());
 707
 708         if (pullingAssertion) {
 709             //assert another pulling on same topic return no result
 710             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 711         }
 712
 713         return job;
 714     }
 715
 716     private JobStatus inverseTopic(JobStatus topic) {
 717         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 718     }
 719
 720
 721     @Test
 722     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 723         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(500));
 724         //assert that when get ProcessingException from restMso, status remain the same
 725         List<UUID> uuids = pushMacroBulk();
 726         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, JobStatus.FAILED, JobStatus.STOPPED);
 727     }
 728
 729     @Test
 730     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 731         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty()))).thenReturn(createResponse(200));
 732         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 733             thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 734
 735         //assert that when get ProcessingException from restMso, status remain the same
 736         List<UUID> uuids = pushMacroBulk();
 737         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, PAUSE, PENDING);
 738
 739
 740         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 741         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 742             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 743         processJobsCountTimesAndAssertStatus(uuids.get(0), 30, IN_PROGRESS, PENDING);
 744
 745         //the job get COMPLETE response this job is copmpleted and then also other jobs
 746         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 747             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 748         processJobsCountTimesAndAssertStatus(uuids.get(0), 200, COMPLETED, COMPLETED);
 749
 750         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 751         List<String> vidStatuses = auditService.getAuditStatuses(uuids.get(0), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 752         assertThat(vidStatuses, is(expectedStatusesForVid));
 753     }
 754
 755     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 756         pushMacroBulk();
 757         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 758     }
 759
 760     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 761         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 762         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job.getUuid());
 763         return job;
 764     }
 765
 766     @Test
 767     public void test2BulksLifeCyclesAreIndependent() {
 768
 769         final String SERVICE1_REQUEST_ID = UUID.randomUUID().toString();
 770         final String SERVICE1_INSTANCE_ID = UUID.randomUUID().toString();
 771         final String SERVICE2_REQUEST_ID = UUID.randomUUID().toString();
 772         final String SERVICE2_INSTANCE_ID = UUID.randomUUID().toString();
 773         final String SERVICE3_4_REQUEST_ID = UUID.randomUUID().toString();
 774         final String SERVICE3_4_INSTANCE_ID = UUID.randomUUID().toString();
 775
 776
 777         //create first bulk and make one job in progress
 778         List<UUID> firstBulksIDs = pushMacroBulk();
 779         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 780             .thenReturn(createResponse(200, SERVICE1_INSTANCE_ID, SERVICE1_REQUEST_ID));
 781         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 782             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 783         processJobsCountTimesAndAssertStatus(firstBulksIDs.get(0), 30, IN_PROGRESS, PENDING);
 784
 785         //create 2nd bulk, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 786         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 787             .thenReturn(createResponse(200, SERVICE2_INSTANCE_ID, SERVICE2_REQUEST_ID));
 788         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 789             thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 790         List<UUID> secondBulksIDs = pushMacroBulk();
 791         processJobsCountTimes(30);
 792         Map<JobStatus, Long> statusCount = getJobStatusesCount();
 793         assertThat(statusCount.get(IN_PROGRESS), is(2L));
 794         assertThat(statusCount.get(PENDING), is(4L));
 795
 796         //return failed to first job
 797         //first bulk statuses shall be: FAILED, STOPPED, STOPPED
 798         //second bulk statuses shall be: IN_PROGRESS, PENDING, PENDING
 799         when(restMso.GetForObject(endsWith(SERVICE1_REQUEST_ID), eq(AsyncRequestStatus.class))).
 800             thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 801         processJobsCountTimes(30);
 802         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 803             asyncInstantiationBL.getAllServicesInfo()
 804                 .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 805         ServiceInfo failedJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(firstBulksIDs.get(0))).findFirst().get();
 806         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 807         ServiceInfo successJob = asyncInstantiationBL.getAllServicesInfo().stream().filter(x->x.getJobId().equals(secondBulksIDs.get(0))).findFirst().get();
 808         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob.getJobId());
 809
 810         //return completed to all other jobs
 811         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), any(), eq(Optional.empty())))
 812             .thenReturn(createResponse(200, SERVICE3_4_INSTANCE_ID, SERVICE3_4_REQUEST_ID));
 813         when(restMso.GetForObject(endsWith(SERVICE2_REQUEST_ID), eq(AsyncRequestStatus.class))).
 814             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 815         when(restMso.GetForObject(endsWith(SERVICE3_4_REQUEST_ID), eq(AsyncRequestStatus.class))).
 816             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 817
 818         processJobsCountTimes(30);
 819         servicesByTemplateId = asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getTemplateId));
 820         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob.getJobId());
 821         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob.getJobId());
 822         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 823         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 824         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 825     }
 826
 827     protected Map<JobStatus, Long> getJobStatusesCount() {
 828         return asyncInstantiationBL.getAllServicesInfo().stream().collect(groupingBy(ServiceInfo::getJobStatus, counting()));
 829     }
 830
 831     @Test
 832     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 833
 834         /*
 835         new feature: skip service (existing impl) and skip group (new impl)
 836         service+group aren't touched, 2 groups ARE created
 837
 838         [v]  success if all GROUPs success
 839
 840         Next test variation should:
 841         [ ]  error if all GROUPs error
 842         [ ]  completed with error if 1 GROUP error
 843
 844
 845         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 846         [v]    verify STATUS pending
 847         [v]  + pull+execute  (should NOT post to MSO)
 848         [v]    verify STATUS in progress; TYPE watching
 849                ...
 850
 851         [v]  + pull+execute job#1
 852         [v]    verify job#1 STATUS in progress; TYPE watching
 853
 854         [v]  + pull+execute job#6 (should post to MSO)
 855         [v]    verify job#6 STATUS resource in progress
 856         [v]  + pull+execute job#1
 857         [v]    verify job#1 STATUS in progress; TYPE watching
 858         [v]  + pull+execute job#6 (should get from MSO)
 859         [v]    verify job#6 STATUS completed
 860         [v]  + pull+execute job#1
 861         [v]    verify job#1 STATUS in progress; TYPE watching
 862
 863         [v]  + pull+execute job#7 (should post to MSO)
 864         [v]    verify job#7 STATUS resource in progress
 865         [v]  + pull+execute job#1
 866         [v]    verify job#1 STATUS in progress; TYPE watching
 867         [v]  + pull+execute job#7 (should get from MSO)
 868         [v]    verify job#7 STATUS completed
 869         [v]  + pull+execute job#1
 870         [v]    verify job#1 STATUS completed
 871
 872          */
 873
 874         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 875         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 876         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 877         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 878
 879         // Utility method
 880         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 881             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 882         };
 883
 884         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 885         UUID uuid = pushALaCarteUpdateWithGroups();
 886         singleServicesAndAssertStatus(PENDING, uuid);
 887
 888         // take from pending, put in-progress -> 3 delete-child were born
 889         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Create, InternalState.INITIAL, 1);
 890         verifyQueueSizes(ImmutableMap.of(
 891             IN_PROGRESS, 1
 892         ));
 893
 894         // take job #1 from phase delete to phase create -> 3 create-child were born
 895         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 896         verifyQueueSizes(ImmutableMap.of(
 897             IN_PROGRESS, 1, PENDING_RESOURCE, 3
 898         ));
 899
 900         // prepare MSO mock
 901         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("instanceGroups"), eq(Optional.empty())))
 902             .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 903             .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 904             .thenReturn(null);
 905         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 906             thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 907
 908         // take first "none" child from creating to COMPLETED_WITH_NO_ACTION
 909         // note there's no concrete mechanism that makes the first child be
 910         // the "action=None" case, but that's what happens, and following lines
 911         // relies on that fact.
 912         pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroup, Action.Create, null, 1);
 913
 914         // take each of next two children from creating to in-progress, then to completed
 915         // verify job #1 is watching, and MSO is getting requests
 916         Stream.of(1, 2).forEach(i -> {
 917             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 918             pullJobProcessAndPushBackWithTypeAssertion(PENDING_RESOURCE, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Create, InternalState.IN_PROGRESS, 1);
 919             verify(restMso, times(i)).restCall(any(), any(), any(), any(), any());
 920
 921             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 922             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.InstanceGroup, Action.Create, null, 3);
 923             verify(restMso, times(i)).GetForObject(any(), any());
 924         });
 925
 926         // job #1 is done as all children are done
 927         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 928         verifyQueueSizes(ImmutableMap.of(COMPLETED, 3, COMPLETED_WITH_NO_ACTION, 1));
 929     }
 930
 931     @DataProvider
 932     public static Object[][] createAndDeleteIntegrationTestDataProvider(Method test) {
 933         return new Object[][]{
 934             {"create and delete both bad http code", createResponse(400), createResponse(500), null, null, FAILED, 0},
 935             {"create and delete success and status is success ", createResponseRandomIds(202), createResponseRandomIds(202),
 936                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED, 2},
 937             {"create and delete success, create status FAILED, delete status COMPLETED", createResponseRandomIds(202), createResponseRandomIds(202),
 938                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 2},
 939             {"create and delete success, create status FAILED, delete status FAILED", createResponseRandomIds(202), createResponseRandomIds(202),
 940                 asyncRequestStatusResponseAsRestObject(FAILED_STR), asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 2},
 941             {"create success but delete failed and status is success ", createResponseRandomIds(202), createResponseRandomIds(400),
 942                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR), null, COMPLETED_WITH_ERRORS, 1},
 943             {"delete success but create failed and status is success ", createResponseRandomIds(400), createResponseRandomIds(202),
 944                 null, asyncRequestStatusResponseAsRestObject(COMPLETE_STR), COMPLETED_WITH_ERRORS, 1},
 945             {"delete success but create failed and status of delete is FAILED ", createResponseRandomIds(400), createResponseRandomIds(202),
 946                 null, asyncRequestStatusResponseAsRestObject(FAILED_STR), FAILED, 1}
 947         };
 948     }
 949
 950     //this test is going along with AsyncInstantiationALaCarteApiTest.viewEditVnfGroup__verifyStatusAndAudit API test
 951     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 952     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 953     public void vnfGropingIntegrationTest(
 954         String desc,
 955         RestObject<RequestReferencesContainer> createGroupResponse,
 956         RestObject<RequestReferencesContainer> deleteGroupResponse,
 957         RestObject<AsyncRequestStatus> createStatusResponse,
 958         RestObject<AsyncRequestStatus> deleteStatusResponse,
 959         JobStatus expectedJobStatus,
 960         int getStatusCounter) {
 961
 962         UUID jobUUID = createAndDeleteIntegrationTest("/payload_jsons/VnfGroupCreate1Delete1None1Request.json",
 963             "/serviceInstantiation/v7/instanceGroups",
 964             createGroupResponse,
 965             "/serviceInstantiation/v7/instanceGroups/VNF_GROUP1_INSTANCE_ID",
 966             deleteGroupResponse,
 967             createStatusResponse,
 968             deleteStatusResponse,
 969             expectedJobStatus,
 970             getStatusCounter);
 971
 972         ServiceInstantiation bulkForRetry = asyncInstantiationBL.getBulkForRetry(jobUUID);
 973         InstanceGroup vnfGroupShouldBeDeleted = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0:001");
 974         InstanceGroup vnfGroupShouldBeCreated = bulkForRetry.getVnfGroups().get("groupingservicefortest..ResourceInstanceGroup..0");
 975
 976         if (deleteStatusResponse == null || deleteStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 977             assertThat(vnfGroupShouldBeDeleted.getAction(), equalTo(Action.Delete));
 978             assertErrorForResource(vnfGroupShouldBeDeleted, deleteGroupResponse, deleteStatusResponse);
 979         }
 980
 981         if (createStatusResponse == null || createStatusResponse.get().request.requestStatus.getRequestState().equals(FAILED_STR)) {
 982             assertThat(vnfGroupShouldBeCreated.getAction(), equalTo(Action.Create));
 983             assertErrorForResource(vnfGroupShouldBeCreated, createGroupResponse, createStatusResponse);
 984         }
 985     }
 986
 987     //this test is going along with AsyncInstantiationALaCarteApiTest3.delete1Create1VnfFromService API test
 988     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
 989     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
 990     public void vnfsIntegrationTest(
 991         String desc,
 992         RestObject<RequestReferencesContainer> createVnfResponse,
 993         RestObject<RequestReferencesContainer> deleteVnfResponse,
 994         RestObject<AsyncRequestStatus> createStatusResponse,
 995         RestObject<AsyncRequestStatus> deleteStatusResponse,
 996         JobStatus expectedJobStatus,
 997         int getStatusCounter) {
 998
 999         createAndDeleteIntegrationTest("/payload_jsons/vnfDelete1Create1Request.json",
1000             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs",
1001             createVnfResponse,
1002             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID",
1003             deleteVnfResponse,
1004             createStatusResponse,
1005             deleteStatusResponse,
1006             expectedJobStatus,
1007             getStatusCounter);
1008     }
1009
1010     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1011     public void vfModulesIntegrationTest(
1012         String desc,
1013         RestObject<RequestReferencesContainer> createVfModuleResponse,
1014         RestObject<RequestReferencesContainer> deleteVfModuleResponse,
1015         RestObject<AsyncRequestStatus> createStatusResponse,
1016         RestObject<AsyncRequestStatus> deleteStatusResponse,
1017         JobStatus expectedJobStatus,
1018         int getStatusCounter) throws IOException, AsdcCatalogException {
1019
1020         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "f8360508-3f17-4414-a2ed-6bc71161e8db")).thenReturn(true);
1021         when(commandUtils.isVfModuleBaseModule("6b528779-44a3-4472-bdff-9cd15ec93450", "25284168-24bb-4698-8cb4-3f509146eca5")).thenReturn(false);
1022
1023         createAndDeleteIntegrationTest("/payload_jsons/vfModuleDelete1Create1None1Request.json",
1024             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules",
1025             createVfModuleResponse,
1026             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/vnfs/VNF_INSTANCE_ID/vfModules/VF_MODULE_INSTANCE_ID",
1027             deleteVfModuleResponse,
1028             createStatusResponse,
1029             deleteStatusResponse,
1030             expectedJobStatus,
1031             getStatusCounter);
1032     }
1033
1034     //this test is going along with AsyncInstantiationALaCarteApiTest.delete1Create1NetworkFromService API test
1035     //The API test has only the happy flow scenario, while this test also test additional MSO responses (mostly non happy)
1036     @Test(dataProvider="createAndDeleteIntegrationTestDataProvider")
1037     public void networksIntegrationTest(
1038         String desc,
1039         RestObject<RequestReferencesContainer> createNetworkResponse,
1040         RestObject<RequestReferencesContainer> deleteNetworkResponse,
1041         RestObject<AsyncRequestStatus> createStatusResponse,
1042         RestObject<AsyncRequestStatus> deleteStatusResponse,
1043         JobStatus expectedJobStatus,
1044         int getStatusCounter) {
1045
1046         createAndDeleteIntegrationTest("/payload_jsons/networkDelete1Create1Request.json",
1047             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks",
1048             createNetworkResponse,
1049             "/serviceInstantiation/v7/serviceInstances/f8791436-8d55-4fde-b4d5-72dd2cf13cfb/networks/NETWORK_INSTANCE_ID",
1050             deleteNetworkResponse,
1051             createStatusResponse,
1052             deleteStatusResponse,
1053             expectedJobStatus,
1054             getStatusCounter);
1055     }
1056
1057     private UUID createAndDeleteIntegrationTest(String payload,
1058         String createPath,
1059         RestObject<RequestReferencesContainer> createResponse,
1060         String deletePath,
1061         RestObject<RequestReferencesContainer> deleteResponse,
1062         RestObject<AsyncRequestStatus> createStatusResponse,
1063         RestObject<AsyncRequestStatus> deleteStatusResponse,
1064         JobStatus expectedJobStatus,
1065         int getStatusCounter) {
1066         UUID jobUUID = asyncInstantiationBL.pushBulkJob(
1067             readJsonResourceFileAsObject(payload, ServiceInstantiation.class), "userId")
1068             .get(0);
1069
1070         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(createPath), any())).thenReturn(createResponse);
1071         when(restMso.restCall(eq(HttpMethod.DELETE), eq(RequestReferencesContainer.class), any(), eq(deletePath), any())).thenReturn(deleteResponse);
1072         if  (createStatusResponse!=null) {
1073             when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(createStatusResponse);
1074         }
1075         if  (deleteStatusResponse!=null) {
1076             when(restMso.GetForObject(endsWith(deleteResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class))).thenReturn(deleteStatusResponse);
1077         }
1078
1079         processJobsCountTimesAndAssertStatus(jobUUID, 40, expectedJobStatus);
1080
1081         verify(restMso, times(1)).restCall(eq(HttpMethod.POST), any(), any(), eq(createPath), any());
1082         verify(restMso, times(1)).restCall(eq(HttpMethod.DELETE), any(), any(), eq(deletePath), any());
1083         verify(restMso, times(getStatusCounter)).GetForObject(any(), any());
1084
1085         return jobUUID;
1086     }
1087
1088     @Test
1089     public void whenCreateTransportService_thanExpectedPre1806MacroRequestSent() {
1090         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generatePre1806MacroTransportServiceInstantiationPayload(null, null),"az2016").get(0);
1091         RestObject<RequestReferencesContainer> createResponse = createResponseRandomIds(202);
1092
1093         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq("/serviceInstantiation/v7/serviceInstances"), any()))
1094             .thenReturn(createResponse);
1095         when(restMso.GetForObject(endsWith(createResponse.get().getRequestReferences().getRequestId()), eq(AsyncRequestStatus.class)))
1096             .thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1097         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1098
1099         JsonNode expectedJson = readJsonResourceFileAsObject("/payload_jsons/pre_1806_macro_without_cloudConfiguration.json", JsonNode.class);
1100         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1101         verify(restMso).restCall(any(), any(), requestCaptor.capture(), any(), any());
1102         requestCaptor.getAllValues().forEach(x->assertJsonEquals(expectedJson, x));
1103     }
1104
1105     private void assertErrorForResource(BaseResource resource,
1106         RestObject<RequestReferencesContainer> deleteOrCreateResponse,
1107         RestObject<AsyncRequestStatus> statusResponse) {
1108         JobAuditStatus auditStatus = auditService.getResourceAuditStatus(resource.getTrackById());
1109         assertThat(auditStatus, is(notNullValue()));
1110         assertThat(auditStatus.getJobStatus(), equalTo(FAILED_STR));
1111         if (statusResponse == null) {
1112             String errorMessage = "Http Code:" + deleteOrCreateResponse.getStatusCode() + ", " + RAW_DATA_FROM_MSO;
1113             assertThat(auditStatus.getAdditionalInfo(), equalTo(errorMessage));
1114             assertThat(auditStatus.getRequestId(), is(nullValue()));
1115         } else {
1116             assertThat(auditStatus.getRequestId().toString(), equalTo(statusResponse.get().request.requestId));
1117         }
1118     }
1119
1120     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus) {
1121         processJobsCountTimes(times);
1122         singleServicesAndAssertStatus(expectedStatus, serviceJobId);
1123     }
1124
1125     private void processJobsCountTimes(int times) {
1126         for (int i = 0; i < times; i++) {
1127             WORKERS_TOPICS.forEach(this::simplePullJobProcessAndPushBack);
1128         }
1129     }
1130
1131     protected void processJobsCountTimesAndAssertStatus(UUID serviceJobId, int times, JobStatus expectedStatus, JobStatus otherJobsStatus) {
1132         processJobsCountTimes(times);
1133         listServicesAndAssertStatus(expectedStatus, otherJobsStatus, serviceJobId);
1134     }
1135
1136
1137     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
1138         final Collection<Job> peek = jobsBrokerService.peek();
1139         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
1140         assertThat(jobTypes, jsonEquals(expected));
1141     }
1142
1143     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1144         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1145         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, jobUUID);
1146
1147         return serviceInfoList;
1148     }
1149
1150     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
1151         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1152         assertEquals(1, serviceInfoList.size());
1153         ServiceInfo serviceInfo = serviceInfoList.get(0);
1154         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
1155         assertThat(serviceInfo.getJobId(), is(jobUUID));
1156         return serviceInfo;
1157     }
1158
1159     private boolean isServiceOnStatus(JobStatus expectedStatus) {
1160         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
1161         assertEquals(1, serviceInfoList.size());
1162         return serviceInfoList.get(0).getJobStatus()==expectedStatus;
1163     }
1164
1165     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, UUID jobUUID) {
1166         serviceInfoList.forEach(si->{
1167             if (si.getJobId().equals(jobUUID)) {
1168                 assertThat(si.getJobStatus(), is(pulledJobStatus));
1169             }
1170             else {
1171                 assertThat(si.getJobStatus(), is(otherJobsStatus));
1172             }
1173         });
1174     }
1175
1176     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
1177         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
1178             .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
1179         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
1180     }
1181
1182     private String randomUuid() {
1183         return UUID.randomUUID().toString();
1184     }
1185
1186     @Test
1187     public void whenResumeService_thanExpectedResumeRequestSent() {
1188         String instanceId = "a565e6ad-75d1-4493-98f1-33234b5c17e2"; //from feRequestResumeMacroService.json
1189         String originalRequestId = "894089b8-f7f4-418d-81da-34186fd32670"; //from msoResponseGetRequestsOfServiceInstance.json
1190         String resumeRequestId = randomUuid();
1191         String userId = TestUtils.generateRandomAlphaNumeric(6);
1192
1193         //prepare mocks for get all requests for instance id
1194         RestObject<AsyncRequestStatusList> getRequestByIdResponse = createAsyncRequestStatusListByInstanceId();
1195         when(restMso.GetForObject(
1196             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1197             eq(AsyncRequestStatusList.class)))
1198             .thenReturn(getRequestByIdResponse);
1199
1200         //prepare mocks resume request
1201         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId))))
1202             .thenReturn(createResponse(202, instanceId, resumeRequestId));
1203
1204         //prepare mocks for get resume status
1205         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class)))
1206             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1207                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1208                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1209
1210
1211         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), userId).get(0);
1212         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1213         verify(restMso).GetForObject(
1214             eq("/orchestrationRequests/v7?filter=serviceInstanceId:EQUALS:" + instanceId),
1215             eq(AsyncRequestStatusList.class));
1216         verify(restMso).restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), isNull(), eq(String.format("/orchestrationRequests/v7/%s/resume", originalRequestId)), eq(Optional.of(userId)));
1217         verify(restMso, times(3)).GetForObject(eq("/orchestrationRequests/v7/" + resumeRequestId), eq(AsyncRequestStatus.class));
1218     }
1219
1220     @Test
1221     public void givenResumeRequest_whenMsoReturnBadResponse_thanJobIsFailed() {
1222         //there is no mocks for restMSO which means restMso return bad response...
1223         UUID jobUUID = asyncInstantiationBL.pushBulkJob(generateResumeMacroPayload(), "abc").get(0);
1224         processJobsCountTimesAndAssertStatus(jobUUID, 20, FAILED);
1225     }
1226
1227     @NotNull
1228     private RestObject<AsyncRequestStatusList> createAsyncRequestStatusListByInstanceId() {
1229         AsyncRequestStatusList asyncRequestStatusList = readJsonResourceFileAsObject(
1230             "/payload_jsons/resume/msoResponseGetRequestsOfServiceInstance.json",
1231             AsyncRequestStatusList.class);
1232         RestObject<AsyncRequestStatusList> getRequestByIdResponse = new RestObject<>();
1233         getRequestByIdResponse.set(asyncRequestStatusList);
1234         getRequestByIdResponse.setStatusCode(200);
1235         return getRequestByIdResponse;
1236     }
1237
1238     private ServiceInstantiation generateResumeMacroPayload() {
1239         return readJsonResourceFileAsObject("/payload_jsons/resume/feRequestResumeMacroService.json", ServiceInstantiation.class);
1240     }
1241
1242     @Test
1243     public void whenUpgradingVfModule_thenExpectedReplaceRequestSent() throws AsdcCatalogException {
1244         String currentServiceInstanceId = "6196ab1f-2349-4b32-9b6c-cffeb0ccc79c";
1245         String currentVnfInstanceId = "d520268f-7489-4662-be59-f81495b3a069";
1246         String currentVfModuleInstanceId = "b0732bed-3ddf-43cc-b193-7f18db84e476";
1247
1248         assertTestPayloadFitsExpectedIds(upgradeVfModulePayload(), currentServiceInstanceId, currentVnfInstanceId, currentVfModuleInstanceId);
1249
1250         String replaceRequestId = randomUuid();
1251         String userId = "az2016";
1252
1253         String modelInvariantId = "b3a1a119-dede-4ed0-b077-2a617fa519a3";
1254         String newestModelUuid = "d9a5b318-187e-476d-97f7-a15687a927a9";
1255
1256         String expectedMsoReplacePath = "/serviceInstantiation/v7/serviceInstances/"
1257             + currentServiceInstanceId + "/vnfs/" + currentVnfInstanceId + "/vfModules/" + currentVfModuleInstanceId + "/replace";
1258
1259         when(commandUtils.getNewestModelUuid(eq(modelInvariantId))).thenReturn(newestModelUuid);
1260         when(commandUtils.getServiceModel(eq(newestModelUuid))).thenReturn(newestServiceModel());
1261
1262         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), eq(expectedMsoReplacePath), eq(Optional.of(userId))))
1263             .thenReturn(createResponse(202, currentVfModuleInstanceId, replaceRequestId));
1264
1265         when(restMso.GetForObject(eq("/orchestrationRequests/v7/" + replaceRequestId), eq(AsyncRequestStatus.class)))
1266             .thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1267                 asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR),
1268                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
1269
1270         enableAddCloudOwnerOnMsoRequest();
1271
1272
1273         UUID jobUUID = asyncInstantiationBL.pushBulkJob(upgradeVfModulePayload(), userId).get(0);
1274         processJobsCountTimesAndAssertStatus(jobUUID, 20, COMPLETED);
1275
1276
1277         ArgumentCaptor<RequestDetailsWrapper> requestCaptor = ArgumentCaptor.forClass(RequestDetailsWrapper.class);
1278         verify(restMso, times(1)).restCall(
1279             eq(HttpMethod.POST),
1280             eq(RequestReferencesContainer.class),
1281             requestCaptor.capture(),
1282             eq(expectedMsoReplacePath),
1283             eq(Optional.of(userId))
1284         );
1285
1286         JsonNode expectedPayloadToMso = readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__payload_to_mso.json", JsonNode.class);
1287         assertThat(requestCaptor.getValue(), jsonEquals(expectedPayloadToMso).when(IGNORING_ARRAY_ORDER));
1288     }
1289
1290     private void assertTestPayloadFitsExpectedIds(ServiceInstantiation upgradeVfModulePayload, String serviceInstanceId,
1291         String vnfInstanceId, String vfModuleInstanceId) {
1292         /*
1293         Just verifies the test and the input-file are using the same set of instance IDs
1294          */
1295         assertThat(upgradeVfModulePayload, jsonPartEquals("instanceId", serviceInstanceId));
1296         assertThat(upgradeVfModulePayload, jsonNodePresent(
1297             "vnfs"
1298                 + "." + vnfInstanceId
1299                 + ".vfModules"
1300                 + ".xbitestmodulereplace0\\.\\.XbiTestModuleReplace\\.\\.base_ocg\\.\\.module-0"
1301                 + "." + vfModuleInstanceId));
1302     }
1303
1304     private ServiceModel newestServiceModel() {
1305         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__target_newest_service_model.json", ServiceModel.class);
1306     }
1307
1308     private ServiceInstantiation upgradeVfModulePayload() {
1309         return readJsonResourceFileAsObject("/payload_jsons/vfmodule/upgrade_vfmodule_e2e__fe_input_cypress.json", ServiceInstantiation.class);
1310     }
1311
1312     @Test
1313     public void deployService_failIt_retryDeploy_getRetryAsTemplate_makeSureFalsyIsFailedInTemplate() {
1314
1315         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
1316
1317         //push alacarte with 1 vnf, verify STATUS pending
1318         UUID uuid = pushALaCarteWithVnf();
1319         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
1320
1321         //mock mso to answer 200 of create service instance request, verify STATUS in progress
1322         when(restMso.restCall(eq(HttpMethod.POST), eq(RequestReferencesContainer.class), any(), endsWith("serviceInstances"), any())).thenReturn(
1323             createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
1324
1325         //mock mso to answer FAILED for service instance create
1326         final RestObject<AsyncRequestStatus> failedResponse = asyncRequestStatusResponseAsRestObject(FAILED_STR);
1327         final String failureDescription = "Some deep failure";
1328         failedResponse.get().request.requestStatus.setStatusMessage(failureDescription);
1329         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
1330             thenReturn(failedResponse);
1331
1332         //Wait till job failed
1333         processJobsCountTimesAndAssertStatus(uuid, 3, FAILED);
1334
1335         //make sure retry request jas isFailed = true, and status message is with failureDescription
1336         ServiceInstantiation retryRequest = asyncInstantiationBL.getBulkForRetry(uuid);
1337         assertTrue(retryRequest.getIsFailed());
1338         assertEquals(failureDescription, retryRequest.getStatusMessage());
1339
1340         //deploy retry job and it's template
1341         UUID retryUuid = asyncInstantiationBL.pushBulkJob(retryRequest, USER_ID).get(0);
1342         ServiceInstantiation templateOfRetry = instantiationTemplates.getJobRequestAsTemplate(retryUuid);
1343
1344         //make sure the template request has isFailed = false, and no status message
1345         assertFalse(templateOfRetry.getIsFailed());
1346         assertNull(templateOfRetry.getStatusMessage());
1347     }
1348
1349 }