vid-app-common/src/test/java/org/onap/vid/job/impl/AsyncInstantiationIntegrationTest.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * VID
   4  * ================================================================================
   5  * Copyright (C) 2017 - 2019 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.vid.job.impl;
  22
  23 import com.google.common.collect.ImmutableList;
  24 import com.google.common.collect.ImmutableMap;
  25 import org.apache.commons.lang3.StringUtils;
  26 import org.mockito.Mockito;
  27 import org.onap.portalsdk.core.service.DataAccessService;
  28 import org.onap.portalsdk.core.util.SystemProperties;
  29 import org.onap.vid.asdc.AsdcCatalogException;
  30 import org.onap.vid.config.DataSourceConfig;
  31 import org.onap.vid.config.JobCommandsConfigWithMockedMso;
  32 import org.onap.vid.config.MockedAaiClientAndFeatureManagerConfig;
  33 import org.onap.vid.job.Job;
  34 import org.onap.vid.job.Job.JobStatus;
  35 import org.onap.vid.job.JobType;
  36 import org.onap.vid.job.JobsBrokerService;
  37 import org.onap.vid.job.command.CommandUtils;
  38 import org.onap.vid.job.command.InternalState;
  39 import org.onap.vid.model.Action;
  40 import org.onap.vid.model.NameCounter;
  41 import org.onap.vid.model.RequestReferencesContainer;
  42 import org.onap.vid.model.ServiceInfo;
  43 import org.onap.vid.model.serviceInstantiation.ServiceInstantiation;
  44 import org.onap.vid.mso.RestMsoImplementation;
  45 import org.onap.vid.mso.RestObject;
  46 import org.onap.vid.mso.model.RequestReferences;
  47 import org.onap.vid.mso.rest.AsyncRequestStatus;
  48 import org.onap.vid.properties.Features;
  49 import org.onap.vid.services.AsyncInstantiationBaseTest;
  50 import org.onap.vid.services.AsyncInstantiationBusinessLogic;
  51 import org.onap.vid.utils.DaoUtils;
  52 import org.springframework.test.context.ContextConfiguration;
  53 import org.testng.annotations.BeforeClass;
  54 import org.testng.annotations.BeforeMethod;
  55 import org.testng.annotations.DataProvider;
  56 import org.togglz.core.manager.FeatureManager;
  57
  58 import javax.inject.Inject;
  59 import javax.ws.rs.ProcessingException;
  60 import java.lang.reflect.Method;
  61 import java.util.*;
  62 import java.util.function.BiConsumer;
  63 import java.util.function.Supplier;
  64 import java.util.stream.Collectors;
  65 import java.util.stream.IntStream;
  66 import java.util.stream.Stream;
  67
  68 import static java.util.stream.Collectors.*;
  69 import static net.javacrumbs.jsonunit.JsonMatchers.jsonPartEquals;
  70 import static org.hamcrest.CoreMatchers.*;
  71 import static org.hamcrest.MatcherAssert.assertThat;
  72 import static org.hamcrest.Matchers.hasProperty;
  73 import static org.hamcrest.Matchers.hasSize;
  74 import static org.hamcrest.core.Every.everyItem;
  75 import static org.mockito.ArgumentMatchers.any;
  76 import static org.mockito.ArgumentMatchers.endsWith;
  77 import static org.mockito.ArgumentMatchers.*;
  78 import static org.mockito.Mockito.*;
  79 import static org.onap.vid.job.Job.JobStatus.*;
  80 import static org.onap.vid.model.JobAuditStatus.SourceStatus.MSO;
  81 import static org.onap.vid.model.JobAuditStatus.SourceStatus.VID;
  82 import static org.testng.AssertJUnit.*;
  83
  84 //it's more like integration test than UT
  85 //But it's very hard to test in API test so I use UT
  86 @ContextConfiguration(classes = {DataSourceConfig.class, SystemProperties.class, MockedAaiClientAndFeatureManagerConfig.class, JobCommandsConfigWithMockedMso.class})
  87 public class AsyncInstantiationIntegrationTest extends AsyncInstantiationBaseTest {
  88
  89     private static final String FAILED_STR = "FAILED";
  90     private static final String COMPLETE_STR = "COMPLETE";
  91     private static final String IN_PROGRESS_STR = "IN_PROGRESS";
  92     private static final String REQUESTED = "REQUESTED";
  93     private static final String PENDING_MANUAL_TASK = "PENDING_MANUAL_TASK";
  94     public static final String RAW_DATA_FROM_MSO = "RAW DATA FROM MSO";
  95     private static String USER_ID =  "123";
  96     public static String REQUEST_ID = UUID.randomUUID().toString();
  97     public static String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
  98
  99     @Inject
 100     private JobsBrokerService jobsBrokerService;
 101
 102     @Inject
 103     private JobWorker jobWorker;
 104
 105     @Inject
 106     private FeatureManager featureManager;
 107
 108     @Inject
 109     private AsyncInstantiationBusinessLogic asyncInstantiationBL;
 110
 111     @Inject
 112     private RestMsoImplementation restMso;
 113
 114     @Inject
 115     private DataAccessService dataAccessService;
 116
 117     @Inject
 118     private CommandUtils commandUtils;
 119
 120     @BeforeClass
 121     void initServicesInfoService() {
 122         createInstanceParamsMaps();
 123     }
 124
 125     @BeforeMethod
 126     void clearDb() {
 127         dataAccessService.deleteDomainObjects(ServiceInfo.class, "1=1", DaoUtils.getPropsMap());
 128         dataAccessService.deleteDomainObjects(JobDaoImpl.class, "1=1", DaoUtils.getPropsMap());
 129         dataAccessService.deleteDomainObjects(NameCounter.class, "1=1", DaoUtils.getPropsMap());
 130     }
 131
 132     @BeforeMethod
 133     void defineMocks() {
 134         mockAaiClientAnyNameFree();
 135     }
 136
 137     //@Test
 138     public void whenPushNewBulk_thenAllServicesAreInPending() {
 139
 140         pushMacroBulk();
 141         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 142         assertThat( serviceInfoList, everyItem(hasProperty("jobStatus", is(PENDING))));
 143     }
 144
 145     private List<UUID> pushMacroBulk() {
 146         ServiceInstantiation serviceInstantiation = generateMockMacroServiceInstantiationPayload(false,
 147                 createVnfList(instanceParamsMapWithoutParams, Collections.EMPTY_LIST, true),
 148                 3, true,PROJECT_NAME, true);
 149         return asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 150     }
 151
 152     private UUID pushALaCarteWithVnf() {
 153         ServiceInstantiation serviceInstantiation = generateALaCarteWithVnfsServiceInstantiationPayload();
 154         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 155         assertThat(uuids, hasSize(1));
 156         return uuids.get(0);
 157     }
 158
 159     private UUID pushALaCarteUpdateWithGroups() {
 160         ServiceInstantiation serviceInstantiation = generateALaCarteUpdateWith1ExistingGroup2NewGroupsPayload();
 161         List<UUID> uuids = asyncInstantiationBL.pushBulkJob(serviceInstantiation, USER_ID);
 162         assertThat(uuids, hasSize(1));
 163         return uuids.get(0);
 164     }
 165
 166     public static RestObject<RequestReferencesContainer> createResponse(int statusCode) {
 167         return createResponse(statusCode, SERVICE_INSTANCE_ID, REQUEST_ID);
 168     }
 169
 170     public static RestObject<RequestReferencesContainer> createResponse(int statusCode, String instanceId, String requestId) {
 171         RequestReferences requestReferences = new RequestReferences();
 172         requestReferences.setRequestId(requestId);
 173         requestReferences.setInstanceId(instanceId);
 174         RestObject<RequestReferencesContainer> restObject = new RestObject<>();
 175         restObject.set(new RequestReferencesContainer(requestReferences));
 176         restObject.setStatusCode(statusCode);
 177         restObject.setRaw(RAW_DATA_FROM_MSO);
 178         return restObject;
 179     }
 180
 181     ImmutableList<String> statusesToStrings(JobStatus... jobStatuses) {
 182         return Stream.of(jobStatuses).map(
 183                 Enum::toString).collect(ImmutableList.toImmutableList());
 184     }
 185
 186     /*
 187     Make sure service state is in progress once request has sent to MSO
 188     Make sure service state is in progress once request has sent to MSO and MSO status is in_progress
 189     Make sure service state is Failed once we got from MSO failure state, and that job's are not collected any more.
 190     Make sure service state is Completed successfully once we got from MSO complete, and that next job is peeked.
 191     Once a service in the bulk is failed, other services moved to Stopped, and no other jobs from the bulk are peeked.
 192     */
 193     //@Test
 194     public void testStatusesOfMacroServiceInBulkDuringBulkLifeCycle() {
 195         when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200));
 196         ImmutableList<ImmutableList<String>> expectedStatusesForVid = ImmutableList.of(
 197                 statusesToStrings(PENDING, IN_PROGRESS, COMPLETED),
 198                 statusesToStrings(PENDING, IN_PROGRESS, FAILED),
 199                 statusesToStrings(PENDING, STOPPED)
 200         );
 201
 202         ImmutableList<ImmutableList<String>> expectedStatusesForMso = ImmutableList.of(
 203                 ImmutableList.of(REQUESTED, IN_PROGRESS_STR, "not a state", FAILED_STR ,COMPLETE_STR),
 204                 ImmutableList.of(REQUESTED, FAILED_STR),
 205                 ImmutableList.of()
 206         );
 207
 208         List<UUID> uuids = pushMacroBulk();
 209         pullPendingJobAndAssertJobStatus(JobStatus.IN_PROGRESS, PENDING);
 210
 211         //assert that when get ProcessingException from restMso, status remain the same
 212         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenThrow(new ProcessingException("fake message"));
 213         Job job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 214         UUID firstHandledJobUUID = job.getUuid();
 215         listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job);
 216
 217         //assert that when get IN_PROGRESS status from restMso, status remain IN_PROGRESS
 218         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 219         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 220         listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job);
 221
 222         //assert that when get unrecognized status from restMso, status remain IN_PROGRESS
 223         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject("not a state"));
 224         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 225         listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job);
 226
 227         //assert that when get non 200 status code during IN_PROGRESS, status remain IN_PROGRESS
 228         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR, 404));
 229         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 230         listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job);
 231
 232         //when get job COMPLETE from MSO, service status become COMPLETED
 233         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 234                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 235         job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 236         List<ServiceInfo> serviceInfoList = listServicesAndAssertStatus(COMPLETED, PENDING, job);
 237
 238
 239         //for use later in the test
 240         Map<UUID, JobStatus> expectedJobStatusMap = serviceInfoList.stream().collect(
 241                 Collectors.toMap(ServiceInfo::getJobId, x-> PENDING));
 242         expectedJobStatusMap.put(job.getUuid(), COMPLETED);
 243
 244         //when handling another PENDING job, statuses are : COMPLETED, IN_PROGRESS, PENDING
 245         job =  pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS);
 246         assertThat(job.getUuid(), not(equalTo(firstHandledJobUUID))); //assert different job was handled now
 247         expectedJobStatusMap.put(job.getUuid(), JobStatus.IN_PROGRESS);
 248         listServicesAndAssertStatus(expectedJobStatusMap);
 249
 250         //when get FAILED status from MSO statuses are : COMPLETED, FAILED, STOPPED
 251         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 252                 thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 253         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED);
 254         expectedJobStatusMap.put(job.getUuid(), JobStatus.FAILED);
 255         expectedJobStatusMap = expectedJobStatusMap.entrySet().stream().collect(Collectors.toMap(
 256                 e -> e.getKey(), e -> e.getValue() == PENDING ? JobStatus.STOPPED : e.getValue()
 257         ));
 258
 259         listServicesAndAssertStatus(expectedJobStatusMap);
 260         IntStream.range(0, uuids.size()).forEach(i -> {
 261             UUID uuid = uuids.get(i);
 262             List<String> msoStatuses = asyncInstantiationBL.getAuditStatuses(uuid, MSO).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 263             List<String> vidStatuses = asyncInstantiationBL.getAuditStatuses(uuid, VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 264             assertThat(msoStatuses, is(expectedStatusesForMso.get(i)));
 265             assertThat(vidStatuses, is(expectedStatusesForVid.get(i)));
 266         });
 267         //
 268         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 269         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 270     }
 271
 272
 273     @DataProvider
 274     public static Object[][] AlaCarteStatuses(Method test) {
 275         return new Object[][]{
 276                 {COMPLETE_STR, JobStatus.COMPLETED, JobStatus.COMPLETED},
 277                 {FAILED_STR, JobStatus.COMPLETED_WITH_ERRORS, JobStatus.FAILED},
 278         };
 279     }
 280
 281     /*
 282     Make sure service state is in progress once request has sent to MSO
 283     Make sure service state is watching until state changes to complemented
 284     Make sure service state is watching until vnf state changes to completed
 285     Make sure service state is Completed successfully once we got from MSO complete for the vnf job.
 286     status Creating
 287      */
 288     //@Test(dataProvider = "AlaCarteStatuses")
 289     public void testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules(String msoVnfStatus, JobStatus expectedServiceStatus,  JobStatus expectedVnfStatus) {
 290         /*
 291             [v]  + push alacarte with 1 vnf
 292             [v]    verify STATUS pending
 293             [v]  + pull+execute  (should post to MSO)
 294             [v]    verify STATUS in progress
 295             [v]  + pull+execute  (should GET completed from MSO)
 296             [v]    verify STATUS in progress; TYPE watching
 297             [v]    verify job#2 *new* VNF job STATUS creating
 298             [v]  + pull+execute job#2 (should post to MSO)
 299             [v]    verify job#2 STATUS resource in progress
 300             [v]    verify job#1 STATUS in progress
 301             [v]  + pull+execute job#2 (should GET completed from MSO)
 302             [v]    verify job#2 STATUS completed
 303             [v]  + pull+execute job#1
 304             [v]    verify job#1 STATUS completed
 305
 306            * not looking on audit (yet)
 307         */
 308         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 309         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(false);
 310         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 311         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 312         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 313
 314
 315         //push alacarte with 1 vnf, verify STATUS pending
 316         UUID uuid = pushALaCarteWithVnf();
 317         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 318
 319         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 320         when(restMso.PostForObject(any(), endsWith("serviceInstances"), eq(RequestReferencesContainer.class))).thenReturn(
 321                 createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 322         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.PENDING, JobStatus.IN_PROGRESS, JobType.InProgressStatus);
 323         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 324
 325         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 326         reset(restMso);
 327         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 328                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 329         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching);
 330         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 331
 332         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 333         reset(restMso);
 334         when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), eq(RequestReferencesContainer.class))).thenReturn(
 335                 createResponse(200, UUID.randomUUID().toString(), VNF_REQUEST_ID));
 336         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.VnfInProgressStatus);
 337
 338         //verify service job  STATUS in progress
 339         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching);
 340         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 341
 342         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 343         //job status shall be final (COMPLETE/COMPLETE_WITH_ERRORS)
 344         reset(restMso);
 345         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 346                 asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 347         pullJobProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, expectedVnfStatus, false);
 348         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 349         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, expectedServiceStatus, true);
 350         singleServicesAndAssertStatus(expectedServiceStatus, uuid);
 351
 352     }
 353
 354     /*
 355     this test is almost duplication of testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules.
 356
 357     IgnoringVfModules test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is off
 358     WithVfModules     test check the scenario while FLAG_ASYNC_ALACARTE_VFMODULE is on
 359
 360     We shall consider later to remove testStatusesOfServiceDuringALaCarteLifeCycleIgnoringVfModules
 361     And union these tests to single one.
 362      */
 363
 364     //@Test
 365     public void testALaCarteLifeCycle1Vnf2VfModules() {
 366
 367
 368         String msoVnfStatus = COMPLETE_STR;
 369         JobStatus expectedServiceStatus = IN_PROGRESS;
 370         JobStatus expectedVnfStatus = RESOURCE_IN_PROGRESS;
 371         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VNF)).thenReturn(true);
 372         when(featureManager.isActive(Features.FLAG_ASYNC_ALACARTE_VFMODULE)).thenReturn(true);
 373         final String SERVICE_REQUEST_ID = UUID.randomUUID().toString();
 374         final String SERVICE_INSTANCE_ID = UUID.randomUUID().toString();
 375         final String VNF_REQUEST_ID = UUID.randomUUID().toString();
 376         final String VNF_INSTANCE_ID = UUID.randomUUID().toString();
 377         final String VG_REQUEST_ID = UUID.randomUUID().toString();
 378         final String VG_INSTANCE_ID = UUID.randomUUID().toString();
 379         final String VF_MODULE_REQUEST_ID = UUID.randomUUID().toString();
 380         final String VF_MODULE_REQUEST_ID2 = UUID.randomUUID().toString();
 381
 382
 383         //push alacarte with 1 vnf, verify STATUS pending
 384         UUID uuid = pushALaCarteWithVnf();
 385         singleServicesAndAssertStatus(JobStatus.PENDING, uuid);
 386
 387         /*---------- service -----------*/
 388
 389         //mock mso to answer 200 of create service instance request, verify STATUS in progress
 390         when(restMso.PostForObject(any(), endsWith("serviceInstances"), eq(RequestReferencesContainer.class))).thenReturn(
 391                 createResponse(200, SERVICE_INSTANCE_ID, SERVICE_REQUEST_ID));
 392         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.PENDING, JobStatus.IN_PROGRESS, JobType.InProgressStatus);
 393         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 394
 395         //mock mso to answer COMPLETE for service instance create, job status shall remain IN_PROGRESS and type shall be Watching
 396         reset(restMso);
 397         when(restMso.GetForObject(endsWith(SERVICE_REQUEST_ID), eq(AsyncRequestStatus.class))).
 398                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 399         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching);
 400         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 401
 402         /*---------- vnf -----------*/
 403
 404         //mock mso to answer 200 of create vnf instance request, pull+execute vnf job, STATUS resource in progress
 405         reset(restMso);
 406         when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs"), eq(RequestReferencesContainer.class))).thenReturn(
 407                 createResponse(200, VNF_INSTANCE_ID, VNF_REQUEST_ID));
 408         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.VnfInProgressStatus);
 409
 410         //verify service job  STATUS in progress
 411         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, JobType.Watching);
 412         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 413
 414         //mock mso to answer msoVnfStatus (COMPLETE/FAILED) for vnf creation status,
 415         //job status shall be final (COMPLETE/COMPLETE_WITH_ERRORS)
 416         reset(restMso);
 417         when(restMso.GetForObject(endsWith(VNF_REQUEST_ID), eq(AsyncRequestStatus.class))).
 418                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 419
 420         try {
 421             reset(commandUtils);
 422             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_0_MODEL_VERSION_ID)).thenReturn(true);
 423             when(commandUtils.isVfModuleBaseModule(SERVICE_MODEL_VERSION_ID, VF_MODULE_1_MODEL_VERSION_ID)).thenReturn(false);
 424         } catch (AsdcCatalogException e) {
 425
 426         }
 427
 428         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.RESOURCE_IN_PROGRESS, JobStatus.RESOURCE_IN_PROGRESS, JobType.WatchingBaseModule);
 429         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 430         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, true);
 431         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 432
 433         /*---------- vf Module without volume group name (base) -----------*/
 434
 435         //vg name not exist, so vf module created immediately
 436         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching);
 437
 438         //verify vnf/volumeGroup job  STATUS still watching with resource in progress
 439         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching);
 440
 441         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 442         reset(restMso);
 443         when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), eq(RequestReferencesContainer.class))).thenReturn(
 444                 createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID));
 445         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus);
 446
 447         //mock mso to answer for vf module orchestration request
 448         reset(restMso);
 449         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 450                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 451         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus, JobStatus.COMPLETED, JobType.ResourceInProgressStatus);
 452
 453         //verify volume group become completed
 454         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching, JobStatus.COMPLETED, JobType.Watching);
 455
 456         //vnf become watching after volume group completed, and new volume group created
 457         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.WatchingBaseModule, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching);
 458
 459         /*---------- volume group & vf module (non base) -----------*/
 460
 461         /*---------- volume group -----------*/
 462
 463         //mock mso to answer 200 of create volumeGroup instance request, pull+execute volumeGroup job, STATUS resource in progress
 464         reset(restMso);
 465         when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/volumeGroups"), eq(RequestReferencesContainer.class))).thenReturn(
 466                 createResponse(200, VG_INSTANCE_ID, VG_REQUEST_ID));
 467         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.VolumeGroupInProgressStatus);
 468
 469         //verify vnf job  STATUS still watching with resource in progress
 470         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching);
 471
 472         //mock mso to answer for volume group orchestration request
 473         reset(restMso);
 474         when(restMso.GetForObject(endsWith(VG_REQUEST_ID), eq(AsyncRequestStatus.class))).thenReturn(
 475                 asyncRequestStatusResponseAsRestObject(msoVnfStatus));
 476         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.VolumeGroupInProgressStatus, JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching);
 477
 478         /*---------- vfModule -----------*/
 479
 480         //mock mso to answer 200 of create vfModule instance request, pull+execute volumeGroup job, STATUS resource in progress
 481         reset(restMso);
 482         when(restMso.PostForObject(any(), endsWith(SERVICE_INSTANCE_ID + "/vnfs/" + VNF_INSTANCE_ID + "/vfModules"), eq(RequestReferencesContainer.class))).thenReturn(
 483                 createResponse(200, UUID.randomUUID().toString(), VF_MODULE_REQUEST_ID2));
 484         pullJobProcessAndPushBackWithTypeAssertion(JobStatus.CREATING, JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus);
 485
 486         //mock mso to answer for vf module orchestration request
 487         reset(restMso);
 488         when(restMso.GetForObject(endsWith(VF_MODULE_REQUEST_ID2), eq(AsyncRequestStatus.class))).thenReturn(
 489                 asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 490         pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus.RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus, JobStatus.COMPLETED, JobType.ResourceInProgressStatus);
 491
 492         //execute twice - 1 for parent volume group, 1 for parent vnf
 493         pullAllJobProcessAndPushBackByType(JobStatus.RESOURCE_IN_PROGRESS, JobType.Watching , JobStatus.COMPLETED);
 494
 495         singleServicesAndAssertStatus(JobStatus.IN_PROGRESS, uuid);
 496         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.COMPLETED, true);
 497         singleServicesAndAssertStatus(JobStatus.COMPLETED, uuid);
 498     }
 499
 500     //@Test
 501     public void testBadAaiResponseForSearchNamesAndBackToNormal() {
 502         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenThrow(aaiNodeQueryBadResponseException());
 503         pushMacroBulk();        //JOB shall become IN_PROGRESS but service info is still pending
 504         Job job = pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, true);
 505         listServicesAndAssertStatus(PENDING, PENDING, job);
 506
 507         //JOB shall remain in IN_PROGRESS
 508         job = pullJobProcessAndPushBack( JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, true);
 509         //make sure the job command is still ServiceInstantiation
 510         assertThat(job.getType(), is(JobType.MacroServiceInstantiation));
 511         listServicesAndAssertStatus(PENDING, PENDING, job);
 512
 513         //simulate AAI back to normal, AAI return name is free, and MSO return good response
 514         Mockito.reset(aaiClient); // must forget the "thenThrow"
 515         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(false);
 516         when(restMso.PostForObject(any(),any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200));
 517         job = pullJobProcessAndPushBack( JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS, true);
 518         listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job);
 519
 520         //when get job COMPLETE from MSO, service status become COMPLETED
 521         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 522                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 523         job = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 524         listServicesAndAssertStatus(COMPLETED, PENDING, job);
 525     }
 526
 527     //@Test
 528     public void testAaiResponseNameUsedTillMaxRetries() {
 529         when(aaiClient.isNodeTypeExistsByName(any(), any())).thenReturn(true);
 530         asyncInstantiationBL.setMaxRetriesGettingFreeNameFromAai(10);
 531         pushMacroBulk();
 532         //JOB shall become IN_PROGRESS but service info is still pending
 533         Job job = pullJobProcessAndPushBack(PENDING, JobStatus.FAILED, true);
 534         listServicesAndAssertStatus(JobStatus.FAILED, JobStatus.STOPPED, job);
 535     }
 536
 537     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus) {
 538         return pullJobProcessAndPushBack(topic, expectedNextJobStatus, true);
 539     }
 540
 541     //return the pulled job (and not the pushed job)
 542     private Job pullJobProcessAndPushBack(JobStatus topic, JobStatus expectedNextJobStatus, boolean pullingAssertion) {
 543         Optional<Job> job = pullJob(topic, pullingAssertion);
 544
 545         Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 546
 547         try {
 548             assertThat("next job not ok: " + nextJob.getData(), nextJob.getStatus(), is(expectedNextJobStatus));
 549
 550             if (pullingAssertion) {
 551                 //assert another pulling on same topic return no result (before push back)
 552                 assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 553             }
 554
 555         } finally {
 556             jobsBrokerService.pushBack(nextJob); // push back to let retries - even if any assertion failure
 557         }
 558         assertThat(jobsBrokerService.peek(job.get().getUuid()).getStatus(), is(expectedNextJobStatus));
 559         return job.get();
 560     }
 561
 562     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 563                                                            JobType expectedNextJobType) {
 564         Job job = pullJobProcessAndPushBack(topic, expectedNextJobStatus, false);
 565         assertThat("job not ok: " + job.getData(), job.getType(), is(expectedNextJobType));
 566         return job;
 567     }
 568
 569     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 570                                                            JobType expectedNextJobType, int retries) {
 571         return retryWithAssertionsLimit(retries, () -> {
 572             return pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 573         });
 574     }
 575
 576     private Job pullJobProcessAndPushBackWithTypeAssertion(JobStatus topic, JobStatus expectedNextJobStatus,
 577                                                            JobType expectedNextJobType, Action actionPhase, InternalState internalState, int retries) {
 578         return retryWithAssertionsLimit(retries, () -> {
 579             Job job = pullJobProcessAndPushBackWithTypeAssertion(topic, expectedNextJobStatus, expectedNextJobType);
 580             assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("actionPhase", actionPhase.name())));
 581             if (internalState != null) {
 582                 assertThat("job not ok: " + job.getData(), job.getData(), is(jsonPartEquals("internalState", internalState.name())));
 583             }
 584             return job;
 585         });
 586     }
 587
 588     private Job retryWithAssertionsLimit(int retries, Supplier<Job> supplier) {
 589         java.util.Stack<AssertionError> history = new Stack<>();
 590
 591         do {
 592             try {
 593                 return supplier.get();
 594             } catch (AssertionError assertionError) {
 595                 history.push(assertionError);
 596             }
 597         } while (history.size() < retries);
 598
 599         // No success:
 600         throw new AssertionError("No luck while all of these assertion errors: " + history.stream()
 601                 .map(Throwable::getMessage)
 602                 .map(s -> s.replace('\n', ' '))
 603                 .map(s -> s.replaceAll("\\s{2,}"," "))
 604                 .distinct()
 605                 .collect(joining("\n   ", "\n   ", "")), history.peek());
 606     }
 607
 608     private Job pullMultipleJobsFindExpectedProcessAndPushBack(JobStatus topic, JobType expectedCurrentJobType, JobStatus expectedNextJobStatus,
 609                                                                JobType expectedNextJobType) {
 610         List<Job> pulledJobs = new ArrayList<>();
 611         Job lastJob = null;
 612         while (lastJob == null || lastJob.getType() != expectedCurrentJobType) {
 613             lastJob = pullJob(topic, false).get();
 614             if (lastJob.getType() != expectedCurrentJobType) {
 615                 pulledJobs.add(lastJob);
 616             }
 617         }
 618
 619         Job nextJob = jobWorker.executeJobAndGetNext(lastJob);
 620         assertThat(nextJob.getStatus(), is(expectedNextJobStatus));
 621         assertThat(nextJob.getType(), is(expectedNextJobType));
 622
 623         jobsBrokerService.pushBack(nextJob);
 624         assertThat(jobsBrokerService.peek(nextJob.getUuid()).getStatus(), is(expectedNextJobStatus));
 625
 626         pulledJobs.forEach(job ->
 627                 jobsBrokerService.pushBack(job)
 628         );
 629
 630         return nextJob;
 631     }
 632
 633     private void pullAllJobProcessAndPushBackByType(JobStatus topic, JobType commandType, JobStatus expectedFinalStatus) {
 634         Map<UUID, JobStatus> jobStatusMap = new HashMap<>();
 635         Optional<Job> job = pullJob(topic, false);
 636         for (int i=0; i<1000 && job.isPresent() && job.get().getType() == commandType; i++) {
 637             Job nextJob = jobWorker.executeJobAndGetNext(job.get());
 638             jobStatusMap.put(nextJob.getUuid(), nextJob.getStatus());
 639             jobsBrokerService.pushBack(nextJob);
 640             job = jobsBrokerService.pull(topic, UUID.randomUUID().toString());
 641         }
 642         assertThat(jobStatusMap.values(), everyItem(is(expectedFinalStatus)));
 643
 644     }
 645
 646     private Optional<Job> pullJob(JobStatus topic, boolean pullingAssertion) {
 647         if (pullingAssertion) {
 648             //assert pulling on inverse topic return no result
 649             assertFalse(jobsBrokerService.pull(inverseTopic(topic), randomUuid()).isPresent());
 650         }
 651
 652         Optional<Job> job =  jobsBrokerService.pull(topic, randomUuid());
 653         assertTrue("no job fetched", job.isPresent());
 654
 655         if (pullingAssertion) {
 656             //assert another pulling on same topic return no result
 657             assertFalse(jobsBrokerService.pull(topic, randomUuid()).isPresent());
 658         }
 659
 660         return job;
 661     }
 662
 663     private JobStatus inverseTopic(JobStatus topic) {
 664         return topic==JobStatus.IN_PROGRESS ? PENDING : JobStatus.IN_PROGRESS;
 665     }
 666
 667
 668     //@Test
 669     public void whenPushNewBulk_andGetNoResponseFromMsoOnCreation_thenServiceMoveToFailedAndOtherToStopped() {
 670         when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(500));
 671         pushBulkPullPendingJobAndAssertJobStatus(JobStatus.FAILED, JobStatus.STOPPED);
 672     }
 673
 674     //@Test
 675     public void whenMsoStatusIsPendingManualTask_ThenJobStatusIsPaused() {
 676         when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200));
 677
 678         Job firstJob = pushBulkPullPendingJobAndAssertJobStatus(JobStatus.IN_PROGRESS, PENDING);
 679
 680         //assert that when get ProcessingException from restMso, status remain the same
 681         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 682                 thenReturn(asyncRequestStatusResponseAsRestObject(PENDING_MANUAL_TASK));
 683         Job job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 684         listServicesAndAssertStatus(PAUSE, PENDING, job);
 685
 686         //The paused job is pulled and remain in pause state. Other jobs from bulk remain pending
 687         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 688         listServicesAndAssertStatus(PAUSE, PENDING, job);
 689
 690         //the job get IN_PROGRESS response (simulate activate operation) and status changed to IN_PROGRESS
 691         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 692                 thenReturn(asyncRequestStatusResponseAsRestObject(IN_PROGRESS_STR));
 693         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.IN_PROGRESS);
 694         listServicesAndAssertStatus(JobStatus.IN_PROGRESS, PENDING, job);
 695
 696         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 697                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 698         job =  pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED);
 699         listServicesAndAssertStatus(COMPLETED, PENDING, job);
 700
 701         //Pulling PENDING job return another job
 702         assertThat(jobsBrokerService.pull(PENDING, randomUuid()).get().getUuid(), not(equalTo(job.getUuid())));
 703
 704
 705         ImmutableList<String> expectedStatusesForMso = ImmutableList.of(REQUESTED, PENDING_MANUAL_TASK, IN_PROGRESS_STR, COMPLETE_STR);
 706         List<String> msoStatuses = asyncInstantiationBL.getAuditStatuses(firstJob.getUuid(), MSO).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 707         assertThat(msoStatuses, is(expectedStatusesForMso));
 708
 709         ImmutableList<String> expectedStatusesForVid = statusesToStrings(PENDING, IN_PROGRESS, PAUSE, IN_PROGRESS, COMPLETED);
 710         List<String> vidStatuses = asyncInstantiationBL.getAuditStatuses(firstJob.getUuid(), VID).stream().map(x -> x.getJobStatus()).collect(Collectors.toList());
 711         assertThat(vidStatuses, is(expectedStatusesForVid));
 712     }
 713
 714     private Job pushBulkPullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 715         pushMacroBulk();
 716         return pullPendingJobAndAssertJobStatus(pulledJobStatus, otherJobsStatus);
 717     }
 718
 719     private Job pullPendingJobAndAssertJobStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus) {
 720         Job job = pullJobProcessAndPushBack(PENDING, pulledJobStatus, false);
 721         listServicesAndAssertStatus(pulledJobStatus, otherJobsStatus, job);
 722         return job;
 723     }
 724
 725     //@Test
 726     public void test2BulksLifeCyclesAreIndependent() {
 727         pushMacroBulk();
 728         when(restMso.PostForObject(any(), any(), eq(RequestReferencesContainer.class))).thenReturn(createResponse(200));
 729         //push 2nd job, then when pulling first job the job become in_progress, other jobs (from 2 bulks) remain pending
 730         Job firstJob = pushBulkPullPendingJobAndAssertJobStatus(JobStatus.IN_PROGRESS, PENDING);
 731
 732         //assert we can pull another job from pending from other template id
 733         Job secondJob = pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, false);
 734         assertThat(firstJob.getTemplateId(), not(equalTo(secondJob.getTemplateId())));
 735
 736         //assert no more PENDING jobs to pull
 737         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 738
 739         //when get FAILED status from MSO statuses for failed bulk are: FAILED, STOPPED, for other bulk: IN_PROGRESS, 2 pending
 740         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 741                 thenReturn(asyncRequestStatusResponseAsRestObject(FAILED_STR));
 742         Job failedJob = pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, JobStatus.FAILED, false);
 743         Map<UUID, List<ServiceInfo>> servicesByTemplateId =
 744                 asyncInstantiationBL.getAllServicesInfo()
 745                         .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 746         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob);
 747         Job successJob = failedJob.getUuid().equals(firstJob.getUuid()) ? secondJob : firstJob;
 748         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), JobStatus.IN_PROGRESS, PENDING, successJob);
 749
 750         //yet no more PENDING jobs to pull
 751         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 752
 753         //assert that job from non failed bulk can progress.
 754         //When completed,  failed bulk statuses: FAILED, STOPPED. Succeeded bulk statuses are : COMPLETED, 2 pending
 755         when(restMso.GetForObject(any(), eq(AsyncRequestStatus.class))).
 756                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 757         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED, false);
 758         servicesByTemplateId =
 759                 asyncInstantiationBL.getAllServicesInfo()
 760                         .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 761         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob);
 762         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, PENDING, successJob);
 763
 764         //advance other jobs of succeeded bulk till al of them reach to COMPLETED
 765         pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, false);
 766         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED, false);
 767         pullJobProcessAndPushBack(PENDING, JobStatus.IN_PROGRESS, false);
 768         pullJobProcessAndPushBack(JobStatus.IN_PROGRESS, COMPLETED, false);
 769         servicesByTemplateId =
 770                 asyncInstantiationBL.getAllServicesInfo()
 771                         .stream().collect(groupingBy(ServiceInfo::getTemplateId));
 772         assertServicesStatus(servicesByTemplateId.get(failedJob.getTemplateId()), JobStatus.FAILED, JobStatus.STOPPED, failedJob);
 773         assertServicesStatus(servicesByTemplateId.get(successJob.getTemplateId()), COMPLETED, COMPLETED, successJob);
 774
 775         //assert no more PENDING jobs nor IN_PROGRESS jobs to pull
 776         assertFalse(jobsBrokerService.pull(PENDING, randomUuid()).isPresent());
 777         assertFalse(jobsBrokerService.pull(JobStatus.IN_PROGRESS, randomUuid()).isPresent());
 778     }
 779
 780     public void deploy2NewGroupsToServiceWith1ExistingGroup() {
 781
 782         /*
 783         new feature: skip service (existing impl) and skip group (new impl)
 784         service+group aren't touched, 2 groups ARE created
 785
 786         [v]  success if all GROUPs success
 787
 788         Next test variation should:
 789         [ ]  error if all GROUPs error
 790         [ ]  completed with error if 1 GROUP error
 791
 792
 793         [v]  + service with 3 groups - 1 action=none, 2 action=create; service's action=none
 794         [v]    verify STATUS pending
 795         [v]  + pull+execute  (should NOT post to MSO)
 796         [v]    verify STATUS in progress; TYPE watching
 797                ...
 798         [v]    verify job#2 *new* GROUP job STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE delete
 799         [v]    verify job#3 *new* GROUP job STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE delete
 800         [v]    verify job#4 *new* GROUP job STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE delete
 801
 802         [v]  + pull+execute job#1 (should NOT post to MSO)
 803         [v]    verify STATUS in progress; TYPE watching
 804         [v]    verify job#5 *new* GROUP job STATUS creating TYPE group INTERNAL STATE initial PHASE create
 805         [v]    verify job#6 *new* GROUP job STATUS creating TYPE group INTERNAL STATE initial PHASE create
 806         [v]    verify job#7 *new* GROUP job STATUS creating TYPE group INTERNAL STATE initial PHASE create
 807
 808         [v]  + pull+execute job#5 (should NOT post to MSO)
 809         [v]    verify job#5 STATUS completed with no action TYPE group INTERNAL STATE terminal PHASE create
 810         [v]  + pull+execute job#1
 811         [v]    verify job#1 STATUS in progress; TYPE watching
 812
 813         [v]  + pull+execute job#6 (should post to MSO)
 814         [v]    verify job#6 STATUS resource in progress
 815         [v]  + pull+execute job#1
 816         [v]    verify job#1 STATUS in progress; TYPE watching
 817         [v]  + pull+execute job#6 (should get from MSO)
 818         [v]    verify job#6 STATUS completed
 819         [v]  + pull+execute job#1
 820         [v]    verify job#1 STATUS in progress; TYPE watching
 821
 822         [v]  + pull+execute job#7 (should post to MSO)
 823         [v]    verify job#7 STATUS resource in progress
 824         [v]  + pull+execute job#1
 825         [v]    verify job#1 STATUS in progress; TYPE watching
 826         [v]  + pull+execute job#7 (should get from MSO)
 827         [v]    verify job#7 STATUS completed
 828         [v]  + pull+execute job#1
 829         [v]    verify job#1 STATUS completed
 830
 831          */
 832
 833         final String GROUP1_REQUEST_ID = UUID.randomUUID().toString();
 834         final String GROUP1_INSTANCE_ID = UUID.randomUUID().toString();
 835         final String GROUP2_REQUEST_ID = UUID.randomUUID().toString();
 836         final String GROUP2_INSTANCE_ID = UUID.randomUUID().toString();
 837
 838         // Utility method
 839         final BiConsumer<Action, JobStatus> verify_Job1InProgress = (phase, nextJobStatus) -> {
 840             pullJobProcessAndPushBackWithTypeAssertion(IN_PROGRESS, nextJobStatus, JobType.ALaCarteService, phase, InternalState.WATCHING, 2);
 841         };
 842
 843         //service with 3 groups - 1 action=none, 2 action=create; service's action=none
 844         UUID uuid = pushALaCarteUpdateWithGroups();
 845         singleServicesAndAssertStatus(PENDING, uuid);
 846
 847         // take from pending, put in-progress -> 3 delete-child were born
 848         pullJobProcessAndPushBackWithTypeAssertion(PENDING, IN_PROGRESS, JobType.ALaCarteService, Action.Delete, InternalState.WATCHING, 1);
 849         verifyQueueSizes(ImmutableMap.of(
 850                 IN_PROGRESS, 1, CREATING, 3
 851         ));
 852
 853         Stream.of(1, 2, 3).forEach(i -> {
 854             // take each child creating, put in-progress
 855             verify_Job1InProgress.accept(Action.Delete, IN_PROGRESS);
 856             pullJobProcessAndPushBackWithTypeAssertion(CREATING, RESOURCE_IN_PROGRESS, JobType.InstanceGroup, Action.Delete, null, 1);
 857
 858             // execute each in-progress -> job is completed
 859             verify_Job1InProgress.accept(Action.Delete, IN_PROGRESS);
 860             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED/*_WITH_NO_ACTION*/, JobType.InstanceGroup,1);
 861         });
 862         verifyQueueSizes(ImmutableMap.of(
 863                 IN_PROGRESS, 1, COMPLETED, 3
 864         ));
 865
 866         // take job #1 from phase delete to phase create -> 3 create-child were born
 867         verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 868         verifyQueueSizes(ImmutableMap.of(
 869                 IN_PROGRESS, 1, CREATING, 3, COMPLETED, 3
 870         ));
 871
 872         // prepare MSO mock
 873         when(restMso.PostForObject(any(), endsWith("instanceGroups"), eq(RequestReferencesContainer.class)))
 874                 .thenReturn(createResponse(200, GROUP1_INSTANCE_ID, GROUP1_REQUEST_ID))
 875                 .thenReturn(createResponse(200, GROUP2_INSTANCE_ID, GROUP2_REQUEST_ID))
 876                 .thenReturn(null);
 877         when(restMso.GetForObject(argThat(uri -> StringUtils.endsWithAny(uri, GROUP1_REQUEST_ID, GROUP2_REQUEST_ID)), eq(AsyncRequestStatus.class))).
 878                 thenReturn(asyncRequestStatusResponseAsRestObject(COMPLETE_STR));
 879
 880         // take first "none" child from creating to completed
 881         // note there's no concrete mechanism that makes the first child be
 882         // the "action=None" case, but that's what happens, and following line
 883         // relies on that fact.
 884         pullJobProcessAndPushBackWithTypeAssertion(CREATING, COMPLETED_WITH_NO_ACTION, JobType.InstanceGroupInstantiation, 1);
 885
 886         // take each of next two children from creating to in-progress, then to completed
 887         // verify job #1 is watching, and MSO is getting requests
 888         Stream.of(1, 2).forEach(i -> {
 889             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 890             pullJobProcessAndPushBackWithTypeAssertion(CREATING, RESOURCE_IN_PROGRESS, JobType.ResourceInProgressStatus);
 891             verify(restMso, times(i)).PostForObject(any(), any(), any());
 892
 893             verify_Job1InProgress.accept(Action.Create, IN_PROGRESS);
 894             pullJobProcessAndPushBackWithTypeAssertion(RESOURCE_IN_PROGRESS, COMPLETED, JobType.ResourceInProgressStatus);
 895             verify(restMso, times(i)).GetForObject(any(), any());
 896         });
 897
 898         // job #1 is done as all children are done
 899         verify_Job1InProgress.accept(Action.Create, COMPLETED);
 900         verifyQueueSizes(ImmutableMap.of(COMPLETED, 7));
 901     }
 902
 903     private void verifyQueueSizes(ImmutableMap<JobStatus, Integer> expected) {
 904         final Collection<Job> peek = jobsBrokerService.peek();
 905         final Map<JobStatus, Long> jobTypes = peek.stream().collect(groupingBy(Job::getStatus, counting()));
 906         assertThat(jobTypes, is(expected));
 907     }
 908
 909     private List<ServiceInfo> listServicesAndAssertStatus(JobStatus pulledJobStatus, JobStatus otherJobsStatus, Job job) {
 910         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 911         assertServicesStatus(serviceInfoList, pulledJobStatus, otherJobsStatus, job);
 912
 913         return serviceInfoList;
 914     }
 915
 916     private ServiceInfo singleServicesAndAssertStatus(JobStatus expectedStatus, UUID jobUUID) {
 917         List<ServiceInfo> serviceInfoList = asyncInstantiationBL.getAllServicesInfo();
 918         assertEquals(1, serviceInfoList.size());
 919         ServiceInfo serviceInfo = serviceInfoList.get(0);
 920         assertThat(serviceInfo.getJobStatus(), is(expectedStatus));
 921         assertThat(serviceInfo.getJobId(), is(jobUUID));
 922         return serviceInfo;
 923     }
 924
 925     private void assertServicesStatus(List<ServiceInfo> serviceInfoList, JobStatus pulledJobStatus, JobStatus otherJobsStatus, Job job) {
 926         serviceInfoList.forEach(si->{
 927             if (si.getJobId().equals(job.getUuid())) {
 928                 assertThat(si.getJobStatus(), is(pulledJobStatus));
 929             }
 930             else {
 931                 assertThat(si.getJobStatus(), is(otherJobsStatus));
 932             }
 933         });
 934     }
 935
 936     private void listServicesAndAssertStatus(Map<UUID, JobStatus> expectedJobStatusMap) {
 937         Map<UUID, JobStatus> actualStatuses = asyncInstantiationBL.getAllServicesInfo()
 938                 .stream().collect(Collectors.toMap(ServiceInfo::getJobId, ServiceInfo::getJobStatus));
 939         assertThat(actualStatuses.entrySet(), equalTo(expectedJobStatusMap.entrySet()));
 940     }
 941
 942     private String randomUuid() {
 943         return UUID.randomUUID().toString();
 944     }
 945 }