vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java

   1 package org.onap.vid.api;
   2
   3 import static java.lang.Boolean.FALSE;
   4 import static java.lang.Boolean.TRUE;
   5 import static java.util.Collections.emptyList;
   6 import static java.util.stream.Collectors.joining;
   7 import static java.util.stream.Collectors.toMap;
   8 import static org.hamcrest.CoreMatchers.hasItem;
   9 import static org.hamcrest.MatcherAssert.assertThat;
  10 import static org.hamcrest.Matchers.containsInAnyOrder;
  11 import static org.hamcrest.Matchers.hasSize;
  12 import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
  13 import static org.testng.Assert.assertNotNull;
  14 import static org.testng.AssertJUnit.assertEquals;
  15 import static org.testng.AssertJUnit.assertTrue;
  16 import static vid.automation.test.utils.ExtendedHamcrestMatcher.hasItemsFromCollection;
  17
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.Uninterruptibles;
  21 import java.time.Instant;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.UUID;
  26 import java.util.concurrent.TimeUnit;
  27 import java.util.concurrent.atomic.AtomicReference;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import java.util.stream.IntStream;
  31 import java.util.stream.Stream;
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
  34 import org.apache.commons.lang3.builder.ToStringStyle;
  35 import org.hamcrest.BaseMatcher;
  36 import org.hamcrest.CoreMatchers;
  37 import org.hamcrest.Description;
  38 import org.hamcrest.MatcherAssert;
  39 import org.onap.simulator.presetGenerator.presets.BasePresets.BaseMSOPreset;
  40 import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
  41 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetCloudOwnersByCloudRegionId;
  42 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
  43 import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
  44 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseCreateInstancePost;
  45 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseDelete;
  46 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
  47 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
  48 import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
  49 import org.onap.vid.model.asyncInstantiation.ServiceInfo;
  50 import org.onap.vid.model.mso.MsoResponseWrapper2;
  51 import org.springframework.core.ParameterizedTypeReference;
  52 import org.springframework.http.HttpMethod;
  53 import org.springframework.http.HttpStatus;
  54 import org.springframework.http.ResponseEntity;
  55 import org.springframework.web.client.RestTemplate;
  56 import org.testng.Assert;
  57 import org.testng.annotations.AfterMethod;
  58 import org.testng.annotations.BeforeClass;
  59 import org.testng.annotations.DataProvider;
  60 import vid.automation.test.infra.Features;
  61 import vid.automation.test.infra.Wait;
  62 import vid.automation.test.model.JobStatus;
  63 import vid.automation.test.model.ServiceAction;
  64 import vid.automation.test.services.AsyncJobsService;
  65 import vid.automation.test.services.SimulatorApi;
  66
  67 public class AsyncInstantiationBase extends BaseMsoApiTest {
  68
  69     public static final String CREATE_BULK_OF_ALACARTE_REQUEST_WITH_VNF = "asyncInstantiation/vidRequestCreateALaCarteWithVnf.json";
  70     protected static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
  71
  72     protected static final String MSO_BASE_ERROR =
  73             "Received error from SDN-C: java.lang.IllegalArgumentException: All keys must be specified for class org."+
  74             "opendaylight.yang.gen.v1.org.onap.sdnc.northbound.generic.resource.rev170824.vf.module.assignments.vf."+
  75             "module.assignments.vms.VmKey. Missing key is getVmType. Supplied key is VmKey [].";
  76     protected static final String MSO_ERROR = MSO_BASE_ERROR + StringUtils.repeat(" and a lot of sentences for long message", 60);
  77
  78     @BeforeClass
  79     protected void muteAndDropNameCounter() {
  80         AsyncJobsService asyncJobsService = new AsyncJobsService();
  81         asyncJobsService.muteAllAsyncJobs();
  82         asyncJobsService.dropAllFromNameCounter();
  83     }
  84
  85     @AfterMethod
  86     protected void muteAllAsyncJobs() {
  87         AsyncJobsService asyncJobsService = new AsyncJobsService();
  88         asyncJobsService.muteAllAsyncJobs();
  89     }
  90
  91     @DataProvider
  92     public static Object[][] trueAndFalse() {
  93             return new Object[][]{{TRUE},{FALSE}};
  94     }
  95
  96     protected String getCreateBulkUri() {
  97         return uri.toASCIIString() + "/asyncInstantiation/bulk";
  98     }
  99
 100     protected String getHideServiceUri(String jobId) {
 101         return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
 102     }
 103
 104     protected String getServiceInfoUrl() {
 105         return uri.toASCIIString() + "/asyncInstantiation";
 106     }
 107
 108     protected String getJobAuditUrl() {
 109         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
 110     }
 111
 112     protected String getMsoJobAuditUrl() {
 113         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}/mso";
 114     }
 115
 116     protected String getDeleteServiceUrl(String uuid) {
 117         return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
 118     }
 119
 120     protected String getInstanceAuditInfoUrl() {
 121         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{TYPE}/{INSTANCE_ID}/mso";
 122     }
 123
 124     protected String getRetryJobUrl() {
 125         return uri.toASCIIString() + "/asyncInstantiation/retry/{JOB_ID}";
 126     }
 127     protected String getTopologyForRetryUrl() {
 128         return uri.toASCIIString() + "/asyncInstantiation/bulkForRetry/{JOB_ID}";
 129     }
 130
 131
 132     protected String getRetryJobWithChangedDataUrl() {
 133         return uri.toASCIIString() + "/asyncInstantiation/retryJobWithChangedData/{JOB_ID}";
 134     }
 135
 136     protected boolean getExpectedRetryEnabled(JobStatus jobStatus) {
 137         return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS);
 138     }
 139
 140     public List<BasePreset> getPresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 141
 142         final ImmutableList.Builder<BasePreset> basePresetBuilder = new ImmutableList.Builder<>();
 143         basePresetBuilder
 144                 .add(new PresetGetSessionSlotCheckIntervalGet())
 145                 .add(new PresetAAIGetSubscribersGet())
 146                 .addAll(presetOnDeleteList)
 147                 .addAll(presetOnCreateList)
 148                 .addAll(presetInProgressList);
 149         return basePresetBuilder.build();
 150     }
 151
 152     public List<BasePreset> getDeletePresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 153         return getPresets(presetOnDeleteList, emptyList(), presetInProgressList);
 154     }
 155
 156     public List<BasePreset> getPresets(List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 157         return getPresets(emptyList(), presetOnCreateList, presetInProgressList);
 158     }
 159
 160     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName) {
 161         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, userName, null, ServiceAction.INSTANTIATE);
 162     }
 163
 164     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName, String instanceId, ServiceAction action) {
 165         assertExpectedStatusAndServiceInfo(jobStatus, jobId, new ServiceInfo(
 166                 userName, jobStatus, false,
 167                 "038d99af-0427-42c2-9d15-971b99b9b489", "Lucine Sarika", "zasaki",
 168                 "de738e5f-3704-4a14-b98f-3bf86ac0c0a0", "voloyakane-senamo",
 169                 "c85f0e80-0636-44a4-8cb2-4ec00d056e79", "Hedvika Wendelin",
 170                 "a93f8383-707e-43fa-8191-a6e69a1aab17", null,
 171                 "TYLER SILVIA", "SILVIA ROBBINS",
 172                 instanceId, serviceInstanceName,
 173                 "e3c34d88-a216-4f1d-a782-9af9f9588705", "gayawabawe", "5.1",
 174                 jobId, null, action, false)
 175         );
 176     }
 177
 178     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName) {
 179         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, "us16807000");
 180     }
 181
 182     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses) {
 183         assertAuditStatuses(jobId, expectedVidStatuses, expectedMsoStatuses, 15);
 184     }
 185
 186     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses, long timeoutInSeconds) {
 187         assertAndRetryIfNeeded(() -> {
 188             final List<JobAuditStatus> auditVidStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
 189             assertThat(auditVidStatuses, hasItemsFromCollection(expectedVidStatuses));
 190             if (expectedMsoStatuses!=null) {
 191                 final List<JobAuditStatus> auditMsoStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
 192                 assertThat(auditMsoStatuses, containsInAnyOrder(expectedMsoStatuses.toArray()));
 193             }
 194         }, timeoutInSeconds);
 195     }
 196
 197     protected void assertAndRetryIfNeeded(Runnable asserter, long timeoutInSeconds) {
 198         final Instant expiry = Instant.now().plusSeconds(timeoutInSeconds);
 199         while (true) {
 200             try {
 201                 asserter.run();
 202                 break; // we're cool, assertion passed
 203             } catch (AssertionError fail) {
 204                 Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
 205                 if (Instant.now().isAfter(expiry)) {
 206                     throw fail;
 207                 } else {
 208                     System.out.println("retrying after: " + fail);
 209                 }
 210             }
 211         }
 212     }
 213
 214     protected ImmutableList<JobAuditStatus> vidAuditStatusesCompletedWithErrors(String jobId) {
 215         return ImmutableList.of(
 216                 vidAuditStatus(jobId, "PENDING", false),
 217                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 218                 vidAuditStatus(jobId, "COMPLETED_WITH_ERRORS", true)
 219         );
 220     }
 221
 222     protected ImmutableList<JobAuditStatus> vidAuditStatusesFailed(String jobId) {
 223         return ImmutableList.of(
 224                 vidAuditStatus(jobId, "PENDING", false),
 225                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 226                 vidAuditStatus(jobId, "FAILED", true)
 227         );
 228     }
 229
 230     protected JobAuditStatus vidAuditStatus(String jobId, String jobStatus, boolean isFinal) {
 231         return new JobAuditStatus(UUID.fromString(jobId), jobStatus, JobAuditStatus.SourceStatus.VID, null, null, isFinal);
 232     }
 233
 234     public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
 235         protected String expectedJobId;
 236
 237         public JobIdAndStatusMatcher(String expectedJobId) {
 238             this.expectedJobId = expectedJobId;
 239         }
 240
 241         @Override
 242         public boolean matches(Object item) {
 243             if (!(item instanceof ServiceInfo)) {
 244                 return false;
 245             }
 246             ServiceInfo serviceInfo = (ServiceInfo) item;
 247             return expectedJobId.equals(serviceInfo.jobId);
 248         }
 249
 250         @Override
 251         public void describeTo(Description description) {
 252             description.appendText("failed to find job with uuid ")
 253                     .appendValue(expectedJobId);
 254         }
 255     }
 256
 257
 258
 259     protected Map<Keys,String> generateNames() {
 260         return Stream.of(Keys.values()).collect(
 261                 Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
 262     }
 263
 264     protected ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
 265         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 266         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 267                 .add(new PresetGetSessionSlotCheckIntervalGet())
 268                 .add(new PresetAAIGetSubscribersGet())
 269                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 270                 .addAll(msoBulkPresets)
 271                 .add(new PresetMSOOrchestrationRequestGet())
 272                 .build();
 273         return presets;
 274
 275     }
 276
 277     protected ImmutableList<BasePreset> generateMsoCreateBulkPresets(int bulkSize, Map<Keys, String> names) {
 278         return IntStream.rangeClosed(0, bulkSize-1).
 279                 mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
 280                 .collect(ImmutableList.toImmutableList());
 281     }
 282
 283     protected ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
 284         return restTemplate.exchange(
 285                 url,
 286                 org.springframework.http.HttpMethod.GET,
 287                 null,
 288                 new ParameterizedTypeReference<List<JobAuditStatus>>() {});
 289     }
 290
 291     @DataProvider
 292     public static Object[][] auditSources() {
 293         return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
 294     }
 295
 296
 297
 298     protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
 299         Map<Keys, String> names = generateNames();
 300         ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
 301         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 302         Assert.assertEquals(jobIds.size(),bulkSize);
 303
 304         assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
 305                 jobIds.stream().collect(joining(","))),
 306
 307                 Wait.waitFor(y-> serviceListCall().getBody().stream()
 308                         .filter(si -> jobIds.contains(si.jobId))
 309                         .filter(si -> si.jobStatus==JobStatus.COMPLETED)
 310                         .count() == bulkSize,
 311                 null, 30, 1 ));
 312         return jobIds;
 313     }
 314
 315     protected List<JobAuditStatus> getJobMsoAuditStatusForAlaCarte(String jobUUID, String requestId, String serviceInstanceId){
 316         String url = getMsoJobAuditUrl().replace("{JOB_ID}",jobUUID);
 317
 318         if(!StringUtils.isEmpty(requestId)) {
 319             url = url + "?requestId=" + requestId;
 320             if(!StringUtils.isEmpty(serviceInstanceId)) {
 321                 url = url + "&serviceInstanceId=" + serviceInstanceId;
 322             }
 323         }
 324         return callAuditStatus(url);
 325     }
 326
 327     protected List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
 328         String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
 329         return callAuditStatus(url);
 330     }
 331
 332     protected List<JobAuditStatus> getAuditStatusesForInstance(String type, String instanceId){
 333         String url = getInstanceAuditInfoUrl().replace("{TYPE}",type).replace("{INSTANCE_ID}", instanceId);
 334         return callAuditStatus(url);
 335     }
 336
 337     private List<JobAuditStatus> callAuditStatus(String url) {
 338         ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
 339         assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 340         return statusesResponse.getBody();
 341     }
 342
 343     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
 344         return addBulkAllPendingButOneInProgress(3);
 345     }
 346
 347     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
 348         Map<Keys, String> names = generateNames();
 349         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 350         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 351                 .add(new PresetGetSessionSlotCheckIntervalGet())
 352                 .add(new PresetAAIGetSubscribersGet())
 353                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 354                 .addAll(msoBulkPresets)
 355                 .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
 356                 .build();
 357         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 358
 359         // wait for single IN_PROGRESS, so statuses will stop from changing
 360         Wait.waitFor(foo -> serviceListCall().getBody().stream()
 361                         .filter(si -> jobIds.contains(si.jobId))
 362                         .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
 363                 null, 20, 1);
 364
 365         final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
 366                 .filter(si -> jobIds.contains(si.jobId))
 367                 .collect(toMap(si -> si.jobId, si -> si.jobStatus));
 368
 369         assertThat(jobIds, hasSize(bulkSize));
 370
 371
 372         return statusMapBefore;
 373     }
 374
 375     protected String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
 376         final String jobToDelete = jobIdToStatus.entrySet().stream()
 377                 .filter(entry -> entry.getValue().equals(jobStatus))
 378                 .map(Map.Entry::getKey)
 379                 .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
 380
 381
 382         restTemplate.delete(getDeleteServiceUrl(jobToDelete));
 383
 384         return jobToDelete;
 385     }
 386
 387
 388     protected MsoResponseWrapper2 hideService(String jobId) {
 389         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
 390         return responseWrapper2;
 391     }
 392
 393     protected List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
 394         List<String> jobIds = createBulkOfMacroInstances(presets, isPause, bulkSize, names);
 395         Assert.assertEquals(jobIds.size(), bulkSize);
 396         for(String jobId: jobIds) {
 397             assertExpectedStatusAndServiceInfo(isPause, finalState, names, jobId);
 398         }
 399
 400         return jobIds;
 401     }
 402
 403     protected void assertExpectedStatusAndServiceInfo(boolean isPause, JobStatus finalState, Map<Keys, String> names, String jobId) {
 404         assertExpectedStatusAndServiceInfo(finalState, jobId, new ServiceInfo("us16807000", JobStatus.IN_PROGRESS, isPause, "someID",
 405                 "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "hvf3", null,
 406                 "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
 407                 "5c9e863f-2716-467b-8799-4a67f378dcaa", "AIM_TRANSPORT_00004", "1.0", jobId, null, ServiceAction.INSTANTIATE, false));
 408     }
 409
 410     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, ServiceInfo expectedServiceInfo) {
 411         assertExpectedStatusAndServiceInfo(finalState, jobId, false, expectedServiceInfo);
 412     }
 413
 414     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, boolean longWait, ServiceInfo expectedServiceInfo) {
 415         JobInfoChecker<Integer> jobInfoChecker = new JobInfoChecker<>(
 416                 restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
 417         boolean result = jobInfoChecker.test(null);
 418         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 419
 420         jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
 421         if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState) && expectedServiceInfo.serviceInstanceId==null) {
 422             expectedServiceInfo.serviceInstanceId = BaseMSOPreset.DEFAULT_INSTANCE_ID;
 423         }
 424         result = Wait.waitFor(jobInfoChecker, null, 30, longWait ? 2 : 1);
 425         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 426     }
 427
 428     protected List<String> createBulkOfMacroInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names) {
 429         SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
 430         return createBulkOfInstances(isPause, bulkSize, names, CREATE_BULK_OF_MACRO_REQUEST);
 431     }
 432
 433     public List<String> createBulkOfInstances(boolean isPause, int bulkSize, Map<Keys, String> names, String requestDetailsFileName){
 434
 435         String requestBody = TestUtils.convertRequest(objectMapper, requestDetailsFileName);
 436         requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
 437         for (Map.Entry<Keys, String> e : names.entrySet()) {
 438             requestBody = requestBody.replace(e.getKey().name(), e.getValue());
 439         }
 440         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
 441         assertNotNull(responseWrapper2);
 442         return (List<String>)responseWrapper2.getEntity();
 443     }
 444
 445     protected List<String> retryJob(String jobId) {
 446         ResponseEntity<String> retryBulkPayload = getRetryBulk(jobId);
 447         return retryJobWithChangedData(jobId, retryBulkPayload.getBody());
 448     }
 449
 450     protected List<String> retryJobWithChangedData(String jobId, String requestBody) {
 451         String retryUri = getRetryJobWithChangedDataUrl();
 452         retryUri = retryUri.replace("{JOB_ID}", jobId);
 453         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(HttpMethod.POST, retryUri, requestBody);
 454         assertNotNull(responseWrapper2);
 455         return (List<String>)responseWrapper2.getEntity();
 456     }
 457
 458     protected ResponseEntity<String> getRetryBulk(String jobId) {
 459         String retryUri = getTopologyForRetryUrl();
 460         retryUri = retryUri.replace("{JOB_ID}", jobId);
 461         return restTemplateErrorAgnostic.getForEntity(retryUri, String.class);
 462     }
 463
 464     protected Object getResourceAuditInfo(String trackById) {
 465         return restTemplate.getForObject(buildUri("/asyncInstantiation/auditStatusForRetry/{trackById}"), Object.class, trackById);
 466     }
 467
 468     public class JobInfoChecker<Integer> implements Predicate<Integer> {
 469
 470         protected final RestTemplate restTemplate;
 471         protected Set<JobStatus> expectedJobStatus;
 472         protected ServiceInfo expectedServiceInfo;
 473         protected final String jobId;
 474         protected JobStatus lastStatus;
 475
 476         public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
 477             this.restTemplate = restTemplate;
 478             this.expectedJobStatus = expectedJobStatus;
 479             this.jobId = jobId;
 480             this.expectedServiceInfo = expectedServiceInfo;
 481         }
 482
 483         public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
 484             this.expectedJobStatus = expectedJobStatus;
 485         }
 486
 487         @Override
 488         public boolean test(Integer integer) {
 489             ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
 490             assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 491             assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
 492             ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
 493                     .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
 494                     .findFirst().orElse(null);
 495             Assert.assertNotNull(serviceInfoFromDB);
 496             Assert.assertEquals(serviceInfoDataReflected(expectedServiceInfo), serviceInfoDataReflected(serviceInfoFromDB));
 497             assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
 498                     serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
 499
 500             if (expectedServiceInfo.serviceInstanceId != null && ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE, JobStatus.COMPLETED_WITH_ERRORS).contains(serviceInfoFromDB.jobStatus)) {
 501                 MatcherAssert.assertThat("service instance id is wrong", serviceInfoFromDB.serviceInstanceId, CoreMatchers.is(expectedServiceInfo.serviceInstanceId));
 502             }
 503             if (expectedJobStatus.size()==1) {
 504                 assertEquals("job status is wrong", getExpectedRetryEnabled((JobStatus)(expectedJobStatus.toArray()[0])), serviceInfoFromDB.isRetryEnabled);
 505             }
 506             lastStatus = serviceInfoFromDB.jobStatus;
 507             return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
 508         }
 509     }
 510
 511     protected ResponseEntity<List<ServiceInfo>> serviceListCall() {
 512         return restTemplate.exchange(
 513                 getServiceInfoUrl(),
 514                 org.springframework.http.HttpMethod.GET,
 515                 null,
 516                 new ParameterizedTypeReference<List<ServiceInfo>>() {});
 517     }
 518
 519     //serialize fields except of fields we cannot know ahead of time
 520     protected static String serviceInfoDataReflected(ServiceInfo service1) {
 521         return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
 522                 .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName", "isRetryEnabled")
 523                 .toString();
 524     }
 525
 526     protected void addBulkPendingWithCustomList(List<BasePreset> customPresets){
 527         Map<Keys, String> names = generateNames();
 528         final int bulkSize = 2 + customPresets.size();
 529
 530         List<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 531         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 532                 .add(new PresetGetSessionSlotCheckIntervalGet())
 533                 .add(new PresetAAIGetSubscribersGet())
 534                 .addAll(msoBulkPresets)
 535                 .addAll(customPresets)
 536                 .build();
 537
 538         List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 539         Assert.assertEquals(jobIds.size(),bulkSize);
 540     }
 541
 542     protected void verifyAuditStatuses(String jobId, List<String> statuses, JobAuditStatus.SourceStatus source) {
 543         int statusesSize = statuses.size();
 544         AtomicReference<List<JobAuditStatus>> actualAudits = new AtomicReference<>();
 545         if (source.equals(JobAuditStatus.SourceStatus.VID)) {
 546             actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name()));
 547             org.junit.Assert.assertEquals("Received number of VID statuses is not as expected", statusesSize, actualAudits.get().size());
 548         } else {
 549             boolean isStatusedSizeAsExpected = Wait.waitFor(x-> {
 550                 actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()));
 551                 return actualAudits.get().size() == statusesSize;
 552             },null,5,1);
 553             org.junit.Assert.assertTrue("Received number of MSO statuses is not as expected. Expected: " + statusesSize + ". Received: " + actualAudits.get().size(), isStatusedSizeAsExpected);
 554         }
 555         IntStream.range(0, statusesSize).forEach(i-> org.junit.Assert.assertEquals(source + " status #" + i + " is not as expected", statuses.get(i), actualAudits.get().get(i).getJobStatus()));
 556     }
 557
 558     protected void verifyInstanceAuditStatuses(List<JobAuditStatus> expectedStatuses, List<JobAuditStatus> actualStatuses) {
 559         final int expectedSize = expectedStatuses.size();
 560         assertTrue("Expected statuses size is "+ expectedSize +", actual size is "+actualStatuses.size(), new Integer(expectedSize).equals(actualStatuses.size()));
 561         IntStream.range(0, expectedSize).forEach(i-> {
 562
 563             final JobAuditStatus expectedStatus = expectedStatuses.get(i);
 564             final JobAuditStatus actualStatus = actualStatuses.get(i);
 565             org.junit.Assert.assertEquals("MSO status #" + i + " is not as expected", expectedStatus.getJobStatus(), actualStatus.getJobStatus());
 566             org.junit.Assert.assertEquals("MSO requestId #" + i + " is not as expected", expectedStatus.getRequestId(), actualStatus.getRequestId());
 567             org.junit.Assert.assertEquals("MSO additionalInfo #" + i + " is not as expected", expectedStatus.getAdditionalInfo(), actualStatus.getAdditionalInfo());
 568             org.junit.Assert.assertEquals("MSO jobID #" + i + " is not as expected", expectedStatus.getJobId(), actualStatus.getJobId());
 569             org.junit.Assert.assertEquals("MSO instanceName #" + i + " is not as expected", expectedStatus.getInstanceName(), actualStatus.getInstanceName());
 570             org.junit.Assert.assertEquals("MSO instanceType  #" + i + " is not as expected", expectedStatus.getInstanceType(), actualStatus.getInstanceType());
 571         });
 572     }
 573 }