vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java

   1 package org.onap.vid.api;
   2
   3 import com.google.common.collect.ImmutableList;
   4 import com.google.common.collect.ImmutableSet;
   5 import com.google.common.util.concurrent.Uninterruptibles;
   6 import org.apache.commons.lang3.StringUtils;
   7 import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
   8 import org.apache.commons.lang3.builder.ToStringStyle;
   9 import org.hamcrest.BaseMatcher;
  10 import org.hamcrest.CoreMatchers;
  11 import org.hamcrest.Description;
  12 import org.hamcrest.MatcherAssert;
  13 import org.onap.simulator.presetGenerator.presets.BasePresets.BaseMSOPreset;
  14 import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
  15 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetCloudOwnersByCloudRegionId;
  16 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
  17 import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
  18 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseCreateInstancePost;
  19 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseDelete;
  20 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
  21 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
  22 import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
  23 import org.onap.vid.model.asyncInstantiation.ServiceInfo;
  24 import org.onap.vid.model.mso.MsoResponseWrapper2;
  25 import org.springframework.core.ParameterizedTypeReference;
  26 import org.springframework.http.HttpMethod;
  27 import org.springframework.http.HttpStatus;
  28 import org.springframework.http.ResponseEntity;
  29 import org.springframework.web.client.RestTemplate;
  30 import org.testng.Assert;
  31 import org.testng.annotations.AfterMethod;
  32 import org.testng.annotations.BeforeClass;
  33 import org.testng.annotations.DataProvider;
  34 import vid.automation.test.infra.Features;
  35 import vid.automation.test.infra.Wait;
  36 import vid.automation.test.model.JobStatus;
  37 import vid.automation.test.model.ServiceAction;
  38 import vid.automation.test.services.AsyncJobsService;
  39 import vid.automation.test.services.SimulatorApi;
  40
  41 import java.time.Instant;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Set;
  45 import java.util.UUID;
  46 import java.util.concurrent.TimeUnit;
  47 import java.util.concurrent.atomic.AtomicReference;
  48 import java.util.function.Predicate;
  49 import java.util.stream.Collectors;
  50 import java.util.stream.IntStream;
  51 import java.util.stream.Stream;
  52
  53 import static java.lang.Boolean.FALSE;
  54 import static java.lang.Boolean.TRUE;
  55 import static java.util.Collections.emptyList;
  56 import static java.util.stream.Collectors.joining;
  57 import static java.util.stream.Collectors.toMap;
  58 import static org.hamcrest.CoreMatchers.hasItem;
  59 import static org.hamcrest.MatcherAssert.assertThat;
  60 import static org.hamcrest.Matchers.containsInAnyOrder;
  61 import static org.hamcrest.Matchers.hasSize;
  62 import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
  63 import static org.testng.Assert.assertNotNull;
  64 import static org.testng.AssertJUnit.assertEquals;
  65 import static org.testng.AssertJUnit.assertTrue;
  66 import static vid.automation.test.utils.ExtendedHamcrestMatcher.hasItemsFromCollection;
  67
  68 public class AsyncInstantiationBase extends BaseMsoApiTest {
  69
  70     public static final String CREATE_BULK_OF_ALACARTE_REQUEST_WITH_VNF = "asyncInstantiation/vidRequestCreateALaCarteWithVnf.json";
  71     protected static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
  72
  73     protected static final String MSO_BASE_ERROR =
  74             "Received error from SDN-C: java.lang.IllegalArgumentException: All keys must be specified for class org."+
  75             "opendaylight.yang.gen.v1.org.onap.sdnc.northbound.generic.resource.rev170824.vf.module.assignments.vf."+
  76             "module.assignments.vms.VmKey. Missing key is getVmType. Supplied key is VmKey [].";
  77     protected static final String MSO_ERROR = MSO_BASE_ERROR + StringUtils.repeat(" and a lot of sentences for long message", 60);
  78
  79     @BeforeClass
  80     protected void muteAndDropNameCounter() {
  81         AsyncJobsService asyncJobsService = new AsyncJobsService();
  82         asyncJobsService.muteAllAsyncJobs();
  83         asyncJobsService.dropAllFromNameCounter();
  84     }
  85
  86     @AfterMethod
  87     protected void muteAllAsyncJobs() {
  88         AsyncJobsService asyncJobsService = new AsyncJobsService();
  89         asyncJobsService.muteAllAsyncJobs();
  90     }
  91
  92     @DataProvider
  93     public static Object[][] trueAndFalse() {
  94             return new Object[][]{{TRUE},{FALSE}};
  95     }
  96
  97     protected String getCreateBulkUri() {
  98         return uri.toASCIIString() + "/asyncInstantiation/bulk";
  99     }
 100
 101     protected String getHideServiceUri(String jobId) {
 102         return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
 103     }
 104
 105     protected String getServiceInfoUrl() {
 106         return uri.toASCIIString() + "/asyncInstantiation";
 107     }
 108
 109     protected String getJobAuditUrl() {
 110         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
 111     }
 112
 113     protected String getMsoJobAuditUrl() {
 114         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}/mso";
 115     }
 116
 117     protected String getDeleteServiceUrl(String uuid) {
 118         return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
 119     }
 120
 121     protected String getInstanceAuditInfoUrl() {
 122         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{TYPE}/{INSTANCE_ID}/mso";
 123     }
 124
 125     protected String getRetryJobUrl() {
 126         return uri.toASCIIString() + "/asyncInstantiation/retry/{JOB_ID}";
 127     }
 128     protected String getTopologyForRetryUrl() {
 129         return uri.toASCIIString() + "/asyncInstantiation/bulkForRetry/{JOB_ID}";
 130     }
 131
 132
 133     protected String getRetryJobWithChangedDataUrl() {
 134         return uri.toASCIIString() + "/asyncInstantiation/retryJobWithChangedData/{JOB_ID}";
 135     }
 136
 137     protected boolean getExpectedRetryEnabled(JobStatus jobStatus) {
 138         return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS);
 139     }
 140
 141     public List<BasePreset> getPresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 142
 143         final ImmutableList.Builder<BasePreset> basePresetBuilder = new ImmutableList.Builder<>();
 144         basePresetBuilder
 145                 .add(new PresetGetSessionSlotCheckIntervalGet())
 146                 .add(new PresetAAIGetSubscribersGet())
 147                 .addAll(presetOnDeleteList)
 148                 .addAll(presetOnCreateList)
 149                 .addAll(presetInProgressList);
 150         return basePresetBuilder.build();
 151     }
 152
 153     public List<BasePreset> getDeletePresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 154         return getPresets(presetOnDeleteList, emptyList(), presetInProgressList);
 155     }
 156
 157     public List<BasePreset> getPresets(List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 158         return getPresets(emptyList(), presetOnCreateList, presetInProgressList);
 159     }
 160
 161     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName) {
 162         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, userName, null, ServiceAction.INSTANTIATE);
 163     }
 164
 165     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName, String instanceId, ServiceAction action) {
 166         assertExpectedStatusAndServiceInfo(jobStatus, jobId, new ServiceInfo(
 167                 userName, jobStatus, false,
 168                 "038d99af-0427-42c2-9d15-971b99b9b489", "Lucine Sarika", "zasaki",
 169                 "de738e5f-3704-4a14-b98f-3bf86ac0c0a0", "voloyakane-senamo",
 170                 "c85f0e80-0636-44a4-8cb2-4ec00d056e79", "Hedvika Wendelin",
 171                 "a93f8383-707e-43fa-8191-a6e69a1aab17", null,
 172                 "TYLER SILVIA", "SILVIA ROBBINS",
 173                 instanceId, serviceInstanceName,
 174                 "e3c34d88-a216-4f1d-a782-9af9f9588705", "gayawabawe", "5.1",
 175                 jobId, null, action, false)
 176         );
 177     }
 178
 179     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName) {
 180         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, "us16807000");
 181     }
 182
 183     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses) {
 184         assertAuditStatuses(jobId, expectedVidStatuses, expectedMsoStatuses, 15);
 185     }
 186
 187     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses, long timeoutInSeconds) {
 188         assertAndRetryIfNeeded(() -> {
 189             final List<JobAuditStatus> auditVidStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
 190             assertThat(auditVidStatuses, hasItemsFromCollection(expectedVidStatuses));
 191             if (expectedMsoStatuses!=null) {
 192                 final List<JobAuditStatus> auditMsoStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
 193                 assertThat(auditMsoStatuses, containsInAnyOrder(expectedMsoStatuses.toArray()));
 194             }
 195         }, timeoutInSeconds);
 196     }
 197
 198     protected void assertAndRetryIfNeeded(Runnable asserter, long timeoutInSeconds) {
 199         final Instant expiry = Instant.now().plusSeconds(timeoutInSeconds);
 200         while (true) {
 201             try {
 202                 asserter.run();
 203                 break; // we're cool, assertion passed
 204             } catch (AssertionError fail) {
 205                 Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
 206                 if (Instant.now().isAfter(expiry)) {
 207                     throw fail;
 208                 } else {
 209                     System.out.println("retrying after: " + fail);
 210                 }
 211             }
 212         }
 213     }
 214
 215     protected ImmutableList<JobAuditStatus> vidAuditStatusesCompletedWithErrors(String jobId) {
 216         return ImmutableList.of(
 217                 vidAuditStatus(jobId, "PENDING", false),
 218                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 219                 vidAuditStatus(jobId, "COMPLETED_WITH_ERRORS", true)
 220         );
 221     }
 222
 223     protected ImmutableList<JobAuditStatus> vidAuditStatusesFailed(String jobId) {
 224         return ImmutableList.of(
 225                 vidAuditStatus(jobId, "PENDING", false),
 226                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 227                 vidAuditStatus(jobId, "FAILED", true)
 228         );
 229     }
 230
 231     protected JobAuditStatus vidAuditStatus(String jobId, String jobStatus, boolean isFinal) {
 232         return new JobAuditStatus(UUID.fromString(jobId), jobStatus, JobAuditStatus.SourceStatus.VID, null, null, isFinal);
 233     }
 234
 235     public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
 236         protected String expectedJobId;
 237
 238         public JobIdAndStatusMatcher(String expectedJobId) {
 239             this.expectedJobId = expectedJobId;
 240         }
 241
 242         @Override
 243         public boolean matches(Object item) {
 244             if (!(item instanceof ServiceInfo)) {
 245                 return false;
 246             }
 247             ServiceInfo serviceInfo = (ServiceInfo) item;
 248             return expectedJobId.equals(serviceInfo.jobId);
 249         }
 250
 251         @Override
 252         public void describeTo(Description description) {
 253             description.appendText("failed to find job with uuid ")
 254                     .appendValue(expectedJobId);
 255         }
 256     }
 257
 258
 259
 260     protected Map<Keys,String> generateNames() {
 261         return Stream.of(Keys.values()).collect(
 262                 Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
 263     }
 264
 265     protected ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
 266         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 267         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 268                 .add(new PresetGetSessionSlotCheckIntervalGet())
 269                 .add(new PresetAAIGetSubscribersGet())
 270                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 271                 .addAll(msoBulkPresets)
 272                 .add(new PresetMSOOrchestrationRequestGet())
 273                 .build();
 274         return presets;
 275
 276     }
 277
 278     protected ImmutableList<BasePreset> generateMsoCreateBulkPresets(int bulkSize, Map<Keys, String> names) {
 279         return IntStream.rangeClosed(0, bulkSize-1).
 280                 mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
 281                 .collect(ImmutableList.toImmutableList());
 282     }
 283
 284     protected ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
 285         return restTemplate.exchange(
 286                 url,
 287                 org.springframework.http.HttpMethod.GET,
 288                 null,
 289                 new ParameterizedTypeReference<List<JobAuditStatus>>() {});
 290     }
 291
 292     @DataProvider
 293     public static Object[][] auditSources() {
 294         return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
 295     }
 296
 297
 298
 299     protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
 300         Map<Keys, String> names = generateNames();
 301         ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
 302         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 303         Assert.assertEquals(jobIds.size(),bulkSize);
 304
 305         assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
 306                 jobIds.stream().collect(joining(","))),
 307
 308                 Wait.waitFor(y-> serviceListCall().getBody().stream()
 309                         .filter(si -> jobIds.contains(si.jobId))
 310                         .filter(si -> si.jobStatus==JobStatus.COMPLETED)
 311                         .count() == bulkSize,
 312                 null, 30, 1 ));
 313         return jobIds;
 314     }
 315
 316     protected List<JobAuditStatus> getJobMsoAuditStatusForAlaCarte(String jobUUID, String requestId, String serviceInstanceId){
 317         String url = getMsoJobAuditUrl().replace("{JOB_ID}",jobUUID);
 318
 319         if(!StringUtils.isEmpty(requestId)) {
 320             url = url + "?requestId=" + requestId;
 321             if(!StringUtils.isEmpty(serviceInstanceId)) {
 322                 url = url + "&serviceInstanceId=" + serviceInstanceId;
 323             }
 324         }
 325         return callAuditStatus(url);
 326     }
 327
 328     protected List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
 329         String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
 330         return callAuditStatus(url);
 331     }
 332
 333     protected List<JobAuditStatus> getAuditStatusesForInstance(String type, String instanceId){
 334         String url = getInstanceAuditInfoUrl().replace("{TYPE}",type).replace("{INSTANCE_ID}", instanceId);
 335         return callAuditStatus(url);
 336     }
 337
 338     private List<JobAuditStatus> callAuditStatus(String url) {
 339         ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
 340         assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 341         return statusesResponse.getBody();
 342     }
 343
 344     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
 345         return addBulkAllPendingButOneInProgress(3);
 346     }
 347
 348     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
 349         Map<Keys, String> names = generateNames();
 350         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 351         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 352                 .add(new PresetGetSessionSlotCheckIntervalGet())
 353                 .add(new PresetAAIGetSubscribersGet())
 354                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 355                 .addAll(msoBulkPresets)
 356                 .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
 357                 .build();
 358         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 359
 360         // wait for single IN_PROGRESS, so statuses will stop from changing
 361         Wait.waitFor(foo -> serviceListCall().getBody().stream()
 362                         .filter(si -> jobIds.contains(si.jobId))
 363                         .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
 364                 null, 20, 1);
 365
 366         final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
 367                 .filter(si -> jobIds.contains(si.jobId))
 368                 .collect(toMap(si -> si.jobId, si -> si.jobStatus));
 369
 370         assertThat(jobIds, hasSize(bulkSize));
 371
 372
 373         return statusMapBefore;
 374     }
 375
 376     protected String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
 377         final String jobToDelete = jobIdToStatus.entrySet().stream()
 378                 .filter(entry -> entry.getValue().equals(jobStatus))
 379                 .map(Map.Entry::getKey)
 380                 .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
 381
 382
 383         restTemplate.delete(getDeleteServiceUrl(jobToDelete));
 384
 385         return jobToDelete;
 386     }
 387
 388
 389     protected MsoResponseWrapper2 hideService(String jobId) {
 390         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
 391         return responseWrapper2;
 392     }
 393
 394     protected List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
 395         List<String> jobIds = createBulkOfMacroInstances(presets, isPause, bulkSize, names);
 396         Assert.assertEquals(jobIds.size(), bulkSize);
 397         for(String jobId: jobIds) {
 398             assertExpectedStatusAndServiceInfo(isPause, finalState, names, jobId);
 399         }
 400
 401         return jobIds;
 402     }
 403
 404     protected void assertExpectedStatusAndServiceInfo(boolean isPause, JobStatus finalState, Map<Keys, String> names, String jobId) {
 405         assertExpectedStatusAndServiceInfo(finalState, jobId, new ServiceInfo("us16807000", JobStatus.IN_PROGRESS, isPause, "someID",
 406                 "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "hvf3", null,
 407                 "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
 408                 "5c9e863f-2716-467b-8799-4a67f378dcaa", "AIM_TRANSPORT_00004", "1.0", jobId, null, ServiceAction.INSTANTIATE, false));
 409     }
 410
 411     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, ServiceInfo expectedServiceInfo) {
 412         assertExpectedStatusAndServiceInfo(finalState, jobId, PATIENCE_LEVEL.FAIL_FAST, expectedServiceInfo);
 413     }
 414
 415     enum PATIENCE_LEVEL { FAIL_FAST, FAIL_SLOW, FAIL_VERY_SLOW }
 416
 417     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, PATIENCE_LEVEL patienceLevel, ServiceInfo expectedServiceInfo) {
 418         JobInfoChecker<Integer> jobInfoChecker = new JobInfoChecker<>(
 419                 restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
 420         boolean result = jobInfoChecker.test(null);
 421         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 422
 423         jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
 424         if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState) && expectedServiceInfo.serviceInstanceId==null) {
 425             expectedServiceInfo.serviceInstanceId = BaseMSOPreset.DEFAULT_INSTANCE_ID;
 426         }
 427         result = Wait.waitFor(jobInfoChecker, null, 30, waitIntervalBy(patienceLevel));
 428         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 429     }
 430
 431     private int waitIntervalBy(PATIENCE_LEVEL patienceLevel) {
 432         switch (patienceLevel) {
 433             case FAIL_SLOW:
 434                 return 2;
 435             case FAIL_VERY_SLOW:
 436                 return 3;
 437             default:
 438                 return 1;
 439         }
 440     }
 441
 442     protected List<String> createBulkOfMacroInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names) {
 443         SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
 444         return createBulkOfInstances(isPause, bulkSize, names, CREATE_BULK_OF_MACRO_REQUEST);
 445     }
 446
 447     public List<String> createBulkOfInstances(boolean isPause, int bulkSize, Map<Keys, String> names, String requestDetailsFileName){
 448
 449         String requestBody = TestUtils.convertRequest(objectMapper, requestDetailsFileName);
 450         requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
 451         for (Map.Entry<Keys, String> e : names.entrySet()) {
 452             requestBody = requestBody.replace(e.getKey().name(), e.getValue());
 453         }
 454         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
 455         assertNotNull(responseWrapper2);
 456         return (List<String>)responseWrapper2.getEntity();
 457     }
 458
 459     protected List<String> retryJob(String jobId) {
 460         ResponseEntity<String> retryBulkPayload = getRetryBulk(jobId);
 461         return retryJobWithChangedData(jobId, retryBulkPayload.getBody());
 462     }
 463
 464     protected List<String> retryJobWithChangedData(String jobId, String requestBody) {
 465         String retryUri = getRetryJobWithChangedDataUrl();
 466         retryUri = retryUri.replace("{JOB_ID}", jobId);
 467         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(HttpMethod.POST, retryUri, requestBody);
 468         assertNotNull(responseWrapper2);
 469         return (List<String>)responseWrapper2.getEntity();
 470     }
 471
 472     protected ResponseEntity<String> getRetryBulk(String jobId) {
 473         String retryUri = getTopologyForRetryUrl();
 474         retryUri = retryUri.replace("{JOB_ID}", jobId);
 475         return restTemplateErrorAgnostic.getForEntity(retryUri, String.class);
 476     }
 477
 478     protected Object getResourceAuditInfo(String trackById) {
 479         return restTemplate.getForObject(buildUri("/asyncInstantiation/auditStatusForRetry/{trackById}"), Object.class, trackById);
 480     }
 481
 482     public class JobInfoChecker<Integer> implements Predicate<Integer> {
 483
 484         protected final RestTemplate restTemplate;
 485         protected Set<JobStatus> expectedJobStatus;
 486         protected ServiceInfo expectedServiceInfo;
 487         protected final String jobId;
 488         protected JobStatus lastStatus;
 489
 490         public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
 491             this.restTemplate = restTemplate;
 492             this.expectedJobStatus = expectedJobStatus;
 493             this.jobId = jobId;
 494             this.expectedServiceInfo = expectedServiceInfo;
 495         }
 496
 497         public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
 498             this.expectedJobStatus = expectedJobStatus;
 499         }
 500
 501         @Override
 502         public boolean test(Integer integer) {
 503             ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
 504             assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 505             assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
 506             ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
 507                     .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
 508                     .findFirst().orElse(null);
 509             Assert.assertNotNull(serviceInfoFromDB);
 510             Assert.assertEquals(serviceInfoDataReflected(expectedServiceInfo), serviceInfoDataReflected(serviceInfoFromDB));
 511             assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
 512                     serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
 513
 514             if (expectedServiceInfo.serviceInstanceId != null && ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE, JobStatus.COMPLETED_WITH_ERRORS).contains(serviceInfoFromDB.jobStatus)) {
 515                 MatcherAssert.assertThat("service instance id is wrong", serviceInfoFromDB.serviceInstanceId, CoreMatchers.is(expectedServiceInfo.serviceInstanceId));
 516             }
 517             if (expectedJobStatus.size()==1) {
 518                 assertEquals("job status is wrong", getExpectedRetryEnabled((JobStatus)(expectedJobStatus.toArray()[0])), serviceInfoFromDB.isRetryEnabled);
 519             }
 520             lastStatus = serviceInfoFromDB.jobStatus;
 521             return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
 522         }
 523     }
 524
 525     protected ResponseEntity<List<ServiceInfo>> serviceListCall() {
 526         return restTemplate.exchange(
 527                 getServiceInfoUrl(),
 528                 org.springframework.http.HttpMethod.GET,
 529                 null,
 530                 new ParameterizedTypeReference<List<ServiceInfo>>() {});
 531     }
 532
 533     //serialize fields except of fields we cannot know ahead of time
 534     protected static String serviceInfoDataReflected(ServiceInfo service1) {
 535         return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
 536                 .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName", "isRetryEnabled")
 537                 .toString();
 538     }
 539
 540     protected void addBulkPendingWithCustomList(List<BasePreset> customPresets){
 541         Map<Keys, String> names = generateNames();
 542         final int bulkSize = 2 + customPresets.size();
 543
 544         List<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 545         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 546                 .add(new PresetGetSessionSlotCheckIntervalGet())
 547                 .add(new PresetAAIGetSubscribersGet())
 548                 .addAll(msoBulkPresets)
 549                 .addAll(customPresets)
 550                 .build();
 551
 552         List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 553         Assert.assertEquals(jobIds.size(),bulkSize);
 554     }
 555
 556     protected void verifyAuditStatuses(String jobId, List<String> statuses, JobAuditStatus.SourceStatus source) {
 557         int statusesSize = statuses.size();
 558         AtomicReference<List<JobAuditStatus>> actualAudits = new AtomicReference<>();
 559         if (source.equals(JobAuditStatus.SourceStatus.VID)) {
 560             actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name()));
 561             org.junit.Assert.assertEquals("Received number of VID statuses is not as expected", statusesSize, actualAudits.get().size());
 562         } else {
 563             boolean isStatusedSizeAsExpected = Wait.waitFor(x-> {
 564                 actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()));
 565                 return actualAudits.get().size() == statusesSize;
 566             },null,5,1);
 567             org.junit.Assert.assertTrue("Received number of MSO statuses is not as expected. Expected: " + statusesSize + ". Received: " + actualAudits.get().size(), isStatusedSizeAsExpected);
 568         }
 569         IntStream.range(0, statusesSize).forEach(i-> org.junit.Assert.assertEquals(source + " status #" + i + " is not as expected", statuses.get(i), actualAudits.get().get(i).getJobStatus()));
 570     }
 571
 572     protected void verifyInstanceAuditStatuses(List<JobAuditStatus> expectedStatuses, List<JobAuditStatus> actualStatuses) {
 573         final int expectedSize = expectedStatuses.size();
 574         assertTrue("Expected statuses size is "+ expectedSize +", actual size is "+actualStatuses.size(), new Integer(expectedSize).equals(actualStatuses.size()));
 575         IntStream.range(0, expectedSize).forEach(i-> {
 576
 577             final JobAuditStatus expectedStatus = expectedStatuses.get(i);
 578             final JobAuditStatus actualStatus = actualStatuses.get(i);
 579             org.junit.Assert.assertEquals("MSO status #" + i + " is not as expected", expectedStatus.getJobStatus(), actualStatus.getJobStatus());
 580             org.junit.Assert.assertEquals("MSO requestId #" + i + " is not as expected", expectedStatus.getRequestId(), actualStatus.getRequestId());
 581             org.junit.Assert.assertEquals("MSO additionalInfo #" + i + " is not as expected", expectedStatus.getAdditionalInfo(), actualStatus.getAdditionalInfo());
 582             org.junit.Assert.assertEquals("MSO jobID #" + i + " is not as expected", expectedStatus.getJobId(), actualStatus.getJobId());
 583             org.junit.Assert.assertEquals("MSO instanceName #" + i + " is not as expected", expectedStatus.getInstanceName(), actualStatus.getInstanceName());
 584             org.junit.Assert.assertEquals("MSO instanceType  #" + i + " is not as expected", expectedStatus.getInstanceType(), actualStatus.getInstanceType());
 585         });
 586     }
 587 }