vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java

   1 package org.onap.vid.api;
   2
   3 import static java.lang.Boolean.FALSE;
   4 import static java.lang.Boolean.TRUE;
   5 import static java.util.Collections.emptyList;
   6 import static java.util.stream.Collectors.joining;
   7 import static java.util.stream.Collectors.toMap;
   8 import static org.hamcrest.CoreMatchers.hasItem;
   9 import static org.hamcrest.MatcherAssert.assertThat;
  10 import static org.hamcrest.Matchers.containsInAnyOrder;
  11 import static org.hamcrest.Matchers.hasSize;
  12 import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
  13 import static org.testng.Assert.assertNotNull;
  14 import static org.testng.AssertJUnit.assertEquals;
  15 import static org.testng.AssertJUnit.assertTrue;
  16 import static vid.automation.test.utils.ExtendedHamcrestMatcher.hasItemsFromCollection;
  17
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.Uninterruptibles;
  21 import java.time.Instant;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.UUID;
  26 import java.util.concurrent.TimeUnit;
  27 import java.util.concurrent.atomic.AtomicReference;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import java.util.stream.IntStream;
  31 import java.util.stream.Stream;
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
  34 import org.apache.commons.lang3.builder.ToStringStyle;
  35 import org.hamcrest.BaseMatcher;
  36 import org.hamcrest.CoreMatchers;
  37 import org.hamcrest.Description;
  38 import org.hamcrest.MatcherAssert;
  39 import org.onap.simulator.presetGenerator.presets.BasePresets.BaseMSOPreset;
  40 import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
  41 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetCloudOwnersByCloudRegionId;
  42 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
  43 import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
  44 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseCreateInstancePost;
  45 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseDelete;
  46 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
  47 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
  48 import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
  49 import org.onap.vid.model.asyncInstantiation.ServiceInfo;
  50 import org.onap.vid.model.mso.MsoResponseWrapper2;
  51 import org.springframework.core.ParameterizedTypeReference;
  52 import org.springframework.http.HttpMethod;
  53 import org.springframework.http.HttpStatus;
  54 import org.springframework.http.ResponseEntity;
  55 import org.springframework.web.client.RestTemplate;
  56 import org.testng.Assert;
  57 import org.testng.annotations.AfterMethod;
  58 import org.testng.annotations.BeforeClass;
  59 import org.testng.annotations.DataProvider;
  60 import vid.automation.test.infra.Features;
  61 import vid.automation.test.infra.Wait;
  62 import vid.automation.test.model.JobStatus;
  63 import vid.automation.test.model.ServiceAction;
  64 import vid.automation.test.services.AsyncJobsService;
  65 import vid.automation.test.services.SimulatorApi;
  66
  67 public class AsyncInstantiationBase extends BaseMsoApiTest {
  68
  69     public static final String CREATE_BULK_OF_ALACARTE_REQUEST_WITH_VNF = "asyncInstantiation/vidRequestCreateALaCarteWithVnf.json";
  70     protected static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
  71
  72     protected static final String MSO_BASE_ERROR =
  73             "Received error from SDN-C: java.lang.IllegalArgumentException: All keys must be specified for class org."+
  74             "opendaylight.yang.gen.v1.org.onap.sdnc.northbound.generic.resource.rev170824.vf.module.assignments.vf."+
  75             "module.assignments.vms.VmKey. Missing key is getVmType. Supplied key is VmKey [].";
  76     protected static final String MSO_ERROR = MSO_BASE_ERROR + StringUtils.repeat(" and a lot of sentences for long message", 60);
  77
  78     @BeforeClass
  79     protected void muteAndDropNameCounter() {
  80         AsyncJobsService asyncJobsService = new AsyncJobsService();
  81         asyncJobsService.muteAllAsyncJobs();
  82         asyncJobsService.dropAllFromNameCounter();
  83     }
  84
  85     @AfterMethod
  86     protected void muteAllAsyncJobs() {
  87         AsyncJobsService asyncJobsService = new AsyncJobsService();
  88         asyncJobsService.muteAllAsyncJobs();
  89     }
  90
  91     @DataProvider
  92     public static Object[][] trueAndFalse() {
  93             return new Object[][]{{TRUE},{FALSE}};
  94     }
  95
  96     protected String getCreateBulkUri() {
  97         return uri.toASCIIString() + "/asyncInstantiation/bulk";
  98     }
  99
 100     protected String getHideServiceUri(String jobId) {
 101         return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
 102     }
 103
 104     protected String getServiceInfoUrl() {
 105         return uri.toASCIIString() + "/asyncInstantiation";
 106     }
 107
 108     protected String getTemplateInfoUrl(String serviceModelId) {
 109         return uri.toASCIIString() + "/instantiationTemplates?serviceModelId=" + serviceModelId;
 110     }
 111
 112     protected String getJobAuditUrl() {
 113         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
 114     }
 115
 116     protected String getMsoJobAuditUrl() {
 117         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}/mso";
 118     }
 119
 120     protected String getDeleteServiceUrl(String uuid) {
 121         return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
 122     }
 123
 124     protected String getInstanceAuditInfoUrl() {
 125         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{TYPE}/{INSTANCE_ID}/mso";
 126     }
 127
 128     protected String getRetryJobUrl() {
 129         return uri.toASCIIString() + "/asyncInstantiation/retry/{JOB_ID}";
 130     }
 131     protected String getTopologyForRetryUrl() {
 132         return uri.toASCIIString() + "/asyncInstantiation/bulkForRetry/{JOB_ID}";
 133     }
 134
 135
 136     protected String getRetryJobWithChangedDataUrl() {
 137         return uri.toASCIIString() + "/asyncInstantiation/retryJobWithChangedData/{JOB_ID}";
 138     }
 139
 140     protected boolean getExpectedRetryEnabled(JobStatus jobStatus) {
 141         return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS);
 142     }
 143
 144     public List<BasePreset> getPresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 145
 146         final ImmutableList.Builder<BasePreset> basePresetBuilder = new ImmutableList.Builder<>();
 147         basePresetBuilder
 148                 .add(new PresetGetSessionSlotCheckIntervalGet())
 149                 .add(new PresetAAIGetSubscribersGet())
 150                 .addAll(presetOnDeleteList)
 151                 .addAll(presetOnCreateList)
 152                 .addAll(presetInProgressList);
 153         return basePresetBuilder.build();
 154     }
 155
 156     public List<BasePreset> getDeletePresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 157         return getPresets(presetOnDeleteList, emptyList(), presetInProgressList);
 158     }
 159
 160     public List<BasePreset> getPresets(List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 161         return getPresets(emptyList(), presetOnCreateList, presetInProgressList);
 162     }
 163
 164     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName) {
 165         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, userName, null, ServiceAction.INSTANTIATE);
 166     }
 167
 168     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName, String instanceId, ServiceAction action) {
 169         assertExpectedStatusAndServiceInfo(jobStatus, jobId, new ServiceInfo(
 170                 userName, jobStatus, false,
 171                 "038d99af-0427-42c2-9d15-971b99b9b489", "Lucine Sarika", "zasaki",
 172                 "de738e5f-3704-4a14-b98f-3bf86ac0c0a0", "voloyakane-senamo",
 173                 "c85f0e80-0636-44a4-8cb2-4ec00d056e79", "Hedvika Wendelin",
 174                 "a93f8383-707e-43fa-8191-a6e69a1aab17", null,
 175                 "TYLER SILVIA", "SILVIA ROBBINS",
 176                 instanceId, serviceInstanceName,
 177                 "e3c34d88-a216-4f1d-a782-9af9f9588705", "gayawabawe", "5.1",
 178                 jobId, null, action, false)
 179         );
 180     }
 181
 182     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName) {
 183         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, "us16807000");
 184     }
 185
 186     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses) {
 187         assertAuditStatuses(jobId, expectedVidStatuses, expectedMsoStatuses, 15);
 188     }
 189
 190     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses, long timeoutInSeconds) {
 191         assertAndRetryIfNeeded(() -> {
 192             final List<JobAuditStatus> auditVidStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
 193             assertThat(auditVidStatuses, hasItemsFromCollection(expectedVidStatuses));
 194             if (expectedMsoStatuses!=null) {
 195                 final List<JobAuditStatus> auditMsoStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
 196                 assertThat(auditMsoStatuses, containsInAnyOrder(expectedMsoStatuses.toArray()));
 197             }
 198         }, timeoutInSeconds);
 199     }
 200
 201     protected void assertAndRetryIfNeeded(Runnable asserter, long timeoutInSeconds) {
 202         final Instant expiry = Instant.now().plusSeconds(timeoutInSeconds);
 203         while (true) {
 204             try {
 205                 asserter.run();
 206                 break; // we're cool, assertion passed
 207             } catch (AssertionError fail) {
 208                 Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
 209                 if (Instant.now().isAfter(expiry)) {
 210                     throw fail;
 211                 } else {
 212                     System.out.println("retrying after: " + fail);
 213                 }
 214             }
 215         }
 216     }
 217
 218     protected ImmutableList<JobAuditStatus> vidAuditStatusesCompletedWithErrors(String jobId) {
 219         return ImmutableList.of(
 220                 vidAuditStatus(jobId, "PENDING", false),
 221                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 222                 vidAuditStatus(jobId, "COMPLETED_WITH_ERRORS", true)
 223         );
 224     }
 225
 226     protected ImmutableList<JobAuditStatus> vidAuditStatusesFailed(String jobId) {
 227         return ImmutableList.of(
 228                 vidAuditStatus(jobId, "PENDING", false),
 229                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 230                 vidAuditStatus(jobId, "FAILED", true)
 231         );
 232     }
 233
 234     protected JobAuditStatus vidAuditStatus(String jobId, String jobStatus, boolean isFinal) {
 235         return new JobAuditStatus(UUID.fromString(jobId), jobStatus, JobAuditStatus.SourceStatus.VID, null, null, isFinal);
 236     }
 237
 238     public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
 239         protected String expectedJobId;
 240
 241         public JobIdAndStatusMatcher(String expectedJobId) {
 242             this.expectedJobId = expectedJobId;
 243         }
 244
 245         @Override
 246         public boolean matches(Object item) {
 247             if (!(item instanceof ServiceInfo)) {
 248                 return false;
 249             }
 250             ServiceInfo serviceInfo = (ServiceInfo) item;
 251             return expectedJobId.equals(serviceInfo.jobId);
 252         }
 253
 254         @Override
 255         public void describeTo(Description description) {
 256             description.appendText("failed to find job with uuid ")
 257                     .appendValue(expectedJobId);
 258         }
 259     }
 260
 261
 262
 263     protected Map<Keys,String> generateNames() {
 264         return Stream.of(Keys.values()).collect(
 265                 Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
 266     }
 267
 268     protected ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
 269         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 270         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 271                 .add(new PresetGetSessionSlotCheckIntervalGet())
 272                 .add(new PresetAAIGetSubscribersGet())
 273                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 274                 .addAll(msoBulkPresets)
 275                 .add(new PresetMSOOrchestrationRequestGet())
 276                 .build();
 277         return presets;
 278
 279     }
 280
 281     protected ImmutableList<BasePreset> generateMsoCreateBulkPresets(int bulkSize, Map<Keys, String> names) {
 282         return IntStream.rangeClosed(0, bulkSize-1).
 283                 mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
 284                 .collect(ImmutableList.toImmutableList());
 285     }
 286
 287     protected ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
 288         return restTemplate.exchange(
 289                 url,
 290                 org.springframework.http.HttpMethod.GET,
 291                 null,
 292                 new ParameterizedTypeReference<List<JobAuditStatus>>() {});
 293     }
 294
 295     @DataProvider
 296     public static Object[][] auditSources() {
 297         return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
 298     }
 299
 300
 301
 302     protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
 303         Map<Keys, String> names = generateNames();
 304         ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
 305         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 306         Assert.assertEquals(jobIds.size(),bulkSize);
 307
 308         assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
 309                 jobIds.stream().collect(joining(","))),
 310
 311                 Wait.waitFor(y-> serviceListCall().getBody().stream()
 312                         .filter(si -> jobIds.contains(si.jobId))
 313                         .filter(si -> si.jobStatus==JobStatus.COMPLETED)
 314                         .count() == bulkSize,
 315                 null, 30, 1 ));
 316         return jobIds;
 317     }
 318
 319     protected List<JobAuditStatus> getJobMsoAuditStatusForAlaCarte(String jobUUID, String requestId, String serviceInstanceId){
 320         String url = getMsoJobAuditUrl().replace("{JOB_ID}",jobUUID);
 321
 322         if(!StringUtils.isEmpty(requestId)) {
 323             url = url + "?requestId=" + requestId;
 324             if(!StringUtils.isEmpty(serviceInstanceId)) {
 325                 url = url + "&serviceInstanceId=" + serviceInstanceId;
 326             }
 327         }
 328         return callAuditStatus(url);
 329     }
 330
 331     protected List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
 332         String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
 333         return callAuditStatus(url);
 334     }
 335
 336     protected List<JobAuditStatus> getAuditStatusesForInstance(String type, String instanceId){
 337         String url = getInstanceAuditInfoUrl().replace("{TYPE}",type).replace("{INSTANCE_ID}", instanceId);
 338         return callAuditStatus(url);
 339     }
 340
 341     private List<JobAuditStatus> callAuditStatus(String url) {
 342         ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
 343         assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 344         return statusesResponse.getBody();
 345     }
 346
 347     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
 348         return addBulkAllPendingButOneInProgress(3);
 349     }
 350
 351     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
 352         Map<Keys, String> names = generateNames();
 353         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 354         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 355                 .add(new PresetGetSessionSlotCheckIntervalGet())
 356                 .add(new PresetAAIGetSubscribersGet())
 357                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 358                 .addAll(msoBulkPresets)
 359                 .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
 360                 .build();
 361         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 362
 363         // wait for single IN_PROGRESS, so statuses will stop from changing
 364         Wait.waitFor(foo -> serviceListCall().getBody().stream()
 365                         .filter(si -> jobIds.contains(si.jobId))
 366                         .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
 367                 null, 20, 1);
 368
 369         final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
 370                 .filter(si -> jobIds.contains(si.jobId))
 371                 .collect(toMap(si -> si.jobId, si -> si.jobStatus));
 372
 373         assertThat(jobIds, hasSize(bulkSize));
 374
 375
 376         return statusMapBefore;
 377     }
 378
 379     protected String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
 380         final String jobToDelete = jobIdToStatus.entrySet().stream()
 381                 .filter(entry -> entry.getValue().equals(jobStatus))
 382                 .map(Map.Entry::getKey)
 383                 .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
 384
 385
 386         restTemplate.delete(getDeleteServiceUrl(jobToDelete));
 387
 388         return jobToDelete;
 389     }
 390
 391
 392     protected MsoResponseWrapper2 hideService(String jobId) {
 393         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
 394         return responseWrapper2;
 395     }
 396
 397     protected List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
 398         List<String> jobIds = createBulkOfMacroInstances(presets, isPause, bulkSize, names);
 399         Assert.assertEquals(jobIds.size(), bulkSize);
 400         for(String jobId: jobIds) {
 401             assertExpectedStatusAndServiceInfo(isPause, finalState, names, jobId);
 402         }
 403
 404         return jobIds;
 405     }
 406
 407     protected void assertExpectedStatusAndServiceInfo(boolean isPause, JobStatus finalState, Map<Keys, String> names, String jobId) {
 408         assertExpectedStatusAndServiceInfo(finalState, jobId, new ServiceInfo("us16807000", JobStatus.IN_PROGRESS, isPause, "someID",
 409                 "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "hvf3", null,
 410                 "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
 411                 "5c9e863f-2716-467b-8799-4a67f378dcaa", "AIM_TRANSPORT_00004", "1.0", jobId, null, ServiceAction.INSTANTIATE, false));
 412     }
 413
 414     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, ServiceInfo expectedServiceInfo) {
 415         assertExpectedStatusAndServiceInfo(finalState, jobId, PATIENCE_LEVEL.FAIL_FAST, expectedServiceInfo);
 416     }
 417
 418     enum PATIENCE_LEVEL { FAIL_FAST, FAIL_SLOW, FAIL_VERY_SLOW }
 419
 420     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, PATIENCE_LEVEL patienceLevel, ServiceInfo expectedServiceInfo) {
 421         JobInfoChecker<Integer> jobInfoChecker = new JobInfoChecker<>(
 422                 restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
 423         boolean result = jobInfoChecker.test(null);
 424         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 425
 426         jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
 427         if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState) && expectedServiceInfo.serviceInstanceId==null) {
 428             expectedServiceInfo.serviceInstanceId = BaseMSOPreset.DEFAULT_INSTANCE_ID;
 429         }
 430         result = Wait.waitFor(jobInfoChecker, null, 30, waitIntervalBy(patienceLevel));
 431         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 432     }
 433
 434     private int waitIntervalBy(PATIENCE_LEVEL patienceLevel) {
 435         switch (patienceLevel) {
 436             case FAIL_SLOW:
 437                 return 2;
 438             case FAIL_VERY_SLOW:
 439                 return 3;
 440             default:
 441                 return 1;
 442         }
 443     }
 444
 445     protected List<String> createBulkOfMacroInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names) {
 446         SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
 447         return createBulkOfInstances(isPause, bulkSize, names, CREATE_BULK_OF_MACRO_REQUEST);
 448     }
 449
 450     public List<String> createBulkOfInstances(boolean isPause, int bulkSize, Map<Keys, String> names, String requestDetailsFileName){
 451
 452         String requestBody = TestUtils.convertRequest(objectMapper, requestDetailsFileName);
 453         requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
 454         for (Map.Entry<Keys, String> e : names.entrySet()) {
 455             requestBody = requestBody.replace(e.getKey().name(), e.getValue());
 456         }
 457         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
 458         assertNotNull(responseWrapper2);
 459         return (List<String>)responseWrapper2.getEntity();
 460     }
 461
 462     protected List<String> retryJob(String jobId) {
 463         ResponseEntity<String> retryBulkPayload = getRetryBulk(jobId);
 464         return retryJobWithChangedData(jobId, retryBulkPayload.getBody());
 465     }
 466
 467     protected List<String> retryJobWithChangedData(String jobId, String requestBody) {
 468         String retryUri = getRetryJobWithChangedDataUrl();
 469         retryUri = retryUri.replace("{JOB_ID}", jobId);
 470         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(HttpMethod.POST, retryUri, requestBody);
 471         assertNotNull(responseWrapper2);
 472         return (List<String>)responseWrapper2.getEntity();
 473     }
 474
 475     protected ResponseEntity<String> getRetryBulk(String jobId) {
 476         String retryUri = getTopologyForRetryUrl();
 477         retryUri = retryUri.replace("{JOB_ID}", jobId);
 478         return restTemplateErrorAgnostic.getForEntity(retryUri, String.class);
 479     }
 480
 481     protected Object getResourceAuditInfo(String trackById) {
 482         return restTemplate.getForObject(buildUri("/asyncInstantiation/auditStatusForRetry/{trackById}"), Object.class, trackById);
 483     }
 484
 485     public class JobInfoChecker<Integer> implements Predicate<Integer> {
 486
 487         protected final RestTemplate restTemplate;
 488         protected Set<JobStatus> expectedJobStatus;
 489         protected ServiceInfo expectedServiceInfo;
 490         protected final String jobId;
 491         protected JobStatus lastStatus;
 492
 493         public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
 494             this.restTemplate = restTemplate;
 495             this.expectedJobStatus = expectedJobStatus;
 496             this.jobId = jobId;
 497             this.expectedServiceInfo = expectedServiceInfo;
 498         }
 499
 500         public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
 501             this.expectedJobStatus = expectedJobStatus;
 502         }
 503
 504         @Override
 505         public boolean test(Integer integer) {
 506             ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
 507             assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 508             assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
 509             ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
 510                     .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
 511                     .findFirst().orElse(null);
 512             Assert.assertNotNull(serviceInfoFromDB);
 513             Assert.assertEquals(serviceInfoDataReflected(serviceInfoFromDB), serviceInfoDataReflected(expectedServiceInfo));
 514             assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
 515                     serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
 516
 517             if (expectedServiceInfo.serviceInstanceId != null && ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE, JobStatus.COMPLETED_WITH_ERRORS).contains(serviceInfoFromDB.jobStatus)) {
 518                 MatcherAssert.assertThat("service instance id is wrong", serviceInfoFromDB.serviceInstanceId, CoreMatchers.is(expectedServiceInfo.serviceInstanceId));
 519             }
 520             if (expectedJobStatus.size()==1) {
 521                 assertEquals("job status is wrong", getExpectedRetryEnabled((JobStatus)(expectedJobStatus.toArray()[0])), serviceInfoFromDB.isRetryEnabled);
 522             }
 523             lastStatus = serviceInfoFromDB.jobStatus;
 524             return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
 525         }
 526     }
 527
 528     protected ResponseEntity<List<ServiceInfo>> serviceListCall() {
 529         return restTemplate.exchange(
 530                 getServiceInfoUrl(),
 531                 org.springframework.http.HttpMethod.GET,
 532                 null,
 533                 new ParameterizedTypeReference<List<ServiceInfo>>() {});
 534     }
 535
 536     //serialize fields except of fields we cannot know ahead of time
 537     protected static String serviceInfoDataReflected(ServiceInfo service1) {
 538         return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
 539                 .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName", "isRetryEnabled")
 540                 .toString();
 541     }
 542
 543     protected void addBulkPendingWithCustomList(List<BasePreset> customPresets){
 544         Map<Keys, String> names = generateNames();
 545         final int bulkSize = 2 + customPresets.size();
 546
 547         List<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 548         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 549                 .add(new PresetGetSessionSlotCheckIntervalGet())
 550                 .add(new PresetAAIGetSubscribersGet())
 551                 .addAll(msoBulkPresets)
 552                 .addAll(customPresets)
 553                 .build();
 554
 555         List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 556         Assert.assertEquals(jobIds.size(),bulkSize);
 557     }
 558
 559     protected void verifyAuditStatuses(String jobId, List<String> statuses, JobAuditStatus.SourceStatus source) {
 560         int statusesSize = statuses.size();
 561         AtomicReference<List<JobAuditStatus>> actualAudits = new AtomicReference<>();
 562         if (source.equals(JobAuditStatus.SourceStatus.VID)) {
 563             actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name()));
 564             org.junit.Assert.assertEquals("Received number of VID statuses is not as expected", statusesSize, actualAudits.get().size());
 565         } else {
 566             boolean isStatusedSizeAsExpected = Wait.waitFor(x-> {
 567                 actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()));
 568                 return actualAudits.get().size() == statusesSize;
 569             },null,5,1);
 570             org.junit.Assert.assertTrue("Received number of MSO statuses is not as expected. Expected: " + statusesSize + ". Received: " + actualAudits.get().size(), isStatusedSizeAsExpected);
 571         }
 572         IntStream.range(0, statusesSize).forEach(i-> org.junit.Assert.assertEquals(source + " status #" + i + " is not as expected", statuses.get(i), actualAudits.get().get(i).getJobStatus()));
 573     }
 574
 575     protected void verifyInstanceAuditStatuses(List<JobAuditStatus> expectedStatuses, List<JobAuditStatus> actualStatuses) {
 576         final int expectedSize = expectedStatuses.size();
 577         assertTrue("Expected statuses size is "+ expectedSize +", actual size is "+actualStatuses.size(), new Integer(expectedSize).equals(actualStatuses.size()));
 578         IntStream.range(0, expectedSize).forEach(i-> {
 579
 580             final JobAuditStatus expectedStatus = expectedStatuses.get(i);
 581             final JobAuditStatus actualStatus = actualStatuses.get(i);
 582             org.junit.Assert.assertEquals("MSO status #" + i + " is not as expected", expectedStatus.getJobStatus(), actualStatus.getJobStatus());
 583             org.junit.Assert.assertEquals("MSO requestId #" + i + " is not as expected", expectedStatus.getRequestId(), actualStatus.getRequestId());
 584             org.junit.Assert.assertEquals("MSO additionalInfo #" + i + " is not as expected", expectedStatus.getAdditionalInfo(), actualStatus.getAdditionalInfo());
 585             org.junit.Assert.assertEquals("MSO jobID #" + i + " is not as expected", expectedStatus.getJobId(), actualStatus.getJobId());
 586             org.junit.Assert.assertEquals("MSO instanceName #" + i + " is not as expected", expectedStatus.getInstanceName(), actualStatus.getInstanceName());
 587             org.junit.Assert.assertEquals("MSO instanceType  #" + i + " is not as expected", expectedStatus.getInstanceType(), actualStatus.getInstanceType());
 588         });
 589     }
 590 }