vid-automation/src/main/java/org/onap/vid/api/AsyncInstantiationBase.java

   1 package org.onap.vid.api;
   2
   3 import static java.lang.Boolean.FALSE;
   4 import static java.lang.Boolean.TRUE;
   5 import static java.util.Collections.emptyList;
   6 import static java.util.stream.Collectors.joining;
   7 import static java.util.stream.Collectors.toMap;
   8 import static org.hamcrest.CoreMatchers.hasItem;
   9 import static org.hamcrest.MatcherAssert.assertThat;
  10 import static org.hamcrest.Matchers.containsInAnyOrder;
  11 import static org.hamcrest.Matchers.hasSize;
  12 import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
  13 import static org.testng.Assert.assertNotNull;
  14 import static org.testng.AssertJUnit.assertEquals;
  15 import static org.testng.AssertJUnit.assertTrue;
  16 import static vid.automation.test.utils.ExtendedHamcrestMatcher.hasItemsFromCollection;
  17
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Set;
  23 import java.util.UUID;
  24 import java.util.concurrent.atomic.AtomicReference;
  25 import java.util.function.Predicate;
  26 import java.util.stream.Collectors;
  27 import java.util.stream.IntStream;
  28 import java.util.stream.Stream;
  29 import org.apache.commons.lang3.StringUtils;
  30 import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
  31 import org.apache.commons.lang3.builder.ToStringStyle;
  32 import org.hamcrest.BaseMatcher;
  33 import org.hamcrest.CoreMatchers;
  34 import org.hamcrest.Description;
  35 import org.hamcrest.MatcherAssert;
  36 import org.onap.simulator.presetGenerator.presets.BasePresets.BaseMSOPreset;
  37 import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
  38 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetCloudOwnersByCloudRegionId;
  39 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
  40 import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
  41 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseCreateInstancePost;
  42 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOBaseDelete;
  43 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
  44 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
  45 import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
  46 import org.onap.vid.model.asyncInstantiation.ServiceInfo;
  47 import org.onap.vid.model.mso.MsoResponseWrapper2;
  48 import org.springframework.core.ParameterizedTypeReference;
  49 import org.springframework.http.HttpMethod;
  50 import org.springframework.http.HttpStatus;
  51 import org.springframework.http.ResponseEntity;
  52 import org.springframework.web.client.RestTemplate;
  53 import org.testng.Assert;
  54 import org.testng.annotations.AfterMethod;
  55 import org.testng.annotations.BeforeClass;
  56 import org.testng.annotations.DataProvider;
  57 import vid.automation.test.infra.Features;
  58 import vid.automation.test.infra.Wait;
  59 import vid.automation.test.model.JobStatus;
  60 import vid.automation.test.model.ServiceAction;
  61 import vid.automation.test.services.AsyncJobsService;
  62 import vid.automation.test.services.SimulatorApi;
  63
  64 public class AsyncInstantiationBase extends BaseMsoApiTest {
  65
  66     public static final String CREATE_BULK_OF_ALACARTE_REQUEST_WITH_VNF = "asyncInstantiation/vidRequestCreateALaCarteWithVnf.json";
  67     protected static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
  68
  69     protected static final String MSO_BASE_ERROR =
  70             "Received error from SDN-C: java.lang.IllegalArgumentException: All keys must be specified for class org."+
  71             "opendaylight.yang.gen.v1.org.onap.sdnc.northbound.generic.resource.rev170824.vf.module.assignments.vf."+
  72             "module.assignments.vms.VmKey. Missing key is getVmType. Supplied key is VmKey [].";
  73     protected static final String MSO_ERROR = MSO_BASE_ERROR + StringUtils.repeat(" and a lot of sentences for long message", 60);
  74
  75     @BeforeClass
  76     protected void muteAndDropNameCounter() {
  77         AsyncJobsService asyncJobsService = new AsyncJobsService();
  78         asyncJobsService.muteAllAsyncJobs();
  79         asyncJobsService.dropAllFromNameCounter();
  80     }
  81
  82     @AfterMethod
  83     protected void muteAllAsyncJobs() {
  84         AsyncJobsService asyncJobsService = new AsyncJobsService();
  85         asyncJobsService.muteAllAsyncJobs();
  86     }
  87
  88     @DataProvider
  89     public static Object[][] trueAndFalse() {
  90             return new Object[][]{{TRUE},{FALSE}};
  91     }
  92
  93     protected String getCreateBulkUri() {
  94         return uri.toASCIIString() + "/asyncInstantiation/bulk";
  95     }
  96
  97     protected String getHideServiceUri(String jobId) {
  98         return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
  99     }
 100
 101     protected String getServiceInfoUrl() {
 102         return uri.toASCIIString() + "/asyncInstantiation";
 103     }
 104
 105     protected String getTemplateInfoUrl(String serviceModelId) {
 106         return uri.toASCIIString() + "/instantiationTemplates?serviceModelId=" + serviceModelId;
 107     }
 108
 109     protected String getJobAuditUrl() {
 110         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
 111     }
 112
 113     protected String getMsoJobAuditUrl() {
 114         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}/mso";
 115     }
 116
 117     protected String getDeleteServiceUrl(String uuid) {
 118         return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
 119     }
 120
 121     protected String getInstanceAuditInfoUrl() {
 122         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{TYPE}/{INSTANCE_ID}/mso";
 123     }
 124
 125     protected String getRetryJobUrl() {
 126         return uri.toASCIIString() + "/asyncInstantiation/retry/{JOB_ID}";
 127     }
 128     protected String getTopologyForRetryUrl() {
 129         return uri.toASCIIString() + "/asyncInstantiation/bulkForRetry/{JOB_ID}";
 130     }
 131
 132
 133     protected String getRetryJobWithChangedDataUrl() {
 134         return uri.toASCIIString() + "/asyncInstantiation/retryJobWithChangedData/{JOB_ID}";
 135     }
 136
 137     protected boolean getExpectedRetryEnabled(JobStatus jobStatus) {
 138         return Features.FLAG_1902_RETRY_JOB.isActive() && (jobStatus==JobStatus.FAILED || jobStatus==JobStatus.COMPLETED_WITH_ERRORS);
 139     }
 140
 141     public List<BasePreset> getPresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 142
 143         final ImmutableList.Builder<BasePreset> basePresetBuilder = new ImmutableList.Builder<>();
 144         basePresetBuilder
 145                 .add(new PresetGetSessionSlotCheckIntervalGet())
 146                 .add(new PresetAAIGetSubscribersGet())
 147                 .addAll(presetOnDeleteList)
 148                 .addAll(presetOnCreateList)
 149                 .addAll(presetInProgressList);
 150         return basePresetBuilder.build();
 151     }
 152
 153     public List<BasePreset> getDeletePresets(List<PresetMSOBaseDelete> presetOnDeleteList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 154         return getPresets(presetOnDeleteList, emptyList(), presetInProgressList);
 155     }
 156
 157     public List<BasePreset> getPresets(List<PresetMSOBaseCreateInstancePost> presetOnCreateList, List<PresetMSOOrchestrationRequestGet> presetInProgressList) {
 158         return getPresets(emptyList(), presetOnCreateList, presetInProgressList);
 159     }
 160
 161     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName) {
 162         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, userName, null, ServiceAction.INSTANTIATE);
 163     }
 164
 165     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName, String userName, String instanceId, ServiceAction action) {
 166         assertExpectedStatusAndServiceInfo(jobStatus, jobId, new ServiceInfo(
 167                 userName, jobStatus, false,
 168                 "038d99af-0427-42c2-9d15-971b99b9b489", "Lucine Sarika", "zasaki",
 169                 "de738e5f-3704-4a14-b98f-3bf86ac0c0a0", "voloyakane-senamo",
 170                 "c85f0e80-0636-44a4-8cb2-4ec00d056e79", "Hedvika Wendelin",
 171                 "a93f8383-707e-43fa-8191-a6e69a1aab17", null,
 172                 "TYLER SILVIA", "SILVIA ROBBINS",
 173                 instanceId, serviceInstanceName,
 174                 "e3c34d88-a216-4f1d-a782-9af9f9588705", "gayawabawe", "5.1",
 175                 jobId, null, action, false)
 176         );
 177     }
 178
 179     public void assertServiceInfoSpecific1(String jobId, JobStatus jobStatus, String serviceInstanceName) {
 180         assertServiceInfoSpecific1(jobId, jobStatus, serviceInstanceName, "us16807000");
 181     }
 182
 183     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses) {
 184         assertAuditStatuses(jobId, expectedVidStatuses, expectedMsoStatuses, 15);
 185     }
 186
 187     protected void assertAuditStatuses(String jobId, List<JobAuditStatus> expectedVidStatuses, List<JobAuditStatus> expectedMsoStatuses, long timeoutInSeconds) {
 188         assertAndRetryIfNeeded(() -> {
 189             final List<JobAuditStatus> auditVidStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
 190             assertThat(auditVidStatuses, hasItemsFromCollection(expectedVidStatuses));
 191             if (expectedMsoStatuses!=null) {
 192                 final List<JobAuditStatus> auditMsoStatuses = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
 193                 assertThat(auditMsoStatuses, containsInAnyOrder(expectedMsoStatuses.toArray()));
 194             }
 195         }, timeoutInSeconds);
 196     }
 197
 198     protected void assertAndRetryIfNeeded(Runnable asserter, long timeoutInSeconds) {
 199         TestUtils.assertAndRetryIfNeeded(timeoutInSeconds, asserter);
 200     }
 201
 202     protected ImmutableList<JobAuditStatus> vidAuditStatusesCompletedWithErrors(String jobId) {
 203         return ImmutableList.of(
 204                 vidAuditStatus(jobId, "PENDING", false),
 205                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 206                 vidAuditStatus(jobId, "COMPLETED_WITH_ERRORS", true)
 207         );
 208     }
 209
 210     protected ImmutableList<JobAuditStatus> vidAuditStatusesFailed(String jobId) {
 211         return ImmutableList.of(
 212                 vidAuditStatus(jobId, "PENDING", false),
 213                 vidAuditStatus(jobId, "IN_PROGRESS", false),
 214                 vidAuditStatus(jobId, "FAILED", true)
 215         );
 216     }
 217
 218     protected JobAuditStatus vidAuditStatus(String jobId, String jobStatus, boolean isFinal) {
 219         return new JobAuditStatus(UUID.fromString(jobId), jobStatus, JobAuditStatus.SourceStatus.VID, null, null, isFinal);
 220     }
 221
 222     public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
 223         protected String expectedJobId;
 224
 225         public JobIdAndStatusMatcher(String expectedJobId) {
 226             this.expectedJobId = expectedJobId;
 227         }
 228
 229         @Override
 230         public boolean matches(Object item) {
 231             if (!(item instanceof ServiceInfo)) {
 232                 return false;
 233             }
 234             ServiceInfo serviceInfo = (ServiceInfo) item;
 235             return expectedJobId.equals(serviceInfo.jobId);
 236         }
 237
 238         @Override
 239         public void describeTo(Description description) {
 240             description.appendText("failed to find job with uuid ")
 241                     .appendValue(expectedJobId);
 242         }
 243     }
 244
 245
 246
 247     protected Map<Keys,String> generateNames() {
 248         return Stream.of(Keys.values()).collect(
 249                 Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
 250     }
 251
 252     protected ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
 253         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 254         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 255                 .add(new PresetGetSessionSlotCheckIntervalGet())
 256                 .add(new PresetAAIGetSubscribersGet())
 257                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 258                 .addAll(msoBulkPresets)
 259                 .add(new PresetMSOOrchestrationRequestGet())
 260                 .build();
 261         return presets;
 262
 263     }
 264
 265     protected ImmutableList<BasePreset> generateMsoCreateBulkPresets(int bulkSize, Map<Keys, String> names) {
 266         return IntStream.rangeClosed(0, bulkSize-1).
 267                 mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
 268                 .collect(ImmutableList.toImmutableList());
 269     }
 270
 271     protected ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
 272         return restTemplate.exchange(
 273                 url,
 274                 org.springframework.http.HttpMethod.GET,
 275                 null,
 276                 new ParameterizedTypeReference<List<JobAuditStatus>>() {});
 277     }
 278
 279     @DataProvider
 280     public static Object[][] auditSources() {
 281         return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
 282     }
 283
 284
 285
 286     protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
 287         Map<Keys, String> names = generateNames();
 288         ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
 289         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 290         Assert.assertEquals(jobIds.size(),bulkSize);
 291
 292         waitForJobsToSuccessfullyCompleted(bulkSize, jobIds);
 293         return jobIds;
 294     }
 295
 296     public void waitForJobsToSuccessfullyCompleted(int bulkSize, List<String> jobIds) {
 297         assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
 298                 jobIds.stream().collect(joining(","))),
 299
 300                 Wait.waitFor(y-> serviceListCall().getBody().stream()
 301                         .filter(si -> jobIds.contains(si.jobId))
 302                         .filter(si -> si.jobStatus== JobStatus.COMPLETED)
 303                         .count() == bulkSize,
 304                 null, 30, 1 ));
 305     }
 306
 307     protected List<JobAuditStatus> getJobMsoAuditStatusForAlaCarte(String jobUUID, String requestId, String serviceInstanceId){
 308         String url = getMsoJobAuditUrl().replace("{JOB_ID}",jobUUID);
 309
 310         if(!StringUtils.isEmpty(requestId)) {
 311             url = url + "?requestId=" + requestId;
 312             if(!StringUtils.isEmpty(serviceInstanceId)) {
 313                 url = url + "&serviceInstanceId=" + serviceInstanceId;
 314             }
 315         }
 316         return callAuditStatus(url);
 317     }
 318
 319     protected List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
 320         String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
 321         return callAuditStatus(url);
 322     }
 323
 324     protected List<JobAuditStatus> getAuditStatusesForInstance(String type, String instanceId){
 325         String url = getInstanceAuditInfoUrl().replace("{TYPE}",type).replace("{INSTANCE_ID}", instanceId);
 326         return callAuditStatus(url);
 327     }
 328
 329     private List<JobAuditStatus> callAuditStatus(String url) {
 330         ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
 331         assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 332         return statusesResponse.getBody();
 333     }
 334
 335     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
 336         return addBulkAllPendingButOneInProgress(3);
 337     }
 338
 339     protected Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
 340         Map<Keys, String> names = generateNames();
 341         ImmutableList<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 342         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 343                 .add(new PresetGetSessionSlotCheckIntervalGet())
 344                 .add(new PresetAAIGetSubscribersGet())
 345                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA)
 346                 .addAll(msoBulkPresets)
 347                 .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
 348                 .build();
 349         final List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 350
 351         // wait for single IN_PROGRESS, so statuses will stop from changing
 352         Wait.waitFor(foo -> serviceListCall().getBody().stream()
 353                         .filter(si -> jobIds.contains(si.jobId))
 354                         .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
 355                 null, 20, 1);
 356
 357         final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
 358                 .filter(si -> jobIds.contains(si.jobId))
 359                 .collect(toMap(si -> si.jobId, si -> si.jobStatus));
 360
 361         assertThat(jobIds, hasSize(bulkSize));
 362
 363
 364         return statusMapBefore;
 365     }
 366
 367     protected String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
 368         final String jobToDelete = jobIdToStatus.entrySet().stream()
 369                 .filter(entry -> entry.getValue().equals(jobStatus))
 370                 .map(Map.Entry::getKey)
 371                 .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
 372
 373
 374         restTemplate.delete(getDeleteServiceUrl(jobToDelete));
 375
 376         return jobToDelete;
 377     }
 378
 379
 380     protected MsoResponseWrapper2 hideService(String jobId) {
 381         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
 382         return responseWrapper2;
 383     }
 384
 385     protected List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
 386         List<String> jobIds = createBulkOfMacroInstances(presets, isPause, bulkSize, names);
 387         Assert.assertEquals(jobIds.size(), bulkSize);
 388         for(String jobId: jobIds) {
 389             assertExpectedStatusAndServiceInfo(isPause, finalState, names, jobId);
 390         }
 391
 392         return jobIds;
 393     }
 394
 395     protected void assertExpectedStatusAndServiceInfo(boolean isPause, JobStatus finalState, Map<Keys, String> names, String jobId) {
 396         assertExpectedStatusAndServiceInfo(finalState, jobId, new ServiceInfo("us16807000", JobStatus.IN_PROGRESS, isPause, "someID",
 397                 "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "hvf3", null,
 398                 "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
 399                 "5c9e863f-2716-467b-8799-4a67f378dcaa", "AIM_TRANSPORT_00004", "1.0", jobId, null, ServiceAction.INSTANTIATE, false));
 400     }
 401
 402     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, ServiceInfo expectedServiceInfo) {
 403         assertExpectedStatusAndServiceInfo(finalState, jobId, PATIENCE_LEVEL.FAIL_FAST, expectedServiceInfo);
 404     }
 405
 406     enum PATIENCE_LEVEL { FAIL_FAST, FAIL_SLOW, FAIL_VERY_SLOW }
 407
 408     protected void assertExpectedStatusAndServiceInfo(JobStatus finalState, String jobId, PATIENCE_LEVEL patienceLevel, ServiceInfo expectedServiceInfo) {
 409         JobInfoChecker<Integer> jobInfoChecker = new JobInfoChecker<>(
 410                 restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
 411         boolean result = jobInfoChecker.test(null);
 412         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 413
 414         jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
 415         if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState) && expectedServiceInfo.serviceInstanceId==null) {
 416             expectedServiceInfo.serviceInstanceId = BaseMSOPreset.DEFAULT_INSTANCE_ID;
 417         }
 418         result = Wait.waitFor(jobInfoChecker, null, 30, waitIntervalBy(patienceLevel));
 419         assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
 420     }
 421
 422     private int waitIntervalBy(PATIENCE_LEVEL patienceLevel) {
 423         switch (patienceLevel) {
 424             case FAIL_SLOW:
 425                 return 2;
 426             case FAIL_VERY_SLOW:
 427                 return 3;
 428             default:
 429                 return 1;
 430         }
 431     }
 432
 433     protected List<String> createBulkOfMacroInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names) {
 434         SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
 435         return createBulkOfInstances(isPause, bulkSize, names, CREATE_BULK_OF_MACRO_REQUEST);
 436     }
 437
 438     public List<String> createBulkOfInstances(boolean isPause, int bulkSize, Map<Keys, String> names, String requestDetailsFileName){
 439
 440         String requestBody = TestUtils.convertRequest(objectMapper, requestDetailsFileName);
 441         requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
 442         for (Map.Entry<Keys, String> e : names.entrySet()) {
 443             requestBody = requestBody.replace(e.getKey().name(), e.getValue());
 444         }
 445         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
 446         assertNotNull(responseWrapper2);
 447         return (List<String>)responseWrapper2.getEntity();
 448     }
 449
 450     protected List<String> retryJob(String jobId) {
 451         ResponseEntity<String> retryBulkPayload = getRetryBulk(jobId);
 452         return retryJobWithChangedData(jobId, retryBulkPayload.getBody());
 453     }
 454
 455     protected List<String> retryJobWithChangedData(String jobId, String requestBody) {
 456         String retryUri = getRetryJobWithChangedDataUrl();
 457         retryUri = retryUri.replace("{JOB_ID}", jobId);
 458         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(HttpMethod.POST, retryUri, requestBody);
 459         assertNotNull(responseWrapper2);
 460         return (List<String>)responseWrapper2.getEntity();
 461     }
 462
 463     protected ResponseEntity<String> getRetryBulk(String jobId) {
 464         String retryUri = getTopologyForRetryUrl();
 465         retryUri = retryUri.replace("{JOB_ID}", jobId);
 466         return restTemplateErrorAgnostic.getForEntity(retryUri, String.class);
 467     }
 468
 469     protected Object getResourceAuditInfo(String trackById) {
 470         return restTemplate.getForObject(buildUri("/asyncInstantiation/auditStatusForRetry/{trackById}"), Object.class, trackById);
 471     }
 472
 473     public class JobInfoChecker<Integer> implements Predicate<Integer> {
 474
 475         protected final RestTemplate restTemplate;
 476         protected Set<JobStatus> expectedJobStatus;
 477         protected ServiceInfo expectedServiceInfo;
 478         protected final String jobId;
 479         protected JobStatus lastStatus;
 480
 481         public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
 482             this.restTemplate = restTemplate;
 483             this.expectedJobStatus = expectedJobStatus;
 484             this.jobId = jobId;
 485             this.expectedServiceInfo = expectedServiceInfo;
 486         }
 487
 488         public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
 489             this.expectedJobStatus = expectedJobStatus;
 490         }
 491
 492         @Override
 493         public boolean test(Integer integer) {
 494             ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
 495             assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
 496             assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
 497             ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
 498                     .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
 499                     .findFirst().orElse(null);
 500             Assert.assertNotNull(serviceInfoFromDB);
 501             Assert.assertEquals(serviceInfoDataReflected(serviceInfoFromDB), serviceInfoDataReflected(expectedServiceInfo));
 502             assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
 503                     serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
 504
 505             if (expectedServiceInfo.serviceInstanceId != null && ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE, JobStatus.COMPLETED_WITH_ERRORS).contains(serviceInfoFromDB.jobStatus)) {
 506                 MatcherAssert.assertThat("service instance id is wrong", serviceInfoFromDB.serviceInstanceId, CoreMatchers.is(expectedServiceInfo.serviceInstanceId));
 507             }
 508             if (expectedJobStatus.size()==1) {
 509                 assertEquals("job status is wrong", getExpectedRetryEnabled((JobStatus)(expectedJobStatus.toArray()[0])), serviceInfoFromDB.isRetryEnabled);
 510             }
 511             lastStatus = serviceInfoFromDB.jobStatus;
 512             return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
 513         }
 514     }
 515
 516     protected ResponseEntity<List<ServiceInfo>> serviceListCall() {
 517         return restTemplate.exchange(
 518                 getServiceInfoUrl(),
 519                 org.springframework.http.HttpMethod.GET,
 520                 null,
 521                 new ParameterizedTypeReference<List<ServiceInfo>>() {});
 522     }
 523
 524     //serialize fields except of fields we cannot know ahead of time
 525     protected static String serviceInfoDataReflected(ServiceInfo service1) {
 526         return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
 527                 .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName", "isRetryEnabled")
 528                 .toString();
 529     }
 530
 531     protected Map<Keys, String> addBulkPendingWithCustomList(List<BasePreset> customPresets){
 532         Map<Keys, String> names = generateNames();
 533         final int bulkSize = 2 + customPresets.size();
 534
 535         List<BasePreset> msoBulkPresets = generateMsoCreateBulkPresets(bulkSize, names);
 536         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
 537                 .add(new PresetGetSessionSlotCheckIntervalGet())
 538                 .add(new PresetAAIGetSubscribersGet())
 539                 .add(PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MTN3_TO_ATT_SABABA, PresetAAIGetCloudOwnersByCloudRegionId.PRESET_MDT1_TO_ATT_NC)
 540                 .addAll(msoBulkPresets)
 541                 .addAll(customPresets)
 542                 .build();
 543
 544         List<String> jobIds = createBulkOfMacroInstances(presets, false, bulkSize, names);
 545         Assert.assertEquals(jobIds.size(),bulkSize);
 546
 547         return names;
 548     }
 549
 550     protected void verifyAuditStatuses(String jobId, List<String> statuses, JobAuditStatus.SourceStatus source) {
 551         int statusesSize = statuses.size();
 552         AtomicReference<List<JobAuditStatus>> actualAudits = new AtomicReference<>();
 553         if (source.equals(JobAuditStatus.SourceStatus.VID)) {
 554             actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name()));
 555             org.junit.Assert.assertEquals("Received number of VID statuses is not as expected", statusesSize, actualAudits.get().size());
 556         } else {
 557             boolean isStatusedSizeAsExpected = Wait.waitFor(x-> {
 558                 actualAudits.set(getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()));
 559                 return actualAudits.get().size() == statusesSize;
 560             },null,5,1);
 561             org.junit.Assert.assertTrue("Received number of MSO statuses is not as expected. Expected: " + statusesSize + ". Received: " + actualAudits.get().size(), isStatusedSizeAsExpected);
 562         }
 563         IntStream.range(0, statusesSize).forEach(i-> org.junit.Assert.assertEquals(source + " status #" + i + " is not as expected", statuses.get(i), actualAudits.get().get(i).getJobStatus()));
 564     }
 565
 566     protected void verifyInstanceAuditStatuses(List<JobAuditStatus> expectedStatuses, List<JobAuditStatus> actualStatuses) {
 567         final int expectedSize = expectedStatuses.size();
 568         assertTrue("Expected statuses size is "+ expectedSize +", actual size is "+actualStatuses.size(), new Integer(expectedSize).equals(actualStatuses.size()));
 569         IntStream.range(0, expectedSize).forEach(i-> {
 570
 571             final JobAuditStatus expectedStatus = expectedStatuses.get(i);
 572             final JobAuditStatus actualStatus = actualStatuses.get(i);
 573             org.junit.Assert.assertEquals("MSO status #" + i + " is not as expected", expectedStatus.getJobStatus(), actualStatus.getJobStatus());
 574             org.junit.Assert.assertEquals("MSO requestId #" + i + " is not as expected", expectedStatus.getRequestId(), actualStatus.getRequestId());
 575             org.junit.Assert.assertEquals("MSO additionalInfo #" + i + " is not as expected", expectedStatus.getAdditionalInfo(), actualStatus.getAdditionalInfo());
 576             org.junit.Assert.assertEquals("MSO jobID #" + i + " is not as expected", expectedStatus.getJobId(), actualStatus.getJobId());
 577             org.junit.Assert.assertEquals("MSO instanceName #" + i + " is not as expected", expectedStatus.getInstanceName(), actualStatus.getInstanceName());
 578             org.junit.Assert.assertEquals("MSO instanceType  #" + i + " is not as expected", expectedStatus.getInstanceType(), actualStatus.getInstanceType());
 579         });
 580     }
 581 }