Fix for overriding skipTests variable.
[vid.git] / vid-automation / src / test / java / org / onap / vid / api / AsyncInstantiationApiTest.java
1 package org.onap.vid.api;
2
3 import com.google.common.collect.ImmutableList;
4 import com.google.common.collect.ImmutableSet;
5 import net.bytebuddy.utility.RandomString;
6 import net.javacrumbs.jsonunit.JsonAssert;
7 import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
8 import org.apache.commons.lang3.builder.ToStringStyle;
9 import org.hamcrest.BaseMatcher;
10 import org.hamcrest.CoreMatchers;
11 import org.hamcrest.Description;
12 import org.onap.simulator.presetGenerator.presets.BasePresets.BasePreset;
13 import org.onap.simulator.presetGenerator.presets.aai.PresetAAIGetSubscribersGet;
14 import org.onap.simulator.presetGenerator.presets.aai.PresetAAISearchNodeQueryEmptyResult;
15 import org.onap.simulator.presetGenerator.presets.ecompportal_att.PresetGetSessionSlotCheckIntervalGet;
16 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOAssignServiceInstanceGen2WithNames;
17 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOCreateServiceInstanceGen2WithNames;
18 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGet;
19 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOOrchestrationRequestGetErrorResponse;
20 import org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2ErrorResponse;
21 import org.onap.vid.model.asyncInstantiation.JobAuditStatus;
22 import org.onap.vid.model.asyncInstantiation.ServiceInfo;
23 import org.onap.vid.model.mso.MsoResponseWrapper2;
24 import org.springframework.core.ParameterizedTypeReference;
25 import org.springframework.http.HttpStatus;
26 import org.springframework.http.ResponseEntity;
27 import org.springframework.web.client.HttpClientErrorException;
28
29 import org.springframework.web.client.RestTemplate;
30 import org.testng.Assert;
31 import org.testng.annotations.DataProvider;
32 import org.testng.annotations.Test;
33 import vid.automation.test.infra.FeatureTogglingTest;
34 import vid.automation.test.infra.Features;
35 import vid.automation.test.infra.Wait;
36 import vid.automation.test.model.JobStatus;
37 import vid.automation.test.services.SimulatorApi;
38 import java.util.*;
39
40 import java.util.function.Predicate;
41 import java.util.stream.Collectors;
42 import java.util.stream.IntStream;
43 import java.util.stream.Stream;
44
45 import static java.lang.Boolean.FALSE;
46 import static java.lang.Boolean.TRUE;
47 import static java.util.stream.Collectors.*;
48 import static org.hamcrest.CoreMatchers.*;
49 import static org.hamcrest.MatcherAssert.assertThat;
50 import static org.hamcrest.Matchers.hasProperty;
51 import static org.hamcrest.Matchers.hasSize;
52 import static org.onap.simulator.presetGenerator.presets.mso.PresetMSOServiceInstanceGen2WithNames.Keys;
53 import static org.testng.Assert.assertNotNull;
54 import static org.testng.AssertJUnit.assertTrue;
55
56 @FeatureTogglingTest({Features.FLAG_ASYNC_JOBS, Features.FLAG_ASYNC_INSTANTIATION})
57 public class AsyncInstantiationApiTest extends BaseMsoApiTest {
58
59     private static final String CREATE_BULK_OF_MACRO_REQUEST = "asyncInstantiation/vidRequestCreateBulkOfMacro.json";
60
61     @DataProvider
62     public static Object[][] trueAndFalse() {
63             return new Object[][]{{TRUE},{FALSE}};
64     }
65
66     private String getCreateBulkUri() {
67         return uri.toASCIIString() + "/asyncInstantiation/bulk";
68     }
69
70     private String getHideServiceUri(String jobId) {
71         return uri.toASCIIString() + "/asyncInstantiation/hide/"+jobId;
72     }
73
74     private String getServiceInfoUrl() {
75         return uri.toASCIIString() + "/asyncInstantiation";
76     }
77
78     private String getJobAuditUrl() {
79         return uri.toASCIIString() + "/asyncInstantiation/auditStatus/{JOB_ID}?source={SOURCE}";
80     }
81
82     private String getDeleteServiceUrl(String uuid) {
83         return uri.toASCIIString() + "/asyncInstantiation/job/" + uuid;
84     }
85
86     public static class JobIdAndStatusMatcher extends BaseMatcher<ServiceInfo> {
87         private String expectedJobId;
88
89         public JobIdAndStatusMatcher(String expectedJobId) {
90             this.expectedJobId = expectedJobId;
91         }
92
93         @Override
94         public boolean matches(Object item) {
95             if (!(item instanceof ServiceInfo)) {
96                 return false;
97             }
98             ServiceInfo serviceInfo = (ServiceInfo) item;
99             return expectedJobId.equals(serviceInfo.jobId);
100         }
101
102         @Override
103         public void describeTo(Description description) {
104             description.appendText("failed to find job with uuid ")
105                     .appendValue(expectedJobId);
106         }
107     }
108
109
110
111     @Test
112     public void createBulkOfCreateInstances(){
113         Map<Keys, String> names = generateNames();
114         final int bulkSize = 3;
115         ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
116         createBulkOfInstancesAndAssert(presets, false, bulkSize, JobStatus.COMPLETED, names);
117     }
118
119     private Map<Keys,String> generateNames() {
120         return Stream.of(Keys.values()).collect(
121                 Collectors.toMap(x->x, x -> UUID.randomUUID().toString().replace("-","")));
122     }
123
124     private ImmutableList<BasePreset> addPresetsForCreateBulkOfCreateInstances(int bulkSize, Map<Keys, String> names){
125         ImmutableList<BasePreset> msoBulkPresets = IntStream.rangeClosed(1,bulkSize).
126                 mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
127                 .collect(ImmutableList.toImmutableList());
128         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
129                 .add(new PresetGetSessionSlotCheckIntervalGet())
130                 .add(new PresetAAIGetSubscribersGet())
131                 .add(new PresetAAISearchNodeQueryEmptyResult())
132                 .addAll(msoBulkPresets)
133                 .add(new PresetMSOOrchestrationRequestGet())
134                 .build();
135         return presets;
136
137     }
138
139     private ResponseEntity<List<JobAuditStatus>> auditStatusCall(String url) {
140         return restTemplate.exchange(
141                 url,
142                 org.springframework.http.HttpMethod.GET,
143                 null,
144                 new ParameterizedTypeReference<List<JobAuditStatus>>() {});
145     }
146
147     @DataProvider
148     public static Object[][] auditSources() {
149         return new Object[][]{{JobAuditStatus.SourceStatus.VID},{JobAuditStatus.SourceStatus.MSO}};
150     }
151
152
153     @Test(dataProvider = "auditSources")
154    public void getAuditStatus_nonExistingJobId_returnsEmptyList(JobAuditStatus.SourceStatus source){
155        List<JobAuditStatus> audits = getAuditStatuses(UUID.randomUUID().toString(), source.name());
156        Assert.assertEquals(audits.size(),0);
157    }
158
159     @Test(expectedExceptions = HttpClientErrorException.class)
160     public void getAuditStatus_nonExistingSource_returnsError() {
161         try {
162             getAuditStatuses(UUID.randomUUID().toString(), new RandomString(8).nextString());
163         } catch (HttpClientErrorException e) {
164             Assert.assertEquals(e.getResponseBodyAsString(),"The parameter source must have a value among : MSO, VID");
165             assertThat(e.getStatusCode(), is(HttpStatus.BAD_REQUEST));
166             throw e;
167         }
168     }
169
170     @Test()
171     public void simulateBulkRequest_getAuditStatus_auditStatusesReturnedAccordingSource() {
172         final int bulkSize = 2;
173         final List<String> jobIds = createBulkAndWaitForBeCompleted(bulkSize);
174
175         for(String jobId: jobIds) {
176             List<JobAuditStatus> actualVidAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.VID.name());
177             List<JobAuditStatus> expectedVidAudits = Stream.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, JobStatus.COMPLETED)
178                     .map(status->new JobAuditStatus(UUID.fromString(jobId),
179                             status.name(),
180                             JobAuditStatus.SourceStatus.VID,
181                             null,
182                             null,
183                             status.equals(JobStatus.COMPLETED))).collect(toList());
184             assertThat(actualVidAudits, is(expectedVidAudits));
185
186             List<JobAuditStatus> actualMsoAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
187             List<JobAuditStatus> expectedMsoAudits = Stream.of("REQUESTED", "COMPLETE")
188                     .map(status-> new JobAuditStatus(UUID.fromString(jobId),
189                             status,
190                             JobAuditStatus.SourceStatus.MSO,
191                             UUID.fromString("c0011670-0e1a-4b74-945d-8bf5aede1d9c"),
192                             status.equals("COMPLETE") ? "Service Instance was created successfully." : null,
193                             false)).collect(toList());
194             assertThat(actualMsoAudits, is(expectedMsoAudits));
195         }
196     }
197
198     protected List<String> createBulkAndWaitForBeCompleted(int bulkSize){
199         Map<Keys, String> names = generateNames();
200         ImmutableList<BasePreset> presets = addPresetsForCreateBulkOfCreateInstances(bulkSize, names);
201         final List<String> jobIds = createBulkOfInstances(presets, false, bulkSize, names);
202         Assert.assertEquals(jobIds.size(),bulkSize);
203
204         assertTrue(String.format("Not all services with ids: %s are in state completed after 30 sec",
205                 jobIds.stream().collect(joining(","))),
206
207                 Wait.waitFor(y-> serviceListCall().getBody().stream()
208                         .filter(si -> jobIds.contains(si.jobId))
209                         .filter(si -> si.jobStatus==JobStatus.COMPLETED)
210                         .count() == bulkSize,
211                 null, 30, 1 ));
212         return jobIds;
213     }
214
215     private List<JobAuditStatus> getAuditStatuses(String jobUUID, String source){
216         String url = getJobAuditUrl().replace("{JOB_ID}",jobUUID).replace("{SOURCE}", source);
217         ResponseEntity<List<JobAuditStatus>> statusesResponse = auditStatusCall(url);
218         assertThat(statusesResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
219         return statusesResponse.getBody();
220     }
221
222     @Test(expectedExceptions = HttpClientErrorException.class)
223     public void addBulkAndDeleteInProgress_deletionIsRejected(){
224         try {
225             final Map<String, JobStatus> jobs = addBulkAllPendingButOneInProgress();
226             deleteOneJobHavingTheStatus(jobs, JobStatus.IN_PROGRESS);
227         } catch (HttpClientErrorException e) {
228             JsonAssert.assertJsonPartEquals(
229                     "Service status does not allow deletion from the queue (Request id: null)",
230                     e.getResponseBodyAsString(),
231                     "message"
232             );
233             assertThat(e.getStatusCode(), is(HttpStatus.METHOD_NOT_ALLOWED));
234
235             throw e;
236         }
237     }
238
239     @Test
240     public void addBulkAndDeletePending_deletedIsHiddenFromServiceInfoResults(){
241         Map<String, JobStatus> statusesBefore = addBulkAllPendingButOneInProgress();
242
243         final String deletedJob = deleteOneJobHavingTheStatus(statusesBefore, JobStatus.PENDING);
244
245         final Map<String, JobStatus> statusesNow = serviceListCall().getBody().stream()
246                 .filter(si -> statusesBefore.keySet().contains(si.jobId))
247                 .collect(toMap(si -> si.jobId, si -> si.jobStatus));
248
249         statusesBefore.remove(deletedJob);
250         assertThat("deleted job shall not be present in StatusInfo response", statusesNow, is(statusesBefore));
251     }
252
253     private Map<String, JobStatus> addBulkAllPendingButOneInProgress(){
254         return addBulkAllPendingButOneInProgress(3);
255     }
256     
257     private Map<String, JobStatus> addBulkAllPendingButOneInProgress(int bulkSize){
258         Map<Keys, String> names = generateNames();
259         ImmutableList<BasePreset> msoBulkPresets = IntStream.rangeClosed(1,bulkSize)
260                 .mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
261                 .collect(ImmutableList.toImmutableList());
262         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
263                 .add(new PresetGetSessionSlotCheckIntervalGet())
264                 .add(new PresetAAISearchNodeQueryEmptyResult())
265                 .add(new PresetAAIGetSubscribersGet())
266                 .addAll(msoBulkPresets)
267                 .add(new PresetMSOOrchestrationRequestGet("IN_PROGRESS"))
268                 .build();
269         final List<String> jobIds = createBulkOfInstances(presets, false, bulkSize, names);
270
271         // wait for single IN_PROGRESS, so statuses will stop from changing
272         Wait.waitFor(foo -> serviceListCall().getBody().stream()
273                         .filter(si -> jobIds.contains(si.jobId))
274                         .anyMatch(si -> si.jobStatus.equals(JobStatus.IN_PROGRESS)),
275                 null, 20, 1);
276
277         final Map<String, JobStatus> statusMapBefore = serviceListCall().getBody().stream()
278                 .filter(si -> jobIds.contains(si.jobId))
279                 .collect(toMap(si -> si.jobId, si -> si.jobStatus));
280
281         assertThat(jobIds, hasSize(bulkSize));
282
283
284         return statusMapBefore;
285     }
286
287     private String deleteOneJobHavingTheStatus(Map<String, JobStatus> jobIdToStatus, JobStatus jobStatus) {
288         final String jobToDelete = jobIdToStatus.entrySet().stream()
289                 .filter(entry -> entry.getValue().equals(jobStatus))
290                 .map(Map.Entry::getKey)
291                 .findFirst().orElseThrow(() -> new AssertionError("no job in " + jobStatus + " state: " + jobIdToStatus));
292
293
294         restTemplate.delete(getDeleteServiceUrl(jobToDelete));
295
296         return jobToDelete;
297     }
298
299     @Test(invocationCount = 3)
300     public void createBulkOfCreateInstancesWithSinglePreset_firstOneInProgressOtherArePending(){
301         final int bulkSize = 3;
302         Map<String, JobStatus> statusMap = addBulkAllPendingButOneInProgress(bulkSize);
303         Set<String> jobIds = statusMap.keySet();
304
305         final Map<JobStatus, List<ServiceInfo>> statuses = serviceListCall().getBody().stream()
306                 .filter(si -> jobIds.contains(si.jobId))
307                 .collect(groupingBy(ServiceInfo::getJobStatus));
308
309         // Variable "statuses" contains two lists by status:
310         // IN_PROGRESS:  The ultimate first job - named with _001 - is always the only one in progress
311         // PENDING:      The other two jobs - named with _002 and _003 - are the still pending
312         assertThat(jobIds, hasSize(bulkSize));
313         assertThat(statuses.get(JobStatus.IN_PROGRESS), hasSize(1));
314         assertThat(statuses.get(JobStatus.IN_PROGRESS), everyItem(hasProperty("serviceInstanceName", endsWith("_001"))));
315
316         assertThat(statuses.get(JobStatus.PENDING), hasSize(bulkSize - 1));
317     }
318
319
320     @Test(dataProvider = "trueAndFalse" )
321     public void whenServiceInBulkFailed_otherServicesAreStopped(Boolean isPresetForCreate){
322         Map<Keys, String> names = generateNames();
323         final int bulkSize = 3;
324
325         //if there is a preset for create,  service shall failed during in_progress (upon get status)
326         //it there is no preset for create, service shall failed during pending (upon create request)
327         List<BasePreset> msoBulkPresets = isPresetForCreate ?
328                 IntStream.rangeClosed(1,bulkSize)
329                         .mapToObj(i-> new PresetMSOCreateServiceInstanceGen2WithNames(names, i))
330                         .collect(ImmutableList.toImmutableList()) :
331                 new LinkedList<>();
332         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
333                 .add(new PresetGetSessionSlotCheckIntervalGet())
334                 .add(new PresetAAIGetSubscribersGet())
335                 .add(new PresetAAISearchNodeQueryEmptyResult())
336                 .addAll(msoBulkPresets)
337                 .add(new PresetMSOOrchestrationRequestGet("FAILED"))
338                 .build();
339         List<String> jobIds = createBulkOfInstances(presets, false, bulkSize, names);
340         Assert.assertEquals(jobIds.size(),bulkSize);
341         boolean result = Wait.waitFor(x->{
342                 List<ServiceInfo> serviceInfoList = serviceListCall().getBody();
343                 Map<JobStatus, Long> statusCount = serviceInfoList.stream().filter(si->jobIds.contains(si.jobId)).collect(groupingBy(ServiceInfo::getJobStatus, counting()));
344                 return Objects.equals(statusCount.get(JobStatus.FAILED), 1L) && Objects.equals(statusCount.get(JobStatus.STOPPED), 2L);
345             }, null, 15, 1);
346         assertTrue(String.format("failed to get jobs [%s] to state of: 1 failed and 2 stopped ",
347                 String.join(",", jobIds)),result);
348     }
349
350     @Test
351     public void createBulkOfAssignInstances(){
352         Map<Keys, String> names = generateNames();
353         final int bulkSize = 2;
354         ImmutableList<BasePreset> msoBulkPresets = IntStream.rangeClosed(1,bulkSize)
355                 .mapToObj(i-> new PresetMSOAssignServiceInstanceGen2WithNames(names, i))
356                 .collect(ImmutableList.toImmutableList());
357         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
358                 .add(new PresetGetSessionSlotCheckIntervalGet())
359                 .add(new PresetAAIGetSubscribersGet())
360                 .add(new PresetAAISearchNodeQueryEmptyResult())
361                 .addAll(msoBulkPresets)
362                 .add(new PresetMSOOrchestrationRequestGet())
363                 .build();
364         createBulkOfInstancesAndAssert(presets, true, bulkSize, JobStatus.COMPLETED, names);
365     }  
366
367     @Test
368     public void tryToCreateBulkOfAssignInstancesErrorResponseFromMso(){
369         ImmutableList<BasePreset> presets = ImmutableList.of(
370                 new PresetGetSessionSlotCheckIntervalGet(),
371                 new PresetAAIGetSubscribersGet(),
372                 new PresetAAISearchNodeQueryEmptyResult(),
373                 new PresetMSOServiceInstanceGen2ErrorResponse(406));
374
375         List<String> jobIds = createBulkOfInstancesAndAssert(presets, true,1, JobStatus.FAILED, generateNames());
376         String jobId  = jobIds.get(0);
377         List<JobAuditStatus> actualMsoAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
378         JobAuditStatus expectedMsoAudit = new JobAuditStatus(UUID.fromString(jobId),"FAILED",JobAuditStatus.SourceStatus.MSO,
379                         null,
380                         "Http Code:406, \"messageId\":\"SVC0002\",\"text\":\"JSON Object Mapping Request\"" ,
381                         false);
382         assertThat(actualMsoAudits.get(0), is(expectedMsoAudit));
383     }
384
385     @Test
386     public void whenHideCompletedService_thenServiceNotReturnInServiceList(){
387         List<String> services = createBulkAndWaitForBeCompleted(2);
388         hideService(services.get(0));
389         List<String> serviceInfoList = serviceListCall().getBody().stream().map(ServiceInfo::getJobId).collect(toList());
390         assertThat(serviceInfoList, hasItem(services.get(1)));
391         assertThat(serviceInfoList, not(hasItem(services.get(0))));
392     }
393
394     private MsoResponseWrapper2 hideService(String jobId) {
395         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getHideServiceUri(jobId), "");
396         return responseWrapper2;
397     }
398
399     private List<String> createBulkOfInstancesAndAssert(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, JobStatus finalState, Map<Keys, String> names){
400         List<String> jobIds = createBulkOfInstances(presets, isPause, bulkSize, names);
401         Assert.assertEquals(jobIds.size(),bulkSize);
402         for(String jobId: jobIds) {
403             ServiceInfo expectedServiceInfo = new ServiceInfo("ab2222", JobStatus.IN_PROGRESS, isPause, "someID",
404                     "someName", "myProject", "NFT1", "NFTJSSSS-NFT1", "greatTenant", "greatTenant", "mtn3", null,
405                     "mySubType", "a9a77d5a-123e-4ca2-9eb9-0b015d2ee0fb", null, names.get(Keys.SERVICE_NAME),
406                     "300adb1e-9b0c-4d52-bfb5-fa5393c4eabb", "AIM_TRANSPORT_00004", "1.0", jobId, null);
407             JobInfoChecker jobInfoChecker = new JobInfoChecker(
408                     restTemplate, ImmutableSet.of(JobStatus.PENDING, JobStatus.IN_PROGRESS, finalState), jobId, expectedServiceInfo);
409             boolean result = jobInfoChecker.test(null);
410             assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
411
412             jobInfoChecker.setExpectedJobStatus(ImmutableSet.of(finalState));
413             if (ImmutableList.of(JobStatus.COMPLETED, JobStatus.PAUSE).contains(finalState)) {
414                 expectedServiceInfo.serviceInstanceId = "f8791436-8d55-4fde-b4d5-72dd2cf13cfb";
415             }
416             result = Wait.waitFor(jobInfoChecker, null, 20, 1);
417             assertTrue("service info of jobId: " + jobId + " was in status: " + jobInfoChecker.lastStatus, result);
418         }
419
420         return jobIds;
421     }
422
423     private List<String> createBulkOfInstances(ImmutableList<BasePreset> presets, boolean isPause, int bulkSize, Map<Keys, String> names){
424
425         SimulatorApi.registerExpectationFromPresets(presets, SimulatorApi.RegistrationStrategy.CLEAR_THEN_SET);
426
427         String requestBody = TestUtils.convertRequest(objectMapper, CREATE_BULK_OF_MACRO_REQUEST);
428         requestBody = requestBody.replace("\"IS_PAUSE_VALUE\"", String.valueOf(isPause)).replace("\"BULK_SIZE\"", String.valueOf(bulkSize));
429         for (Map.Entry<Keys, String> e : names.entrySet()) {
430             requestBody = requestBody.replace(e.getKey().name(), e.getValue());
431         }
432         MsoResponseWrapper2 responseWrapper2 = callMsoForResponseWrapper(org.springframework.http.HttpMethod.POST, getCreateBulkUri(), requestBody);
433         assertNotNull(responseWrapper2);
434         return (List<String>)responseWrapper2.getEntity();
435     }
436
437     public class JobInfoChecker<Integer> implements Predicate<Integer> {
438
439         private final RestTemplate restTemplate;
440         private Set<JobStatus> expectedJobStatus;
441         private ServiceInfo expectedServiceInfo;
442         private final String jobId;
443         private JobStatus lastStatus;
444
445         public JobInfoChecker(RestTemplate restTemplate, Set<JobStatus> expectedJobStatus, String jobId, ServiceInfo expectedServiceInfo) {
446             this.restTemplate = restTemplate;
447             this.expectedJobStatus = expectedJobStatus;
448             this.jobId = jobId;
449             this.expectedServiceInfo = expectedServiceInfo;
450         }
451
452         public void setExpectedJobStatus(Set<JobStatus> expectedJobStatus) {
453             this.expectedJobStatus = expectedJobStatus;
454         }
455
456         @Override
457         public boolean test(Integer integer) {
458             ResponseEntity<List<ServiceInfo>> serviceListResponse = serviceListCall();
459             assertThat(serviceListResponse.getStatusCode(), CoreMatchers.equalTo(HttpStatus.OK));
460             assertThat(serviceListResponse.getBody(), hasItem(new JobIdAndStatusMatcher(jobId)));
461             ServiceInfo serviceInfoFromDB = serviceListResponse.getBody().stream()
462                     .filter(serviceInfo -> serviceInfo.jobId.equals(jobId))
463                     .findFirst().orElse(null);
464             Assert.assertNotNull(serviceInfoFromDB);
465             Assert.assertEquals(serviceInfoDataReflected(serviceInfoFromDB), serviceInfoDataReflected(expectedServiceInfo));
466             assertTrue("actual service instance doesn't contain template service name:" + expectedServiceInfo.serviceInstanceName,
467                     serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName));
468             if (serviceInfoFromDB.jobStatus==JobStatus.IN_PROGRESS || serviceInfoFromDB.jobStatus==JobStatus.COMPLETED) {
469                 assertTrue("actual service instance doesn't contain template service name and trailing numbers:" + expectedServiceInfo.serviceInstanceName,
470                         serviceInfoFromDB.serviceInstanceName.contains(expectedServiceInfo.serviceInstanceName+"_00"));
471             }
472
473             if (expectedServiceInfo.serviceInstanceId != null) {
474                 assertThat(serviceInfoFromDB.serviceInstanceId, is(expectedServiceInfo.serviceInstanceId));
475             }
476             lastStatus = serviceInfoFromDB.jobStatus;
477             return expectedJobStatus.contains(serviceInfoFromDB.jobStatus);
478         }
479     }
480
481     private ResponseEntity<List<ServiceInfo>> serviceListCall() {
482         return restTemplate.exchange(
483                 getServiceInfoUrl(),
484                 org.springframework.http.HttpMethod.GET,
485                 null,
486                 new ParameterizedTypeReference<List<ServiceInfo>>() {});
487     }
488
489     //serialize fields except of fields we cannot know ahead of time
490     private static String serviceInfoDataReflected(ServiceInfo service1) {
491         return new ReflectionToStringBuilder(service1, ToStringStyle.SHORT_PREFIX_STYLE)
492                 .setExcludeFieldNames("jobStatus", "templateId", "statusModifiedDate", "createdBulkDate", "serviceInstanceId", "serviceInstanceName")
493                 .toString();
494     }
495
496     @Test
497     public void errorResponseInGetStatusFromMso_getAuditStatusFromMso_errorMsgExistInAdditionalInfo(){
498         Map<Keys, String> names = generateNames();
499         ImmutableList<BasePreset> presets = new ImmutableList.Builder<BasePreset>()
500                 .add(new PresetGetSessionSlotCheckIntervalGet())
501                 .add(new PresetAAIGetSubscribersGet())
502                 .add(new PresetAAISearchNodeQueryEmptyResult())
503                 .add(new PresetMSOAssignServiceInstanceGen2WithNames(names, 1))
504                 .add(new PresetMSOOrchestrationRequestGetErrorResponse(406))
505                 .build();
506
507         final List<String> jobIds = createBulkOfInstancesAndAssert(presets, true,1, JobStatus.IN_PROGRESS, names);
508         String jobId = jobIds.get(0);
509         Wait.waitFor(y-> getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name()).stream()
510                         .anyMatch(si -> si.getJobStatus().equals("FAILED")),
511                 null, 10, 1 );
512         List<JobAuditStatus> actualMsoAudits = getAuditStatuses(jobId, JobAuditStatus.SourceStatus.MSO.name());
513         List<JobAuditStatus> expectedMsoAudits = Stream.of("REQUESTED", "FAILED")
514                 .map(status -> new JobAuditStatus(UUID.fromString(jobId),
515                         status,
516                         JobAuditStatus.SourceStatus.MSO,
517                         UUID.fromString("c0011670-0e1a-4b74-945d-8bf5aede1d9c"),
518                         status.equals("FAILED") ? "Http Code:406, \"messageId\":\"SVC0002\",\"text\":\"JSON Object Mapping Request\"" : null,
519                         false)).collect(toList());
520         assertThat(actualMsoAudits, is(expectedMsoAudits));
521
522     }
523
524 }