63b5af876ddc8087825a2593d0b48f29d8507504
[ccsdk/oran.git] /
1 /*-
2  * ========================LICENSE_START=================================
3  * ONAP : ccsdk oran
4  * ======================================================================
5  * Copyright (C) 2019-2022 Nordix Foundation. All rights reserved.
6  * ======================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ========================LICENSE_END===================================
19  */
20
21 package org.onap.ccsdk.oran.a1policymanagementservice.controllers.v2;
22
23 import static org.assertj.core.api.Assertions.assertThat;
24 import static org.awaitility.Awaitility.await;
25 import static org.junit.jupiter.api.Assertions.assertEquals;
26 import static org.junit.jupiter.api.Assertions.assertTrue;
27 import static org.mockito.ArgumentMatchers.any;
28 import static org.mockito.Mockito.doReturn;
29
30 import com.google.gson.Gson;
31 import com.google.gson.GsonBuilder;
32
33 import java.io.FileOutputStream;
34 import java.io.PrintStream;
35 import java.lang.invoke.MethodHandles;
36 import java.nio.charset.StandardCharsets;
37 import java.nio.file.Files;
38 import java.nio.file.Path;
39 import java.nio.file.Paths;
40 import java.time.Duration;
41 import java.time.Instant;
42 import java.util.ArrayList;
43 import java.util.Collections;
44 import java.util.List;
45
46 import org.json.JSONObject;
47 import org.junit.jupiter.api.AfterAll;
48 import org.junit.jupiter.api.AfterEach;
49 import org.junit.jupiter.api.BeforeEach;
50 import org.junit.jupiter.api.DisplayName;
51 import org.junit.jupiter.api.MethodOrderer;
52 import org.junit.jupiter.api.Test;
53 import org.junit.jupiter.api.TestMethodOrder;
54 import org.onap.ccsdk.oran.a1policymanagementservice.clients.A1ClientFactory;
55 import org.onap.ccsdk.oran.a1policymanagementservice.clients.AsyncRestClient;
56 import org.onap.ccsdk.oran.a1policymanagementservice.clients.AsyncRestClientFactory;
57 import org.onap.ccsdk.oran.a1policymanagementservice.clients.SecurityContext;
58 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationConfig;
59 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationConfig.RicConfigUpdate;
60 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.RicConfig;
61 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.WebClientConfig;
62 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.OpenPolicyAgentSimulatorController;
63 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.ServiceCallbackInfo;
64 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.authorization.PolicyAuthorizationRequest;
65 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.authorization.PolicyAuthorizationRequest.Input.AccessType;
66 import org.onap.ccsdk.oran.a1policymanagementservice.exceptions.ServiceException;
67 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Lock;
68 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Lock.LockType;
69 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policies;
70 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policy;
71 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyType;
72 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyTypes;
73 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric;
74 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric.RicState;
75 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Rics;
76 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Service;
77 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Services;
78 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.RefreshConfigTask;
79 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.RicSupervision;
80 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.ServiceSupervision;
81 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1Client;
82 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1ClientFactory;
83 import org.slf4j.Logger;
84 import org.slf4j.LoggerFactory;
85 import org.springframework.beans.factory.annotation.Autowired;
86 import org.springframework.boot.test.context.SpringBootTest;
87 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
88 import org.springframework.boot.test.context.TestConfiguration;
89 import org.springframework.boot.test.web.server.LocalServerPort;
90 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
91 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
92 import org.springframework.context.ApplicationContext;
93 import org.springframework.context.annotation.Bean;
94 import org.springframework.http.HttpStatus;
95 import org.springframework.http.MediaType;
96 import org.springframework.http.ResponseEntity;
97 import org.springframework.test.context.TestPropertySource;
98 import org.springframework.util.FileSystemUtils;
99 import org.springframework.web.reactive.function.client.WebClientRequestException;
100 import org.springframework.web.reactive.function.client.WebClientResponseException;
101
102 import reactor.core.publisher.Mono;
103 import reactor.test.StepVerifier;
104 import reactor.util.annotation.Nullable;
105
106 @TestMethodOrder(MethodOrderer.MethodName.class)
107 @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
108 @TestPropertySource(properties = { //
109         "server.ssl.key-store=./config/keystore.jks", //
110         "app.webclient.trust-store=./config/truststore.jks", //
111         "app.webclient.trust-store-used=true", //
112         "app.vardata-directory=/tmp/pmstest", //
113         "app.filepath=", //
114         "app.s3.bucket=" // If this is set, S3 will be used to store data.
115 })
116 class ApplicationTest {
117     private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
118
119     @Autowired
120     ApplicationContext context;
121
122     @Autowired
123     private Rics rics;
124
125     @Autowired
126     private Policies policies;
127
128     @Autowired
129     private PolicyTypes policyTypes;
130
131     @Autowired
132     MockA1ClientFactory a1ClientFactory;
133
134     @Autowired
135     RicSupervision supervision;
136
137     @Autowired
138     ApplicationConfig applicationConfig;
139
140     @Autowired
141     Services services;
142
143     @Autowired
144     RappSimulatorController rAppSimulator;
145
146     @Autowired
147     RefreshConfigTask refreshConfigTask;
148
149     @Autowired
150     SecurityContext securityContext;
151
152     @Autowired
153     OpenPolicyAgentSimulatorController openPolicyAgentSimulatorController;
154
155     private static Gson gson = new GsonBuilder().create();
156
157     /**
158      * Overrides the BeanFactory.
159      */
160     @TestConfiguration
161     static class TestBeanFactory {
162
163         @Bean
164         A1ClientFactory getA1ClientFactory(@Autowired ApplicationConfig appConfig, @Autowired PolicyTypes types) {
165             return new MockA1ClientFactory(appConfig, types);
166         }
167
168         @Bean
169         public ServiceSupervision getServiceSupervision(@Autowired Services services,
170                 @Autowired A1ClientFactory a1ClientFactory, @Autowired Policies policies) {
171             Duration checkInterval = Duration.ofMillis(1);
172             return new ServiceSupervision(services, policies, a1ClientFactory, checkInterval);
173         }
174
175         @Bean
176         public ServletWebServerFactory servletContainer() {
177             return new TomcatServletWebServerFactory();
178         }
179     }
180
181     @LocalServerPort
182     private int port;
183
184     @BeforeEach
185     void init() {
186         this.applicationConfig.setAuthProviderUrl(baseUrl() + OpenPolicyAgentSimulatorController.ACCESS_CONTROL_URL);
187     }
188
189     @AfterEach
190     void reset() {
191         rics.clear();
192         policies.clear();
193         policyTypes.clear();
194         services.clear();
195         a1ClientFactory.reset();
196         this.rAppSimulator.getTestResults().clear();
197         this.a1ClientFactory.setPolicyTypes(policyTypes); // Default same types in RIC and in this app
198         this.securityContext.setAuthTokenFilePath(null);
199         this.openPolicyAgentSimulatorController.getTestResults().reset();
200     }
201
202     @AfterAll
203     static void clearTestDir() {
204         try {
205             FileSystemUtils.deleteRecursively(Path.of("/tmp/pmstest"));
206         } catch (Exception e) {
207             logger.warn("Could test directory : {}", e.getMessage());
208         }
209     }
210
211     @AfterEach
212     void verifyNoRicLocks() {
213         for (Ric ric : this.rics.getRics()) {
214             Lock.Grant grant = ric.getLock().lockBlocking(LockType.EXCLUSIVE, "verifyNoRicLocks");
215             grant.unlockBlocking();
216             assertThat(ric.getLock().getLockCounter()).isZero();
217             assertThat(ric.getState()).isEqualTo(Ric.RicState.AVAILABLE);
218         }
219     }
220
221     @Test
222     @DisplayName("test ZZ Actuator")
223     void testZZActuator() throws Exception {
224         // The test must be run last, hence the "ZZ" in the name. All succeeding tests
225         // will fail.
226         AsyncRestClient client = restClient(baseUrl(), false);
227
228         client.post("/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice",
229                 "{\"configuredLevel\":\"trace\"}").block();
230
231         String resp = client.get("/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice").block();
232         assertThat(resp).contains("TRACE");
233
234         client.post("/actuator/loggers/org.springframework.boot.actuate", "{\"configuredLevel\":\"trace\"}").block();
235
236         // This will stop the web server and all coming tests will fail.
237         ResponseEntity<String> entity = client.postForEntity("/actuator/shutdown", "").block();
238         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.OK);
239         assertThat(((String) entity.getBody())).contains("Shutting down");
240     }
241
242     @Test
243     @DisplayName("test generate Api Doc")
244     void generateApiDoc() throws Exception {
245         String url = "https://localhost:" + this.port + "/v3/api-docs";
246         ResponseEntity<String> resp = restClient("", false).getForEntity(url).block();
247         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
248         JSONObject jsonObj = new JSONObject(resp.getBody());
249         assertThat(jsonObj.remove("servers")).isNotNull();
250
251         String indented = (jsonObj).toString(4);
252         String docDir = "api/";
253         Files.createDirectories(Paths.get(docDir));
254         try (PrintStream out = new PrintStream(new FileOutputStream(docDir + "pms-api.json"))) {
255             out.print(indented);
256         }
257     }
258
259     @Test
260     @DisplayName("test Persistency Policies")
261     void testPersistencyPolicies() throws Exception {
262         Ric ric = this.addRic("ric1");
263         PolicyType type = this.addPolicyType("type1", ric.id());
264
265         final int noOfPolicies = 100;
266         for (int i = 0; i < noOfPolicies; ++i) {
267             addPolicy("id" + i, type.getId(), "service", ric.id());
268         }
269         waitforS3();
270
271         {
272             Policies policiesLocal = new Policies(this.applicationConfig);
273             policiesLocal.restoreFromDatabase(ric, this.policyTypes).blockLast();
274             assertThat(policiesLocal.size()).isEqualTo(noOfPolicies);
275         }
276
277         {
278             restClient().delete("/policies/id2").block();
279             Policies policiesAfterDelete = new Policies(this.applicationConfig);
280             policiesAfterDelete.restoreFromDatabase(ric, this.policyTypes).blockLast();
281             assertThat(policiesAfterDelete.size()).isEqualTo(noOfPolicies - 1);
282         }
283     }
284
285     @Test
286     @DisplayName("test Persistency Policy Types")
287     void testPersistencyPolicyTypes() throws Exception {
288         Ric ric = this.addRic("ric1");
289         this.addPolicyType("type1", ric.id());
290         waitforS3();
291
292         PolicyTypes types = new PolicyTypes(this.applicationConfig);
293         types.restoreFromDatabase().blockLast();
294         assertThat(types.size()).isEqualTo(1);
295     }
296
297     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
298     private void waitforS3() throws Exception {
299         if (applicationConfig.isS3Enabled()) {
300             Thread.sleep(1000);
301         }
302     }
303
304     @Test
305     @DisplayName("test Persistency Service")
306     void testPersistencyService() throws Exception {
307         final String SERVICE = "serviceName";
308         putService(SERVICE, 1234, HttpStatus.CREATED);
309         assertThat(this.services.size()).isEqualTo(1);
310         Service service = this.services.getService(SERVICE);
311         waitforS3();
312
313         Services servicesRestored = new Services(this.applicationConfig);
314         servicesRestored.restoreFromDatabase().blockLast();
315         Service serviceRestored = servicesRestored.getService(SERVICE);
316         assertThat(servicesRestored.size()).isEqualTo(1);
317         assertThat(serviceRestored.getCallbackUrl()).isEqualTo(service.getCallbackUrl());
318         assertThat(serviceRestored.getKeepAliveInterval()).isEqualTo(service.getKeepAliveInterval());
319
320         // check that the service can be deleted
321         this.services.remove(SERVICE);
322         servicesRestored = new Services(this.applicationConfig);
323         assertThat(servicesRestored.size()).isZero();
324     }
325
326     @Test
327     @DisplayName("test Adding Ric From Configuration")
328     void testAddingRicFromConfiguration() throws Exception {
329         // Test adding the RIC from configuration
330
331         final String RIC = "ric1";
332         final String TYPE = "type123";
333         PolicyTypes nearRtRicPolicyTypes = new PolicyTypes(this.applicationConfig);
334         nearRtRicPolicyTypes.put(createPolicyType(TYPE));
335         this.a1ClientFactory.setPolicyTypes(nearRtRicPolicyTypes);
336
337         putService("service");
338
339         refreshConfigTask.handleUpdatedRicConfig( //
340                 new RicConfigUpdate(ricConfig(RIC, "me1"), RicConfigUpdate.Type.ADDED)) //
341                 .block();
342         waitForRicState(RIC, RicState.AVAILABLE);
343
344         // Test that the type has been synched
345         Ric addedRic = this.rics.getRic(RIC);
346         assertThat(addedRic.getSupportedPolicyTypes()).hasSize(1);
347         assertThat(addedRic.getSupportedPolicyTypes().iterator().next().getId()).isEqualTo(TYPE);
348
349         // Check that a service callback for the AVAILABLE RIC is invoked
350         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
351         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
352         ServiceCallbackInfo callbackInfo = receivedCallbacks.getReceivedInfo().get(0);
353         assertThat(callbackInfo.ricId).isEqualTo(RIC);
354         assertThat(callbackInfo.eventType).isEqualTo(ServiceCallbackInfo.EventType.AVAILABLE);
355     }
356
357     @Test
358     @DisplayName("test Adding Ric From Configuration non Responding Ric")
359     void testAddingRicFromConfiguration_nonRespondingRic() throws Exception {
360         putService("service");
361
362         final String RIC = "NonRespondingRic";
363         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client(RIC);
364         doReturn(MockA1Client.monoError("error", HttpStatus.BAD_GATEWAY)).when(a1Client).getPolicyTypeIdentities();
365
366         refreshConfigTask.handleUpdatedRicConfig( //
367                 new RicConfigUpdate(ricConfig(RIC, "me1"), RicConfigUpdate.Type.ADDED)) //
368                 .block();
369
370         waitForRicState(RIC, RicState.UNAVAILABLE);
371
372         // Check that no service callback for the UNAVAILABLE RIC is invoked
373         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
374         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).isEmpty());
375
376         // Run a synch and check that the AVAILABLE notification is received
377         a1ClientFactory.reset();
378         supervision.checkAllRics();
379         waitForRicState(RIC, RicState.AVAILABLE);
380
381         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
382     }
383
384     @Test
385     @DisplayName("test Trust Validation")
386     void testTrustValidation() {
387         addRic("ric1");
388
389         String rsp = restClient(true).get("/rics").block(); // restClient(true) enables trust validation
390         assertThat(rsp).contains("ric1");
391
392     }
393
394     @Test
395     @DisplayName("test Get Rics")
396     void testGetRics() {
397         addRic("ric1");
398         this.addPolicyType("type1", "ric1");
399         String url = "/rics?policytype_id=type1";
400         String rsp = restClient().get(url).block();
401         assertThat(rsp).contains("ric1");
402
403         // nameless type for ORAN A1 1.1
404         addRic("ric2");
405         this.addPolicyType("", "ric2");
406         url = "/rics?policytype_id=";
407         rsp = restClient().get(url).block();
408         assertThat(rsp).contains("ric2") //
409                 .doesNotContain("ric1") //
410                 .contains("AVAILABLE");
411
412         // All RICs
413         rsp = restClient().get("/rics").block();
414         assertThat(rsp).contains("ric2") //
415                 .contains("ric1");
416
417         // Non existing policy type
418         url = "/rics?policytype_id=XXXX";
419         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
420     }
421
422     @Test
423     @DisplayName("test Synchronization")
424     void testSynchronization() throws Exception {
425         // Two polictypes will be put in the NearRT RICs
426         PolicyTypes nearRtRicPolicyTypes = new PolicyTypes(this.applicationConfig);
427         nearRtRicPolicyTypes.put(createPolicyType("typeName"));
428         nearRtRicPolicyTypes.put(createPolicyType("typeName2"));
429         this.a1ClientFactory.setPolicyTypes(nearRtRicPolicyTypes);
430
431         // One type and one instance added to the Policy Management Service's storage
432         final String ric1Name = "ric1";
433         Ric ric1 = addRic(ric1Name);
434         Policy policy2 = addPolicy("policyId2", "typeName", "service", ric1Name);
435         Ric ric2 = addRic("ric2");
436
437         getA1Client(ric1Name).putPolicy(policy2); // put it in the RIC (Near-RT RIC)
438         policies.remove(policy2); // Remove it from the repo -> should be deleted in the RIC
439
440         String policyId = "policyId";
441         Policy policy = addPolicy(policyId, "typeName", "service", ric1Name); // This should be created in the RIC
442         supervision.checkAllRics(); // The created policy should be put in the RIC
443
444         // Wait until synch is completed
445         waitForRicState(ric1Name, RicState.SYNCHRONIZING);
446         waitForRicState(ric1Name, RicState.AVAILABLE);
447         waitForRicState("ric2", RicState.AVAILABLE);
448
449         Policies ricPolicies = getA1Client(ric1Name).getPolicies();
450         assertThat(ricPolicies.size()).isEqualTo(1);
451         Policy ricPolicy = ricPolicies.get(policyId);
452         assertThat(ricPolicy.getJson()).isEqualTo(policy.getJson());
453
454         // Both types should be in the Policy Management Service's storage after the
455         // synch
456         assertThat(ric1.getSupportedPolicyTypes()).hasSize(2);
457         assertThat(ric2.getSupportedPolicyTypes()).hasSize(2);
458     }
459
460     @Test
461     @DisplayName("test Get Ric")
462     void testGetRic() throws Exception {
463         String ricId = "ric1";
464         String managedElementId = "kista_1";
465         addRic(ricId, managedElementId);
466
467         String url = "/rics/ric?managed_element_id=" + managedElementId;
468         String rsp = restClient().get(url).block();
469         RicInfo ricInfo = gson.fromJson(rsp, RicInfo.class);
470         assertThat(ricInfo.ricId).isEqualTo(ricId);
471
472         url = "/rics/ric?ric_id=" + ricId;
473         rsp = restClient().get(url).block();
474         ricInfo = gson.fromJson(rsp, RicInfo.class);
475         assertThat(ricInfo.ricId).isEqualTo(ricId);
476
477         // test GET RIC for ManagedElement that does not exist
478         url = "/rics/ric?managed_element_id=" + "junk";
479         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
480
481         url = "/rics/ric";
482         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST);
483     }
484
485     private String putPolicyBody(String serviceName, String ricId, String policyTypeName, String policyInstanceId,
486             boolean isTransient, String statusNotificationUri) {
487         PolicyInfo info = new PolicyInfo();
488         info.policyId = policyInstanceId;
489         info.policyTypeId = policyTypeName;
490         info.ricId = ricId;
491         info.serviceId = serviceName;
492         info.policyData = gson.fromJson(jsonString(), Object.class);
493
494         if (isTransient) {
495             info.isTransient = isTransient;
496         }
497         info.statusNotificationUri = statusNotificationUri;
498         info.validate();
499         return gson.toJson(info);
500     }
501
502     private String putPolicyBody(String serviceName, String ricId, String policyTypeName, String policyInstanceId) {
503         return putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId, false, "statusUri");
504     }
505
506     @Test
507     @DisplayName("test Put Policy")
508     void testPutPolicy() throws Exception {
509         String serviceName = "service.1";
510         String ricId = "ric.1";
511         String policyTypeName = "type1_1.2.3";
512         String policyInstanceId = "instance_1.2.3";
513
514         putService(serviceName);
515         addPolicyType(policyTypeName, ricId);
516
517         // PUT a transient policy
518         String url = "/policies";
519         String policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId, true, "statusNotif");
520         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
521
522         restClient().put(url, policyBody).block();
523         {
524             // Check the authorization request
525             OpenPolicyAgentSimulatorController.TestResults res =
526                     this.openPolicyAgentSimulatorController.getTestResults();
527             assertThat(res.receivedRequests).hasSize(1);
528             PolicyAuthorizationRequest req = res.receivedRequests.get(0);
529             assertThat(req.getInput().getAccessType()).isEqualTo(AccessType.WRITE);
530             assertThat(req.getInput().getPolicyTypeId()).isEqualTo(policyTypeName);
531         }
532
533         Policy policy = policies.getPolicy(policyInstanceId);
534         assertThat(policy).isNotNull();
535         assertThat(policy.getId()).isEqualTo(policyInstanceId);
536         assertThat(policy.getOwnerServiceId()).isEqualTo(serviceName);
537         assertThat(policy.getRic().id()).isEqualTo(ricId);
538         assertThat(policy.isTransient()).isTrue();
539
540         // Put a non transient policy
541         policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId);
542         restClient().put(url, policyBody).block();
543         policy = policies.getPolicy(policyInstanceId);
544         assertThat(policy.isTransient()).isFalse();
545
546         url = "/policy-instances";
547         String rsp = restClient().get(url).block();
548         assertThat(rsp).as("Response contains policy instance ID.").contains(policyInstanceId);
549
550         url = "/policies/" + policyInstanceId;
551         rsp = restClient().get(url).block();
552         assertThat(rsp).contains(policyBody);
553
554         // Test of error codes
555         url = "/policies";
556         policyBody = putPolicyBody(serviceName, ricId + "XX", policyTypeName, policyInstanceId);
557         testErrorCode(restClient().put(url, policyBody), HttpStatus.NOT_FOUND);
558
559         policyBody = putPolicyBody(serviceName, ricId, policyTypeName + "XX", policyInstanceId);
560         addPolicyType(policyTypeName + "XX", "otherRic");
561         testErrorCode(restClient().put(url, policyBody), HttpStatus.NOT_FOUND);
562
563         policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId);
564         this.rics.getRic(ricId).setState(Ric.RicState.SYNCHRONIZING);
565         testErrorCode(restClient().put(url, policyBody), HttpStatus.LOCKED);
566         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
567     }
568
569     @Test
570     void testFineGrainedAuth() throws Exception {
571         final String POLICY_ID = "policyId";
572         final String RIC_ID = "ric1";
573         final String TYPE_ID = "typeName";
574         addPolicy(POLICY_ID, TYPE_ID, null, RIC_ID);
575         assertThat(policies.size()).isEqualTo(1);
576
577         this.applicationConfig
578                 .setAuthProviderUrl(baseUrl() + OpenPolicyAgentSimulatorController.ACCESS_CONTROL_URL_REJECT);
579
580         String url = "/policy-instances";
581         String rsp = restClient().get(url).block();
582         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
583
584         url = "/policies/" + POLICY_ID;
585         testErrorCode(restClient().delete(url), HttpStatus.UNAUTHORIZED, "Not authorized");
586
587         url = "/policies";
588         String policyBody = putPolicyBody(null, RIC_ID, TYPE_ID, POLICY_ID, false, null);
589         testErrorCode(restClient().put(url, policyBody), HttpStatus.UNAUTHORIZED, "Not authorized");
590
591         rsp = restClient().get(url).block();
592         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
593     }
594
595     @Test
596     void testFineGrainedAuth_OPA_UNAVALIABLE() throws Exception {
597         final String POLICY_ID = "policyId";
598         final String RIC_ID = "ric1";
599         final String TYPE_ID = "typeName";
600         addPolicy(POLICY_ID, TYPE_ID, null, RIC_ID);
601         assertThat(policies.size()).isEqualTo(1);
602
603         this.applicationConfig.setAuthProviderUrl("junk");
604
605         String url = "/policy-instances";
606         String rsp = restClient().get(url).block();
607         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
608
609         url = "/policies/" + POLICY_ID;
610         testErrorCode(restClient().delete(url), HttpStatus.UNAUTHORIZED, "Not authorized");
611
612         url = "/policies";
613         String policyBody = putPolicyBody(null, RIC_ID, TYPE_ID, POLICY_ID, false, null);
614         testErrorCode(restClient().put(url, policyBody), HttpStatus.UNAUTHORIZED, "Not authorized");
615
616         rsp = restClient().get(url).block();
617         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
618     }
619
620     @Test
621     @DisplayName("test Put Policy No Service No Status Uri")
622     void testPutPolicy_NoServiceNoStatusUri() throws Exception {
623         String ricId = "ric.1";
624         String policyTypeName = "type1_1.2.3";
625         String policyInstanceId = "instance_1.2.3";
626
627         addPolicyType(policyTypeName, ricId);
628
629         // PUT a transient policy
630         String url = "/policies";
631         String policyBody = putPolicyBody(null, ricId, policyTypeName, policyInstanceId, true, null);
632         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
633
634         restClient().put(url, policyBody).block();
635
636         Policy policy = policies.getPolicy(policyInstanceId);
637         assertThat(policy).isNotNull();
638         assertThat(policy.getOwnerServiceId()).isBlank();
639         assertThat(policy.getStatusNotificationUri()).isBlank();
640     }
641
642     @Test
643     /**
644      * Test that HttpStatus and body from failing REST call to A1 is passed on to
645      * the caller.
646      *
647      * @throws ServiceException
648      */
649     @DisplayName("test Error From Ric")
650     void testErrorFromRic() throws ServiceException {
651         putService("service1");
652         addPolicyType("type1", "ric1");
653
654         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client("ric1");
655         HttpStatus httpStatus = HttpStatus.INTERNAL_SERVER_ERROR;
656         String responseBody = "Refused";
657         byte[] responseBodyBytes = responseBody.getBytes(StandardCharsets.UTF_8);
658
659         WebClientResponseException a1Exception = new WebClientResponseException(httpStatus.value(), "statusText", null,
660                 responseBodyBytes, StandardCharsets.UTF_8, null);
661         doReturn(Mono.error(a1Exception)).when(a1Client).putPolicy(any());
662
663         // PUT Policy
664         String putBody = putPolicyBody("service1", "ric1", "type1", "id1");
665         String url = "/policies";
666         testErrorCode(restClient().put(url, putBody), httpStatus, responseBody);
667
668         // DELETE POLICY
669         this.addPolicy("instance1", "type1", "service1", "ric1");
670         doReturn(Mono.error(a1Exception)).when(a1Client).deletePolicy(any());
671         testErrorCode(restClient().delete("/policies/instance1"), httpStatus, responseBody);
672
673     }
674
675     @Test
676     @DisplayName("test Put Typeless Policy")
677     void testPutTypelessPolicy() throws Exception {
678         putService("service1");
679         addPolicyType("", "ric1");
680         String body = putPolicyBody("service1", "ric1", "", "id1");
681         restClient().put("/policies", body).block();
682
683         String rsp = restClient().get("/policy-instances").block();
684         PolicyInfoList info = gson.fromJson(rsp, PolicyInfoList.class);
685         assertThat(info.policies).hasSize(1);
686         PolicyInfo policyInfo = info.policies.iterator().next();
687         assertThat(policyInfo.policyId).isEqualTo("id1");
688         assertThat(policyInfo.policyTypeId).isEmpty();
689     }
690
691     @Test
692     @DisplayName("test Update Service")
693     void testUpdateService() throws Exception {
694         this.addRic("ric1");
695         this.addPolicy("p", "type1", "", "ric1");
696
697         String url = "/policies?service_id=";
698         String resp = restClient().get(url).block();
699         assertThat(resp).contains("[\"p\"]");
700
701         this.addPolicy("p", "type1", "service", "ric1");
702         url = "/policies?service_id=";
703         resp = restClient().get(url).block();
704         assertThat(resp).contains("[]");
705
706         url = "/policies?service_id=service";
707         resp = restClient().get(url).block();
708         assertThat(resp).contains("[\"p\"]");
709     }
710
711     @Test
712     @DisplayName("test Refuse To Update Policy")
713     void testRefuseToUpdatePolicy() throws Exception {
714         // Test that only the json can be changed for a already created policy
715         // In this case service is attempted to be changed
716         this.addRic("ric1");
717         this.addRic("ricXXX");
718         this.addPolicy("instance1", "type1", "service1", "ric1");
719         this.addPolicy("instance2", "type1", "service1", "ricXXX");
720
721         // Try change ric1 -> ricXXX
722         String bodyWrongRic = putPolicyBody("service1", "ricXXX", "type1", "instance1");
723         testErrorCode(restClient().put("/policies", bodyWrongRic), HttpStatus.CONFLICT);
724     }
725
726     @Test
727     @DisplayName("test Get Policy")
728     void testGetPolicy() throws Exception {
729         String url = "/policies/id";
730         Policy policy = addPolicy("id", "typeName", "service1", "ric1");
731         {
732             String rsp = restClient().get(url).block();
733             PolicyInfo info = gson.fromJson(rsp, PolicyInfo.class);
734             String policyStr = gson.toJson(info.policyData);
735             assertThat(policyStr).isEqualTo(policy.getJson());
736         }
737         {
738             policies.remove(policy);
739             testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
740         }
741     }
742
743     @Test
744     @DisplayName("test Delete Policy")
745     void testDeletePolicy() throws Exception {
746         String policyId = "id.1";
747         addPolicy(policyId, "typeName", "service1", "ric1");
748         assertThat(policies.size()).isEqualTo(1);
749
750         String url = "/policies/" + policyId;
751         ResponseEntity<String> entity = restClient().deleteForEntity(url).block();
752
753         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
754         assertThat(policies.size()).isZero();
755
756         // Delete a non existing policy
757         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
758     }
759
760     @Test
761     @DisplayName("test Get Policy Type")
762     void testGetPolicyType() throws Exception {
763         String typeId = "AC.D";
764         addPolicyType(typeId, "ric1");
765
766         waitForRicState("ric1", RicState.AVAILABLE);
767
768         String url = "/policy-types/" + typeId;
769
770         String rsp = this.restClient().get(url).block();
771
772         PolicyTypeInfo info = gson.fromJson(rsp, PolicyTypeInfo.class);
773         assertThat(info.schema).isNotNull();
774
775         // Get non existing schema
776         url = "/policy-types/JUNK";
777         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
778     }
779
780     String createPolicyTypesJson(String... types) {
781         List<String> list = new ArrayList<>();
782         Collections.addAll(list, types);
783         PolicyTypeIdList ids = new PolicyTypeIdList(list);
784         return gson.toJson(ids);
785     }
786
787     @Test
788     @DisplayName("test Get Policy Types")
789     void testGetPolicyTypes() {
790         String TYPE_ID_1 = "A_type1_1.9.0";
791         String TYPE_ID_2 = "A_type1_2.0.0";
792         String TYPE_ID_3 = "A_type1_1.5.0";
793         String TYPE_ID_4 = "type3_1.9.0";
794         addPolicyType(TYPE_ID_1, "ric1");
795         addPolicyType(TYPE_ID_2, "ric2");
796         addPolicyType(TYPE_ID_3, "ric2");
797         addPolicyType(TYPE_ID_4, "ric2");
798
799         addPolicyType("junk", "ric2");
800         addPolicyType("junk_a.b.c", "ric2");
801
802         String url = "/policy-types";
803         String rsp = restClient().get(url).block();
804         assertThat(rsp).contains(TYPE_ID_1, TYPE_ID_2);
805
806         url = "/policy-types?ric_id=ric1";
807         rsp = restClient().get(url).block();
808         String expResp = createPolicyTypesJson(TYPE_ID_1);
809         assertThat(rsp).isEqualTo(expResp);
810
811         // Get policy types for non existing RIC
812         url = "/policy-types?ric_id=ric1XXX";
813         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
814
815         // All types with a type_name
816         url = "/policy-types?type_name=A_type1";
817         rsp = restClient().get(url).block();
818         assertThat(rsp).contains(TYPE_ID_1, TYPE_ID_2);
819
820         // All types compatible with type1_1.5.0 (which is type1_1.9.0)
821         url = "/policy-types?type_name=A_type1&&compatible_with_version=1.5.0";
822         rsp = restClient().get(url).block();
823         expResp = createPolicyTypesJson(TYPE_ID_3, TYPE_ID_1);
824         assertThat(rsp).isEqualTo(expResp);
825
826         url = "/policy-types?type_name=A_type1&&compatible_with_version=junk";
827         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "Version must contain major.minor.patch code");
828
829         url = "/policy-types?type_name=A_type1&&compatible_with_version=a.b.c";
830         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "Syntax error in");
831
832         url = "/policy-types?compatible_with_version=1.5.0";
833         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "type_name");
834     }
835
836     @Test
837     @DisplayName("test Get Policy Instances")
838     void testGetPolicyInstances() throws Exception {
839         addPolicy("id1", "type1", "service1");
840
841         String url = "/policy-instances";
842         String rsp = restClient().get(url).block();
843         logger.info(rsp);
844         PolicyInfoList info = gson.fromJson(rsp, PolicyInfoList.class);
845         assertThat(info.policies).hasSize(1);
846         PolicyInfo policyInfo = info.policies.iterator().next();
847         assert (policyInfo.validate());
848         assertThat(policyInfo.policyId).isEqualTo("id1");
849         assertThat(policyInfo.policyTypeId).isEqualTo("type1");
850         assertThat(policyInfo.serviceId).isEqualTo("service1");
851     }
852
853     @Test
854     @DisplayName("test Get Policy Instances Filter")
855     void testGetPolicyInstancesFilter() throws Exception {
856         addPolicy("id1", "type1", "service1");
857         addPolicy("id2", "type1", "service2");
858         addPolicy("id3", "type2", "service1");
859         addPolicy("id4", "type1_1.0.0", "service1");
860
861         String url = "/policy-instances?policytype_id=type1";
862         String rsp = restClient().get(url).block();
863         logger.info(rsp);
864         assertThat(rsp).contains("id1") //
865                 .contains("id2") //
866                 .doesNotContain("id3");
867
868         url = "/policy-instances?policytype_id=type1&service_id=service2";
869         rsp = restClient().get(url).block();
870         logger.info(rsp);
871         assertThat(rsp).doesNotContain("id1") //
872                 .contains("id2") //
873                 .doesNotContain("id3");
874
875         url = "/policy-instances?type_name=type1";
876         rsp = restClient().get(url).block();
877         assertThat(rsp).contains("id1") //
878                 .contains("id2") //
879                 .doesNotContain("id3") //
880                 .contains("id4");
881
882         // Test get policies for non existing type
883         url = "/policy-instances?policytype_id=type1XXX";
884         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
885
886         // Test get policies for non existing RIC
887         url = "/policy-instances?ric_id=XXX";
888         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
889     }
890
891     @Test
892     @DisplayName("test Get Policy Ids Filter")
893     void testGetPolicyIdsFilter() throws Exception {
894         addPolicy("id1", "type1", "service1", "ric1");
895         addPolicy("id2", "type1", "service2", "ric1");
896         addPolicy("id3", "type2", "service1", "ric1");
897         addPolicy("id4", "type1_1.0.0", "service1");
898
899         String url = "/policies?policytype_id=type1";
900         String rsp = restClient().get(url).block();
901         logger.info(rsp);
902         assertThat(rsp).contains("id1") //
903                 .contains("id2") //
904                 .doesNotContain("id3");
905
906         url = "/policies?policytype_id=type1&service_id=service1&ric=ric1";
907         rsp = restClient().get(url).block();
908         PolicyIdList respList = gson.fromJson(rsp, PolicyIdList.class);
909         assertThat(respList.policyIds.iterator().next()).isEqualTo("id1");
910
911         url = "/policies?policytype_name=type1&service_id=service1";
912         rsp = restClient().get(url).block();
913         assertThat(rsp).contains("id1") //
914                 .contains("id3") //
915                 .contains("id4");
916         assertThat(gson.fromJson(rsp, PolicyIdList.class).policyIds).hasSize(3);
917
918         // Test get policy ids for non existing type
919         url = "/policies?policytype_id=type1XXX";
920         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
921
922         // Test get policy ids for non existing RIC
923         url = "/policies?ric_id=XXX";
924         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
925     }
926
927     @Test
928     @DisplayName("test Put And Get Service")
929     void testPutAndGetService() throws Exception {
930         // PUT
931         String serviceName = "ac.dc";
932         putService(serviceName, 0, HttpStatus.CREATED);
933         putService(serviceName, 0, HttpStatus.OK);
934
935         // GET one service
936         String url = "/services?service_id=" + serviceName;
937         String rsp = restClient().get(url).block();
938         ServiceStatusList info = gson.fromJson(rsp, ServiceStatusList.class);
939         assertThat(info.statusList).hasSize(1);
940         ServiceStatus status = info.statusList.iterator().next();
941         assertThat(status.keepAliveIntervalSeconds).isZero();
942         assertThat(status.serviceId).isEqualTo(serviceName);
943
944         // GET (all)
945         url = "/services";
946         rsp = restClient().get(url).block();
947         assertThat(rsp).as("Response contains service name").contains(serviceName);
948         logger.info(rsp);
949
950         // Keep alive
951         url = "/services/" + serviceName + "/keepalive";
952         ResponseEntity<?> entity = restClient().putForEntity(url).block();
953         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.OK);
954
955         // DELETE service
956         addPolicy("id1", "type1", serviceName);
957         assertThat(services.size()).isEqualTo(1);
958         url = "/services/" + serviceName;
959         restClient().delete(url).block();
960         assertThat(services.size()).isZero();
961
962         // Keep alive, no registered service
963         testErrorCode(restClient().put("/services/junk/keepalive", ""), HttpStatus.NOT_FOUND);
964
965         // PUT service with bad payload
966         testErrorCode(restClient().put("/services", "crap"), HttpStatus.BAD_REQUEST, false);
967         testErrorCode(restClient().put("/services", "{}"), HttpStatus.BAD_REQUEST, false);
968         testErrorCode(restClient().put("/services", createServiceJson(serviceName, -123)), HttpStatus.BAD_REQUEST,
969                 false);
970         testErrorCode(restClient().put("/services", createServiceJson(serviceName, 0, "missing.portandprotocol.com")),
971                 HttpStatus.BAD_REQUEST, false);
972
973         // GET non existing service
974         testErrorCode(restClient().get("/services?service_id=XXX"), HttpStatus.NOT_FOUND);
975     }
976
977     @Test
978     @DisplayName("test delete Service with no authorization")
979     void testDeleteServiceWithNoAuth() throws Exception {
980         // PUT service
981         String serviceName = "ac.dc";
982         putService(serviceName, 0, HttpStatus.CREATED);
983
984         // No Authorization to Delete
985         this.applicationConfig
986                 .setAuthProviderUrl(baseUrl() + OpenPolicyAgentSimulatorController.ACCESS_CONTROL_URL_REJECT);
987         addPolicy("id1", "type1", serviceName);
988         assertThat(services.size()).isEqualTo(1);
989         String url = "/services/" + serviceName;
990         restClient().delete(url).block();
991         assertThat(services.size()).isZero();
992         assertThat(policies.size()).isEqualTo(1);
993         testErrorCode(restClient().get("/policies/id1"), HttpStatus.UNAUTHORIZED);
994     }
995
996     @Test
997     @DisplayName("test delete Service with no service")
998     void testDeleteServiceWithNoService() {
999         String url = "/services/" + "NoService";
1000         testErrorCode(restClient().delete(url), HttpStatus.NOT_FOUND);
1001     }
1002
1003     @Test
1004     @DisplayName("test Service Supervision")
1005     void testServiceSupervision() {
1006         putService("service1", 1, HttpStatus.CREATED);
1007         addPolicyType("type1", "ric1");
1008
1009         String policyBody = putPolicyBody("service1", "ric1", "type1", "instance1");
1010         restClient().put("/policies", policyBody).block();
1011
1012         assertThat(policies.size()).isEqualTo(1);
1013         assertThat(services.size()).isEqualTo(1);
1014
1015         // Timeout after ~1 second
1016         await().untilAsserted(() -> assertThat(policies.size()).isZero());
1017         assertThat(services.size()).isZero();
1018     }
1019
1020     @Test
1021     @DisplayName("test Get Policy Status")
1022     void testGetPolicyStatus() throws Exception {
1023         addPolicy("id", "typeName", "service1", "ric1");
1024         assertThat(policies.size()).isEqualTo(1);
1025
1026         String url = "/policies/id/status";
1027         String rsp = restClient().get(url).block();
1028         PolicyStatusInfo info = gson.fromJson(rsp, PolicyStatusInfo.class);
1029         assertThat(info.status).isEqualTo("OK");
1030
1031         // GET non existing policy status
1032         url = "/policies/XXX/status";
1033         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
1034
1035         // GET STATUS, the NearRT RIC returns error
1036         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client("ric1");
1037         url = "/policies/id/status";
1038         WebClientResponseException a1Exception = new WebClientResponseException(404, "", null, null, null);
1039         doReturn(Mono.error(a1Exception)).when(a1Client).getPolicyStatus(any());
1040         rsp = restClient().get(url).block();
1041         info = gson.fromJson(rsp, PolicyStatusInfo.class);
1042         assertThat(info.status).hasToString("{}");
1043     }
1044
1045     @Test
1046     @DisplayName("test Get Service Status")
1047     void testGetServiceStatus() {
1048         String url = "/status";
1049         String rsp = restClient().get(url).block();
1050         assertThat(rsp).contains("success");
1051
1052         rsp = restClient(baseUrl(), false).get(url).block(); // V1 status is used by a readinessProbe
1053         assertThat(rsp).isEqualTo("success");
1054     }
1055
1056     @Test
1057     @DisplayName("test Service Notification")
1058     void testServiceNotification() throws Exception {
1059
1060         final String AUTH_TOKEN = "testToken";
1061         Path authFile = Files.createTempFile("pmsTestAuthToken", ".txt");
1062         Files.write(authFile, AUTH_TOKEN.getBytes());
1063         this.securityContext.setAuthTokenFilePath(authFile);
1064
1065         putService("junkService");
1066         Service junkService = this.services.get("junkService");
1067         junkService.setCallbackUrl("https://junk");
1068         putService("service");
1069
1070         Ric ric = addRic("ric1");
1071         ric.setState(Ric.RicState.UNAVAILABLE);
1072         supervision.checkAllRics();
1073         waitForRicState("ric1", RicState.AVAILABLE);
1074
1075         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
1076         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
1077         ServiceCallbackInfo callbackInfo = receivedCallbacks.getReceivedInfo().get(0);
1078         assertThat(callbackInfo.ricId).isEqualTo("ric1");
1079         assertThat(callbackInfo.eventType).isEqualTo(ServiceCallbackInfo.EventType.AVAILABLE);
1080
1081         var headers = receivedCallbacks.receivedHeaders.get(0);
1082         assertThat(headers).containsEntry("authorization", "Bearer " + AUTH_TOKEN);
1083
1084         Files.delete(authFile);
1085     }
1086
1087     private Policy addPolicy(String id, String typeName, String service, String ric) throws ServiceException {
1088         addRic(ric);
1089         Policy policy = Policy.builder() //
1090                 .id(id) //
1091                 .json(jsonString()) //
1092                 .ownerServiceId(service) //
1093                 .ric(rics.getRic(ric)) //
1094                 .type(addPolicyType(typeName, ric)) //
1095                 .lastModified(Instant.now()) //
1096                 .isTransient(false) //
1097                 .statusNotificationUri("/policy-status?id=XXX") //
1098                 .build();
1099         policies.put(policy);
1100         return policy;
1101     }
1102
1103     private Policy addPolicy(String id, String typeName, String service) throws ServiceException {
1104         return addPolicy(id, typeName, service, "ric");
1105     }
1106
1107     private String createServiceJson(String name, long keepAliveIntervalSeconds) {
1108         String callbackUrl = baseUrl() + RappSimulatorController.SERVICE_CALLBACK_URL;
1109         return createServiceJson(name, keepAliveIntervalSeconds, callbackUrl);
1110     }
1111
1112     private String createServiceJson(String name, long keepAliveIntervalSeconds, String url) {
1113         ServiceRegistrationInfo service = new ServiceRegistrationInfo(name, keepAliveIntervalSeconds, url);
1114
1115         String json = gson.toJson(service);
1116         return json;
1117     }
1118
1119     private void putService(String name) {
1120         putService(name, 0, null);
1121     }
1122
1123     private void putService(String name, long keepAliveIntervalSeconds, @Nullable HttpStatus expectedStatus) {
1124         String url = "/services";
1125         String body = createServiceJson(name, keepAliveIntervalSeconds);
1126         ResponseEntity<String> resp = restClient().putForEntity(url, body).block();
1127         if (expectedStatus != null) {
1128             assertEquals(expectedStatus, resp.getStatusCode(), "");
1129         }
1130     }
1131
1132     private String jsonString() {
1133         return "{\"servingCellNrcgi\":\"1\"}";
1134     }
1135
1136     @Test
1137     @DisplayName("test Concurrency")
1138     void testConcurrency() throws Exception {
1139         this.applicationConfig.setAuthProviderUrl("");
1140         logger.info("Concurrency test starting");
1141         final Instant startTime = Instant.now();
1142         List<Thread> threads = new ArrayList<>();
1143         List<ConcurrencyTestRunnable> tests = new ArrayList<>();
1144         a1ClientFactory.setResponseDelay(Duration.ofMillis(2));
1145         addRic("ric");
1146         addPolicyType("type1", "ric");
1147         addPolicyType("type2", "ric");
1148
1149         final String NON_RESPONDING_RIC = "NonRespondingRic";
1150         Ric nonRespondingRic = addRic(NON_RESPONDING_RIC);
1151         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client(NON_RESPONDING_RIC);
1152         a1Client.setErrorInject("errorInject");
1153
1154         for (int i = 0; i < 10; ++i) {
1155             AsyncRestClient restClient = restClient();
1156             ConcurrencyTestRunnable test =
1157                     new ConcurrencyTestRunnable(restClient, supervision, a1ClientFactory, rics, policyTypes);
1158             Thread thread = new Thread(test, "TestThread_" + i);
1159             thread.start();
1160             threads.add(thread);
1161             tests.add(test);
1162         }
1163         for (Thread t : threads) {
1164             t.join();
1165         }
1166         for (ConcurrencyTestRunnable test : tests) {
1167             assertThat(test.isFailed()).isFalse();
1168         }
1169         assertThat(policies.size()).isZero();
1170         logger.info("Concurrency test took: {}", Duration.between(startTime, Instant.now()));
1171
1172         assertThat(nonRespondingRic.getState()).isEqualTo(RicState.UNAVAILABLE);
1173         nonRespondingRic.setState(RicState.AVAILABLE);
1174     }
1175
1176     private AsyncRestClient restClient(String baseUrl, boolean useTrustValidation) {
1177         WebClientConfig config = this.applicationConfig.getWebClientConfig();
1178         config = WebClientConfig.builder() //
1179                 .keyStoreType(config.getKeyStoreType()) //
1180                 .keyStorePassword(config.getKeyStorePassword()) //
1181                 .keyStore(config.getKeyStore()) //
1182                 .keyPassword(config.getKeyPassword()) //
1183                 .isTrustStoreUsed(useTrustValidation) //
1184                 .trustStore(config.getTrustStore()) //
1185                 .trustStorePassword(config.getTrustStorePassword()) //
1186                 .httpProxyConfig(config.getHttpProxyConfig()) //
1187                 .build();
1188
1189         AsyncRestClientFactory f = new AsyncRestClientFactory(config, new SecurityContext(""));
1190         return f.createRestClientNoHttpProxy(baseUrl);
1191
1192     }
1193
1194     private String baseUrl() {
1195         return "https://localhost:" + port;
1196     }
1197
1198     private AsyncRestClient restClient(boolean useTrustValidation) {
1199         return restClient(baseUrl() + Consts.V2_API_ROOT, useTrustValidation);
1200     }
1201
1202     private AsyncRestClient restClient() {
1203         return restClient(false);
1204     }
1205
1206     private void testErrorCode(Mono<?> request, HttpStatus expStatus) {
1207         testErrorCode(request, expStatus, "", true);
1208     }
1209
1210     private void testErrorCode(Mono<?> request, HttpStatus expStatus, boolean expectApplicationProblemJsonMediaType) {
1211         testErrorCode(request, expStatus, "", expectApplicationProblemJsonMediaType);
1212     }
1213
1214     private void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains) {
1215         testErrorCode(request, expStatus, responseContains, true);
1216     }
1217
1218     private void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains,
1219             boolean expectApplicationProblemJsonMediaType) {
1220         StepVerifier.create(request) //
1221                 .expectSubscription() //
1222                 .expectErrorMatches(
1223                         t -> checkWebClientError(t, expStatus, responseContains, expectApplicationProblemJsonMediaType)) //
1224                 .verify();
1225     }
1226
1227     private void waitForRicState(String ricId, RicState state) throws ServiceException {
1228         Ric ric = rics.getRic(ricId);
1229         await().untilAsserted(() -> state.equals(ric.getState()));
1230     }
1231
1232     private boolean checkWebClientError(Throwable throwable, HttpStatus expStatus, String responseContains,
1233             boolean expectApplicationProblemJsonMediaType) {
1234         assertTrue(throwable instanceof WebClientResponseException);
1235         WebClientResponseException responseException = (WebClientResponseException) throwable;
1236         String body = responseException.getResponseBodyAsString();
1237         assertThat(body).contains(responseContains);
1238         assertThat(responseException.getStatusCode()).isEqualTo(expStatus);
1239
1240         if (expectApplicationProblemJsonMediaType) {
1241             assertThat(responseException.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_PROBLEM_JSON);
1242         }
1243         return true;
1244     }
1245
1246     private MockA1Client getA1Client(String ricId) {
1247         return a1ClientFactory.getOrCreateA1Client(ricId);
1248     }
1249
1250     private PolicyType createPolicyType(String policyTypeName) {
1251         return PolicyType.builder() //
1252                 .id(policyTypeName) //
1253                 .schema("{\"title\":\"" + policyTypeName + "\"}") //
1254                 .build();
1255     }
1256
1257     private PolicyType addPolicyType(String policyTypeName, String ricId) {
1258         PolicyType type = createPolicyType(policyTypeName);
1259         policyTypes.put(type);
1260         addRic(ricId).addSupportedPolicyType(type);
1261         return type;
1262     }
1263
1264     private Ric addRic(String ricId) {
1265         return addRic(ricId, null);
1266     }
1267
1268     private RicConfig ricConfig(String ricId, String managedElement) {
1269         List<String> mes = new ArrayList<>();
1270         if (managedElement != null) {
1271             mes.add(managedElement);
1272         }
1273         return RicConfig.builder() //
1274                 .ricId(ricId) //
1275                 .baseUrl(ricId) //
1276                 .managedElementIds(mes) //
1277                 .build();
1278     }
1279
1280     private Ric addRic(String ricId, String managedElement) {
1281         if (rics.get(ricId) != null) {
1282             return rics.get(ricId);
1283         }
1284
1285         RicConfig conf = ricConfig(ricId, managedElement);
1286         Ric ric = new Ric(conf);
1287         ric.setState(Ric.RicState.AVAILABLE);
1288         this.rics.put(ric);
1289         return ric;
1290     }
1291
1292 }