066adc4845cdb0cf711e60ce9ea187f27682092a
[ccsdk/oran.git] /
1 /*-
2  * ========================LICENSE_START=================================
3  * ONAP : ccsdk oran
4  * ======================================================================
5  * Copyright (C) 2019-2022 Nordix Foundation. All rights reserved.
6  * ======================================================================
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  * ========================LICENSE_END===================================
19  */
20
21 package org.onap.ccsdk.oran.a1policymanagementservice.controllers.v2;
22
23 import static org.assertj.core.api.Assertions.assertThat;
24 import static org.awaitility.Awaitility.await;
25 import static org.junit.jupiter.api.Assertions.assertEquals;
26 import static org.junit.jupiter.api.Assertions.assertTrue;
27 import static org.mockito.ArgumentMatchers.any;
28 import static org.mockito.Mockito.doReturn;
29
30 import com.google.gson.Gson;
31 import com.google.gson.GsonBuilder;
32
33 import java.io.FileOutputStream;
34 import java.io.PrintStream;
35 import java.lang.invoke.MethodHandles;
36 import java.nio.charset.StandardCharsets;
37 import java.nio.file.Files;
38 import java.nio.file.Path;
39 import java.nio.file.Paths;
40 import java.time.Duration;
41 import java.time.Instant;
42 import java.util.ArrayList;
43 import java.util.Collections;
44 import java.util.List;
45
46 import org.json.JSONObject;
47 import org.junit.jupiter.api.AfterAll;
48 import org.junit.jupiter.api.AfterEach;
49 import org.junit.jupiter.api.BeforeEach;
50 import org.junit.jupiter.api.DisplayName;
51 import org.junit.jupiter.api.MethodOrderer;
52 import org.junit.jupiter.api.Test;
53 import org.junit.jupiter.api.TestMethodOrder;
54 import org.onap.ccsdk.oran.a1policymanagementservice.clients.A1ClientFactory;
55 import org.onap.ccsdk.oran.a1policymanagementservice.clients.AsyncRestClient;
56 import org.onap.ccsdk.oran.a1policymanagementservice.clients.AsyncRestClientFactory;
57 import org.onap.ccsdk.oran.a1policymanagementservice.clients.SecurityContext;
58 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationConfig;
59 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationConfig.RicConfigUpdate;
60 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.RicConfig;
61 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.WebClientConfig;
62 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.OpenPolicyAgentSimulatorController;
63 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.ServiceCallbackInfo;
64 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.authorization.PolicyAuthorizationRequest;
65 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.authorization.PolicyAuthorizationRequest.Input.AccessType;
66 import org.onap.ccsdk.oran.a1policymanagementservice.exceptions.ServiceException;
67 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Lock;
68 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Lock.LockType;
69 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policies;
70 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policy;
71 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyType;
72 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyTypes;
73 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric;
74 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric.RicState;
75 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Rics;
76 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Service;
77 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Services;
78 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.RefreshConfigTask;
79 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.RicSupervision;
80 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.ServiceSupervision;
81 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1Client;
82 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1ClientFactory;
83 import org.slf4j.Logger;
84 import org.slf4j.LoggerFactory;
85 import org.springframework.beans.factory.annotation.Autowired;
86 import org.springframework.boot.test.context.SpringBootTest;
87 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
88 import org.springframework.boot.test.context.TestConfiguration;
89 import org.springframework.boot.test.web.server.LocalServerPort;
90 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
91 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
92 import org.springframework.context.ApplicationContext;
93 import org.springframework.context.annotation.Bean;
94 import org.springframework.http.HttpStatus;
95 import org.springframework.http.MediaType;
96 import org.springframework.http.ResponseEntity;
97 import org.springframework.test.context.TestPropertySource;
98 import org.springframework.util.FileSystemUtils;
99 import org.springframework.web.reactive.function.client.WebClientRequestException;
100 import org.springframework.web.reactive.function.client.WebClientResponseException;
101
102 import reactor.core.publisher.Mono;
103 import reactor.test.StepVerifier;
104 import reactor.util.annotation.Nullable;
105
106 @TestMethodOrder(MethodOrderer.MethodName.class)
107 @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
108 @TestPropertySource(properties = { //
109         "server.ssl.key-store=./config/keystore.jks", //
110         "app.webclient.trust-store=./config/truststore.jks", //
111         "app.webclient.trust-store-used=true", //
112         "app.vardata-directory=/tmp/pmstest", //
113         "app.filepath=", //
114         "app.s3.bucket=" // If this is set, S3 will be used to store data.
115 })
116 class ApplicationTest {
117     private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
118
119     @Autowired
120     ApplicationContext context;
121
122     @Autowired
123     private Rics rics;
124
125     @Autowired
126     private Policies policies;
127
128     @Autowired
129     private PolicyTypes policyTypes;
130
131     @Autowired
132     MockA1ClientFactory a1ClientFactory;
133
134     @Autowired
135     RicSupervision supervision;
136
137     @Autowired
138     ApplicationConfig applicationConfig;
139
140     @Autowired
141     Services services;
142
143     @Autowired
144     RappSimulatorController rAppSimulator;
145
146     @Autowired
147     RefreshConfigTask refreshConfigTask;
148
149     @Autowired
150     SecurityContext securityContext;
151
152     @Autowired
153     OpenPolicyAgentSimulatorController openPolicyAgentSimulatorController;
154
155     private static Gson gson = new GsonBuilder().create();
156
157     /**
158      * Overrides the BeanFactory.
159      */
160     @TestConfiguration
161     static class TestBeanFactory {
162
163         @Bean
164         A1ClientFactory getA1ClientFactory(@Autowired ApplicationConfig appConfig, @Autowired PolicyTypes types) {
165             return new MockA1ClientFactory(appConfig, types);
166         }
167
168         @Bean
169         public ServiceSupervision getServiceSupervision(@Autowired Services services,
170                 @Autowired A1ClientFactory a1ClientFactory, @Autowired Policies policies) {
171             Duration checkInterval = Duration.ofMillis(1);
172             return new ServiceSupervision(services, policies, a1ClientFactory, checkInterval);
173         }
174
175         @Bean
176         public ServletWebServerFactory servletContainer() {
177             return new TomcatServletWebServerFactory();
178         }
179     }
180
181     @LocalServerPort
182     private int port;
183
184     @BeforeEach
185     void init() {
186         this.applicationConfig.setAuthProviderUrl(baseUrl() + OpenPolicyAgentSimulatorController.ACCESS_CONTROL_URL);
187     }
188
189     @AfterEach
190     void reset() {
191         rics.clear();
192         policies.clear();
193         policyTypes.clear();
194         services.clear();
195         a1ClientFactory.reset();
196         this.rAppSimulator.getTestResults().clear();
197         this.a1ClientFactory.setPolicyTypes(policyTypes); // Default same types in RIC and in this app
198         this.securityContext.setAuthTokenFilePath(null);
199         this.openPolicyAgentSimulatorController.getTestResults().reset();
200     }
201
202     @AfterAll
203     static void clearTestDir() {
204         try {
205             FileSystemUtils.deleteRecursively(Path.of("/tmp/pmstest"));
206         } catch (Exception e) {
207             logger.warn("Could test directory : {}", e.getMessage());
208         }
209     }
210
211     @AfterEach
212     void verifyNoRicLocks() {
213         for (Ric ric : this.rics.getRics()) {
214             Lock.Grant grant = ric.getLock().lockBlocking(LockType.EXCLUSIVE, "verifyNoRicLocks");
215             grant.unlockBlocking();
216             assertThat(ric.getLock().getLockCounter()).isZero();
217             assertThat(ric.getState()).isEqualTo(Ric.RicState.AVAILABLE);
218         }
219     }
220
221     @Test
222     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
223     @DisplayName("test ZZ Actuator")
224     void testZZActuator() throws Exception {
225         // The test must be run last, hence the "ZZ" in the name. All succeeding tests
226         // will fail.
227         AsyncRestClient client = restClient(baseUrl(), false);
228
229         client.post("/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice",
230                 "{\"configuredLevel\":\"trace\"}").block();
231
232         String resp = client.get("/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice").block();
233         assertThat(resp).contains("TRACE");
234
235         client.post("/actuator/loggers/org.springframework.boot.actuate", "{\"configuredLevel\":\"trace\"}").block();
236
237         // This will stop the web server and all coming tests will fail.
238         client.post("/actuator/shutdown", "").block();
239
240         Thread.sleep(1000);
241
242         StepVerifier.create(restClient().get("/rics")) // Any call
243                 .expectSubscription() //
244                 .expectErrorMatches(t -> t instanceof WebClientRequestException) //
245                 .verify();
246
247     }
248
249     @Test
250     @DisplayName("test generate Api Doc")
251     void generateApiDoc() throws Exception {
252         String url = "https://localhost:" + this.port + "/v3/api-docs";
253         ResponseEntity<String> resp = restClient("", false).getForEntity(url).block();
254         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
255         JSONObject jsonObj = new JSONObject(resp.getBody());
256         assertThat(jsonObj.remove("servers")).isNotNull();
257
258         String indented = (jsonObj).toString(4);
259         String docDir = "api/";
260         Files.createDirectories(Paths.get(docDir));
261         try (PrintStream out = new PrintStream(new FileOutputStream(docDir + "pms-api.json"))) {
262             out.print(indented);
263         }
264     }
265
266     @Test
267     @DisplayName("test Persistency Policies")
268     void testPersistencyPolicies() throws Exception {
269         Ric ric = this.addRic("ric1");
270         PolicyType type = this.addPolicyType("type1", ric.id());
271
272         final int noOfPolicies = 100;
273         for (int i = 0; i < noOfPolicies; ++i) {
274             addPolicy("id" + i, type.getId(), "service", ric.id());
275         }
276         waitforS3();
277
278         {
279             Policies policies = new Policies(this.applicationConfig);
280             policies.restoreFromDatabase(ric, this.policyTypes).blockLast();
281             assertThat(policies.size()).isEqualTo(noOfPolicies);
282         }
283
284         {
285             restClient().delete("/policies/id2").block();
286             Policies policies = new Policies(this.applicationConfig);
287             policies.restoreFromDatabase(ric, this.policyTypes).blockLast();
288             assertThat(policies.size()).isEqualTo(noOfPolicies - 1);
289         }
290     }
291
292     @Test
293     @DisplayName("test Persistency Policy Types")
294     void testPersistencyPolicyTypes() throws Exception {
295         Ric ric = this.addRic("ric1");
296         this.addPolicyType("type1", ric.id());
297         waitforS3();
298
299         PolicyTypes types = new PolicyTypes(this.applicationConfig);
300         types.restoreFromDatabase().blockLast();
301         assertThat(types.size()).isEqualTo(1);
302     }
303
304     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
305     private void waitforS3() throws Exception {
306         if (applicationConfig.isS3Enabled()) {
307             Thread.sleep(1000);
308         }
309     }
310
311     @Test
312     @DisplayName("test Persistency Service")
313     void testPersistencyService() throws Exception {
314         final String SERVICE = "serviceName";
315         putService(SERVICE, 1234, HttpStatus.CREATED);
316         assertThat(this.services.size()).isEqualTo(1);
317         Service service = this.services.getService(SERVICE);
318         waitforS3();
319
320         Services servicesRestored = new Services(this.applicationConfig);
321         servicesRestored.restoreFromDatabase().blockLast();
322         Service serviceRestored = servicesRestored.getService(SERVICE);
323         assertThat(servicesRestored.size()).isEqualTo(1);
324         assertThat(serviceRestored.getCallbackUrl()).isEqualTo(service.getCallbackUrl());
325         assertThat(serviceRestored.getKeepAliveInterval()).isEqualTo(service.getKeepAliveInterval());
326
327         // check that the service can be deleted
328         this.services.remove(SERVICE);
329         servicesRestored = new Services(this.applicationConfig);
330         assertThat(servicesRestored.size()).isZero();
331     }
332
333     @Test
334     @DisplayName("test Adding Ric From Configuration")
335     void testAddingRicFromConfiguration() throws Exception {
336         // Test adding the RIC from configuration
337
338         final String RIC = "ric1";
339         final String TYPE = "type123";
340         PolicyTypes nearRtRicPolicyTypes = new PolicyTypes(this.applicationConfig);
341         nearRtRicPolicyTypes.put(createPolicyType(TYPE));
342         this.a1ClientFactory.setPolicyTypes(nearRtRicPolicyTypes);
343
344         putService("service");
345
346         refreshConfigTask.handleUpdatedRicConfig( //
347                 new RicConfigUpdate(ricConfig(RIC, "me1"), RicConfigUpdate.Type.ADDED)) //
348                 .block();
349         waitForRicState(RIC, RicState.AVAILABLE);
350
351         // Test that the type has been synched
352         Ric addedRic = this.rics.getRic(RIC);
353         assertThat(addedRic.getSupportedPolicyTypes()).hasSize(1);
354         assertThat(addedRic.getSupportedPolicyTypes().iterator().next().getId()).isEqualTo(TYPE);
355
356         // Check that a service callback for the AVAILABLE RIC is invoked
357         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
358         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
359         ServiceCallbackInfo callbackInfo = receivedCallbacks.getReceivedInfo().get(0);
360         assertThat(callbackInfo.ricId).isEqualTo(RIC);
361         assertThat(callbackInfo.eventType).isEqualTo(ServiceCallbackInfo.EventType.AVAILABLE);
362     }
363
364     @Test
365     @DisplayName("test Adding Ric From Configuration non Responding Ric")
366     void testAddingRicFromConfiguration_nonRespondingRic() throws Exception {
367         putService("service");
368
369         final String RIC = "NonRespondingRic";
370         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client(RIC);
371         doReturn(MockA1Client.monoError("error", HttpStatus.BAD_GATEWAY)).when(a1Client).getPolicyTypeIdentities();
372
373         refreshConfigTask.handleUpdatedRicConfig( //
374                 new RicConfigUpdate(ricConfig(RIC, "me1"), RicConfigUpdate.Type.ADDED)) //
375                 .block();
376
377         waitForRicState(RIC, RicState.UNAVAILABLE);
378
379         // Check that no service callback for the UNAVAILABLE RIC is invoked
380         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
381         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).isEmpty());
382
383         // Run a synch and check that the AVAILABLE notification is received
384         a1ClientFactory.reset();
385         supervision.checkAllRics();
386         waitForRicState(RIC, RicState.AVAILABLE);
387
388         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
389     }
390
391     @Test
392     @DisplayName("test Trust Validation")
393     void testTrustValidation() {
394         addRic("ric1");
395
396         String rsp = restClient(true).get("/rics").block(); // restClient(true) enables trust validation
397         assertThat(rsp).contains("ric1");
398
399     }
400
401     @Test
402     @DisplayName("test Get Rics")
403     void testGetRics() throws Exception {
404         addRic("ric1");
405         this.addPolicyType("type1", "ric1");
406         String url = "/rics?policytype_id=type1";
407         String rsp = restClient().get(url).block();
408         assertThat(rsp).contains("ric1");
409
410         // nameless type for ORAN A1 1.1
411         addRic("ric2");
412         this.addPolicyType("", "ric2");
413         url = "/rics?policytype_id=";
414         rsp = restClient().get(url).block();
415         assertThat(rsp).contains("ric2") //
416                 .doesNotContain("ric1") //
417                 .contains("AVAILABLE");
418
419         // All RICs
420         rsp = restClient().get("/rics").block();
421         assertThat(rsp).contains("ric2") //
422                 .contains("ric1");
423
424         // Non existing policy type
425         url = "/rics?policytype_id=XXXX";
426         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
427     }
428
429     @Test
430     @DisplayName("test Synchronization")
431     void testSynchronization() throws Exception {
432         // Two polictypes will be put in the NearRT RICs
433         PolicyTypes nearRtRicPolicyTypes = new PolicyTypes(this.applicationConfig);
434         nearRtRicPolicyTypes.put(createPolicyType("typeName"));
435         nearRtRicPolicyTypes.put(createPolicyType("typeName2"));
436         this.a1ClientFactory.setPolicyTypes(nearRtRicPolicyTypes);
437
438         // One type and one instance added to the Policy Management Service's storage
439         final String ric1Name = "ric1";
440         Ric ric1 = addRic(ric1Name);
441         Policy policy2 = addPolicy("policyId2", "typeName", "service", ric1Name);
442         Ric ric2 = addRic("ric2");
443
444         getA1Client(ric1Name).putPolicy(policy2); // put it in the RIC (Near-RT RIC)
445         policies.remove(policy2); // Remove it from the repo -> should be deleted in the RIC
446
447         String policyId = "policyId";
448         Policy policy = addPolicy(policyId, "typeName", "service", ric1Name); // This should be created in the RIC
449         supervision.checkAllRics(); // The created policy should be put in the RIC
450
451         // Wait until synch is completed
452         waitForRicState(ric1Name, RicState.SYNCHRONIZING);
453         waitForRicState(ric1Name, RicState.AVAILABLE);
454         waitForRicState("ric2", RicState.AVAILABLE);
455
456         Policies ricPolicies = getA1Client(ric1Name).getPolicies();
457         assertThat(ricPolicies.size()).isEqualTo(1);
458         Policy ricPolicy = ricPolicies.get(policyId);
459         assertThat(ricPolicy.getJson()).isEqualTo(policy.getJson());
460
461         // Both types should be in the Policy Management Service's storage after the
462         // synch
463         assertThat(ric1.getSupportedPolicyTypes()).hasSize(2);
464         assertThat(ric2.getSupportedPolicyTypes()).hasSize(2);
465     }
466
467     @Test
468     @DisplayName("test Get Ric")
469     void testGetRic() throws Exception {
470         String ricId = "ric1";
471         String managedElementId = "kista_1";
472         addRic(ricId, managedElementId);
473
474         String url = "/rics/ric?managed_element_id=" + managedElementId;
475         String rsp = restClient().get(url).block();
476         RicInfo ricInfo = gson.fromJson(rsp, RicInfo.class);
477         assertThat(ricInfo.ricId).isEqualTo(ricId);
478
479         url = "/rics/ric?ric_id=" + ricId;
480         rsp = restClient().get(url).block();
481         ricInfo = gson.fromJson(rsp, RicInfo.class);
482         assertThat(ricInfo.ricId).isEqualTo(ricId);
483
484         // test GET RIC for ManagedElement that does not exist
485         url = "/rics/ric?managed_element_id=" + "junk";
486         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
487
488         url = "/rics/ric";
489         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST);
490     }
491
492     private String putPolicyBody(String serviceName, String ricId, String policyTypeName, String policyInstanceId,
493             boolean isTransient, String statusNotificationUri) {
494         PolicyInfo info = new PolicyInfo();
495         info.policyId = policyInstanceId;
496         info.policyTypeId = policyTypeName;
497         info.ricId = ricId;
498         info.serviceId = serviceName;
499         info.policyData = gson.fromJson(jsonString(), Object.class);
500
501         if (isTransient) {
502             info.isTransient = isTransient;
503         }
504         info.statusNotificationUri = statusNotificationUri;
505         return gson.toJson(info);
506     }
507
508     private String putPolicyBody(String serviceName, String ricId, String policyTypeName, String policyInstanceId) {
509         return putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId, false, "statusUri");
510     }
511
512     @Test
513     @DisplayName("test Put Policy")
514     void testPutPolicy() throws Exception {
515         String serviceName = "service.1";
516         String ricId = "ric.1";
517         String policyTypeName = "type1_1.2.3";
518         String policyInstanceId = "instance_1.2.3";
519
520         putService(serviceName);
521         addPolicyType(policyTypeName, ricId);
522
523         // PUT a transient policy
524         String url = "/policies";
525         String policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId, true, "statusNotif");
526         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
527
528         restClient().put(url, policyBody).block();
529         {
530             // Check the authorization request
531             OpenPolicyAgentSimulatorController.TestResults res =
532                     this.openPolicyAgentSimulatorController.getTestResults();
533             assertThat(res.receivedRequests).hasSize(1);
534             PolicyAuthorizationRequest req = res.receivedRequests.get(0);
535             assertThat(req.getInput().getAccessType()).isEqualTo(AccessType.WRITE);
536             assertThat(req.getInput().getPolicyTypeId()).isEqualTo(policyTypeName);
537         }
538
539         Policy policy = policies.getPolicy(policyInstanceId);
540         assertThat(policy).isNotNull();
541         assertThat(policy.getId()).isEqualTo(policyInstanceId);
542         assertThat(policy.getOwnerServiceId()).isEqualTo(serviceName);
543         assertThat(policy.getRic().id()).isEqualTo(ricId);
544         assertThat(policy.isTransient()).isTrue();
545
546         // Put a non transient policy
547         policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId);
548         restClient().put(url, policyBody).block();
549         policy = policies.getPolicy(policyInstanceId);
550         assertThat(policy.isTransient()).isFalse();
551
552         url = "/policy-instances";
553         String rsp = restClient().get(url).block();
554         assertThat(rsp).as("Response contains policy instance ID.").contains(policyInstanceId);
555
556         url = "/policies/" + policyInstanceId;
557         rsp = restClient().get(url).block();
558         assertThat(rsp).contains(policyBody);
559
560         // Test of error codes
561         url = "/policies";
562         policyBody = putPolicyBody(serviceName, ricId + "XX", policyTypeName, policyInstanceId);
563         testErrorCode(restClient().put(url, policyBody), HttpStatus.NOT_FOUND);
564
565         policyBody = putPolicyBody(serviceName, ricId, policyTypeName + "XX", policyInstanceId);
566         addPolicyType(policyTypeName + "XX", "otherRic");
567         testErrorCode(restClient().put(url, policyBody), HttpStatus.NOT_FOUND);
568
569         policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId);
570         this.rics.getRic(ricId).setState(Ric.RicState.SYNCHRONIZING);
571         testErrorCode(restClient().put(url, policyBody), HttpStatus.LOCKED);
572         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
573     }
574
575     @Test
576     void testFineGrainedAuth() throws Exception {
577         final String POLICY_ID = "policyId";
578         final String RIC_ID = "ric1";
579         final String TYPE_ID = "typeName";
580         addPolicy(POLICY_ID, TYPE_ID, null, RIC_ID);
581         assertThat(policies.size()).isEqualTo(1);
582
583         this.applicationConfig
584                 .setAuthProviderUrl(baseUrl() + OpenPolicyAgentSimulatorController.ACCESS_CONTROL_URL_REJECT);
585
586         String url = "/policy-instances";
587         String rsp = restClient().get(url).block();
588         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
589
590         url = "/policies/" + POLICY_ID;
591         testErrorCode(restClient().delete(url), HttpStatus.UNAUTHORIZED, "Not authorized");
592
593         url = "/policies";
594         String policyBody = putPolicyBody(null, RIC_ID, TYPE_ID, POLICY_ID, false, null);
595         testErrorCode(restClient().put(url, policyBody), HttpStatus.UNAUTHORIZED, "Not authorized");
596
597         rsp = restClient().get(url).block();
598         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
599     }
600
601     @Test
602     void testFineGrainedAuth_OPA_UNAVALIABLE() throws Exception {
603         final String POLICY_ID = "policyId";
604         final String RIC_ID = "ric1";
605         final String TYPE_ID = "typeName";
606         addPolicy(POLICY_ID, TYPE_ID, null, RIC_ID);
607         assertThat(policies.size()).isEqualTo(1);
608
609         this.applicationConfig.setAuthProviderUrl("junk");
610
611         String url = "/policy-instances";
612         String rsp = restClient().get(url).block();
613         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
614
615         url = "/policies/" + POLICY_ID;
616         testErrorCode(restClient().delete(url), HttpStatus.UNAUTHORIZED, "Not authorized");
617
618         url = "/policies";
619         String policyBody = putPolicyBody(null, RIC_ID, TYPE_ID, POLICY_ID, false, null);
620         testErrorCode(restClient().put(url, policyBody), HttpStatus.UNAUTHORIZED, "Not authorized");
621
622         rsp = restClient().get(url).block();
623         assertThat(rsp).as("Response contains no policy instance ID.").contains("[]");
624     }
625
626     @Test
627     @DisplayName("test Put Policy No Service No Status Uri")
628     void testPutPolicy_NoServiceNoStatusUri() throws Exception {
629         String ricId = "ric.1";
630         String policyTypeName = "type1_1.2.3";
631         String policyInstanceId = "instance_1.2.3";
632
633         addPolicyType(policyTypeName, ricId);
634
635         // PUT a transient policy
636         String url = "/policies";
637         String policyBody = putPolicyBody(null, ricId, policyTypeName, policyInstanceId, true, null);
638         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
639
640         restClient().put(url, policyBody).block();
641
642         Policy policy = policies.getPolicy(policyInstanceId);
643         assertThat(policy).isNotNull();
644         assertThat(policy.getOwnerServiceId()).isBlank();
645         assertThat(policy.getStatusNotificationUri()).isBlank();
646     }
647
648     @Test
649     /**
650      * Test that HttpStatus and body from failing REST call to A1 is passed on to
651      * the caller.
652      *
653      * @throws ServiceException
654      */
655     @DisplayName("test Error From Ric")
656     void testErrorFromRic() throws ServiceException {
657         putService("service1");
658         addPolicyType("type1", "ric1");
659
660         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client("ric1");
661         HttpStatus httpStatus = HttpStatus.INTERNAL_SERVER_ERROR;
662         String responseBody = "Refused";
663         byte[] responseBodyBytes = responseBody.getBytes(StandardCharsets.UTF_8);
664
665         WebClientResponseException a1Exception = new WebClientResponseException(httpStatus.value(), "statusText", null,
666                 responseBodyBytes, StandardCharsets.UTF_8, null);
667         doReturn(Mono.error(a1Exception)).when(a1Client).putPolicy(any());
668
669         // PUT Policy
670         String putBody = putPolicyBody("service1", "ric1", "type1", "id1");
671         String url = "/policies";
672         testErrorCode(restClient().put(url, putBody), httpStatus, responseBody);
673
674         // DELETE POLICY
675         this.addPolicy("instance1", "type1", "service1", "ric1");
676         doReturn(Mono.error(a1Exception)).when(a1Client).deletePolicy(any());
677         testErrorCode(restClient().delete("/policies/instance1"), httpStatus, responseBody);
678
679     }
680
681     @Test
682     @DisplayName("test Put Typeless Policy")
683     void testPutTypelessPolicy() throws Exception {
684         putService("service1");
685         addPolicyType("", "ric1");
686         String body = putPolicyBody("service1", "ric1", "", "id1");
687         restClient().put("/policies", body).block();
688
689         String rsp = restClient().get("/policy-instances").block();
690         PolicyInfoList info = gson.fromJson(rsp, PolicyInfoList.class);
691         assertThat(info.policies).hasSize(1);
692         PolicyInfo policyInfo = info.policies.iterator().next();
693         assertThat(policyInfo.policyId).isEqualTo("id1");
694         assertThat(policyInfo.policyTypeId).isEmpty();
695     }
696
697     @Test
698     @DisplayName("test Update Service")
699     void testUpdateService() throws Exception {
700         this.addRic("ric1");
701         this.addPolicy("p", "type1", "", "ric1");
702
703         String url = "/policies?service_id=";
704         String resp = restClient().get(url).block();
705         assertThat(resp).contains("[\"p\"]");
706
707         this.addPolicy("p", "type1", "service", "ric1");
708         url = "/policies?service_id=";
709         resp = restClient().get(url).block();
710         assertThat(resp).contains("[]");
711
712         url = "/policies?service_id=service";
713         resp = restClient().get(url).block();
714         assertThat(resp).contains("[\"p\"]");
715     }
716
717     @Test
718     @DisplayName("test Refuse To Update Policy")
719     void testRefuseToUpdatePolicy() throws Exception {
720         // Test that only the json can be changed for a already created policy
721         // In this case service is attempted to be changed
722         this.addRic("ric1");
723         this.addRic("ricXXX");
724         this.addPolicy("instance1", "type1", "service1", "ric1");
725         this.addPolicy("instance2", "type1", "service1", "ricXXX");
726
727         // Try change ric1 -> ricXXX
728         String bodyWrongRic = putPolicyBody("service1", "ricXXX", "type1", "instance1");
729         testErrorCode(restClient().put("/policies", bodyWrongRic), HttpStatus.CONFLICT);
730     }
731
732     @Test
733     @DisplayName("test Get Policy")
734     void testGetPolicy() throws Exception {
735         String url = "/policies/id";
736         Policy policy = addPolicy("id", "typeName", "service1", "ric1");
737         {
738             String rsp = restClient().get(url).block();
739             PolicyInfo info = gson.fromJson(rsp, PolicyInfo.class);
740             String policyStr = gson.toJson(info.policyData);
741             assertThat(policyStr).isEqualTo(policy.getJson());
742         }
743         {
744             policies.remove(policy);
745             testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
746         }
747     }
748
749     @Test
750     @DisplayName("test Delete Policy")
751     void testDeletePolicy() throws Exception {
752         String policyId = "id.1";
753         addPolicy(policyId, "typeName", "service1", "ric1");
754         assertThat(policies.size()).isEqualTo(1);
755
756         String url = "/policies/" + policyId;
757         ResponseEntity<String> entity = restClient().deleteForEntity(url).block();
758
759         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
760         assertThat(policies.size()).isZero();
761
762         // Delete a non existing policy
763         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
764     }
765
766     @Test
767     @DisplayName("test Get Policy Type")
768     void testGetPolicyType() throws Exception {
769         String typeId = "AC.D";
770         addPolicyType(typeId, "ric1");
771
772         waitForRicState("ric1", RicState.AVAILABLE);
773
774         String url = "/policy-types/" + typeId;
775
776         String rsp = this.restClient().get(url).block();
777
778         PolicyTypeInfo info = gson.fromJson(rsp, PolicyTypeInfo.class);
779         assertThat(info.schema).isNotNull();
780
781         // Get non existing schema
782         url = "/policy-types/JUNK";
783         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
784     }
785
786     String createPolicyTypesJson(String... types) {
787         List<String> list = new ArrayList<>();
788         Collections.addAll(list, types);
789         PolicyTypeIdList ids = new PolicyTypeIdList(list);
790         return gson.toJson(ids);
791     }
792
793     @Test
794     @DisplayName("test Get Policy Types")
795     void testGetPolicyTypes() throws Exception {
796         String TYPE_ID_1 = "A_type1_1.9.0";
797         String TYPE_ID_2 = "A_type1_2.0.0";
798         String TYPE_ID_3 = "A_type1_1.5.0";
799         String TYPE_ID_4 = "type3_1.9.0";
800         addPolicyType(TYPE_ID_1, "ric1");
801         addPolicyType(TYPE_ID_2, "ric2");
802         addPolicyType(TYPE_ID_3, "ric2");
803         addPolicyType(TYPE_ID_4, "ric2");
804
805         addPolicyType("junk", "ric2");
806         addPolicyType("junk_a.b.c", "ric2");
807
808         String url = "/policy-types";
809         String rsp = restClient().get(url).block();
810         assertThat(rsp).contains(TYPE_ID_1, TYPE_ID_2);
811
812         url = "/policy-types?ric_id=ric1";
813         rsp = restClient().get(url).block();
814         String expResp = createPolicyTypesJson(TYPE_ID_1);
815         assertThat(rsp).isEqualTo(expResp);
816
817         // Get policy types for non existing RIC
818         url = "/policy-types?ric_id=ric1XXX";
819         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
820
821         // All types with a type_name
822         url = "/policy-types?type_name=A_type1";
823         rsp = restClient().get(url).block();
824         assertThat(rsp).contains(TYPE_ID_1, TYPE_ID_2);
825
826         // All types compatible with type1_1.5.0 (which is type1_1.9.0)
827         url = "/policy-types?type_name=A_type1&&compatible_with_version=1.5.0";
828         rsp = restClient().get(url).block();
829         expResp = createPolicyTypesJson(TYPE_ID_3, TYPE_ID_1);
830         assertThat(rsp).isEqualTo(expResp);
831
832         url = "/policy-types?type_name=A_type1&&compatible_with_version=junk";
833         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "Version must contain major.minor.patch code");
834
835         url = "/policy-types?type_name=A_type1&&compatible_with_version=a.b.c";
836         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "Syntax error in");
837
838         url = "/policy-types?compatible_with_version=1.5.0";
839         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "type_name");
840     }
841
842     @Test
843     @DisplayName("test Get Policy Instances")
844     void testGetPolicyInstances() throws Exception {
845         addPolicy("id1", "type1", "service1");
846
847         String url = "/policy-instances";
848         String rsp = restClient().get(url).block();
849         logger.info(rsp);
850         PolicyInfoList info = gson.fromJson(rsp, PolicyInfoList.class);
851         assertThat(info.policies).hasSize(1);
852         PolicyInfo policyInfo = info.policies.iterator().next();
853         assert (policyInfo.validate());
854         assertThat(policyInfo.policyId).isEqualTo("id1");
855         assertThat(policyInfo.policyTypeId).isEqualTo("type1");
856         assertThat(policyInfo.serviceId).isEqualTo("service1");
857     }
858
859     @Test
860     @DisplayName("test Get Policy Instances Filter")
861     void testGetPolicyInstancesFilter() throws Exception {
862         addPolicy("id1", "type1", "service1");
863         addPolicy("id2", "type1", "service2");
864         addPolicy("id3", "type2", "service1");
865         addPolicy("id4", "type1_1.0.0", "service1");
866
867         String url = "/policy-instances?policytype_id=type1";
868         String rsp = restClient().get(url).block();
869         logger.info(rsp);
870         assertThat(rsp).contains("id1") //
871                 .contains("id2") //
872                 .doesNotContain("id3");
873
874         url = "/policy-instances?policytype_id=type1&service_id=service2";
875         rsp = restClient().get(url).block();
876         logger.info(rsp);
877         assertThat(rsp).doesNotContain("id1") //
878                 .contains("id2") //
879                 .doesNotContain("id3");
880
881         url = "/policy-instances?type_name=type1";
882         rsp = restClient().get(url).block();
883         assertThat(rsp).contains("id1") //
884                 .contains("id2") //
885                 .doesNotContain("id3") //
886                 .contains("id4");
887
888         // Test get policies for non existing type
889         url = "/policy-instances?policytype_id=type1XXX";
890         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
891
892         // Test get policies for non existing RIC
893         url = "/policy-instances?ric_id=XXX";
894         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
895     }
896
897     @Test
898     @DisplayName("test Get Policy Ids Filter")
899     void testGetPolicyIdsFilter() throws Exception {
900         addPolicy("id1", "type1", "service1", "ric1");
901         addPolicy("id2", "type1", "service2", "ric1");
902         addPolicy("id3", "type2", "service1", "ric1");
903         addPolicy("id4", "type1_1.0.0", "service1");
904
905         String url = "/policies?policytype_id=type1";
906         String rsp = restClient().get(url).block();
907         logger.info(rsp);
908         assertThat(rsp).contains("id1") //
909                 .contains("id2") //
910                 .doesNotContain("id3");
911
912         url = "/policies?policytype_id=type1&service_id=service1&ric=ric1";
913         rsp = restClient().get(url).block();
914         PolicyIdList respList = gson.fromJson(rsp, PolicyIdList.class);
915         assertThat(respList.policyIds.iterator().next()).isEqualTo("id1");
916
917         url = "/policies?policytype_name=type1&service_id=service1";
918         rsp = restClient().get(url).block();
919         assertThat(rsp).contains("id1") //
920                 .contains("id3") //
921                 .contains("id4");
922         assertThat(gson.fromJson(rsp, PolicyIdList.class).policyIds).hasSize(3);
923
924         // Test get policy ids for non existing type
925         url = "/policies?policytype_id=type1XXX";
926         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
927
928         // Test get policy ids for non existing RIC
929         url = "/policies?ric_id=XXX";
930         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
931     }
932
933     @Test
934     @DisplayName("test Put And Get Service")
935     void testPutAndGetService() throws Exception {
936         // PUT
937         String serviceName = "ac.dc";
938         putService(serviceName, 0, HttpStatus.CREATED);
939         putService(serviceName, 0, HttpStatus.OK);
940
941         // GET one service
942         String url = "/services?service_id=" + serviceName;
943         String rsp = restClient().get(url).block();
944         ServiceStatusList info = gson.fromJson(rsp, ServiceStatusList.class);
945         assertThat(info.statusList).hasSize(1);
946         ServiceStatus status = info.statusList.iterator().next();
947         assertThat(status.keepAliveIntervalSeconds).isZero();
948         assertThat(status.serviceId).isEqualTo(serviceName);
949
950         // GET (all)
951         url = "/services";
952         rsp = restClient().get(url).block();
953         assertThat(rsp).as("Response contains service name").contains(serviceName);
954         logger.info(rsp);
955
956         // Keep alive
957         url = "/services/" + serviceName + "/keepalive";
958         ResponseEntity<?> entity = restClient().putForEntity(url).block();
959         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.OK);
960
961         // DELETE service
962         assertThat(services.size()).isEqualTo(1);
963         url = "/services/" + serviceName;
964         restClient().delete(url).block();
965         assertThat(services.size()).isZero();
966
967         // Keep alive, no registered service
968         testErrorCode(restClient().put("/services/junk/keepalive", ""), HttpStatus.NOT_FOUND);
969
970         // PUT service with bad payload
971         testErrorCode(restClient().put("/services", "crap"), HttpStatus.BAD_REQUEST, false);
972         testErrorCode(restClient().put("/services", "{}"), HttpStatus.BAD_REQUEST, false);
973         testErrorCode(restClient().put("/services", createServiceJson(serviceName, -123)), HttpStatus.BAD_REQUEST,
974                 false);
975         testErrorCode(restClient().put("/services", createServiceJson(serviceName, 0, "missing.portandprotocol.com")),
976                 HttpStatus.BAD_REQUEST, false);
977
978         // GET non existing service
979         testErrorCode(restClient().get("/services?service_id=XXX"), HttpStatus.NOT_FOUND);
980     }
981
982     @Test
983     @DisplayName("test Service Supervision")
984     void testServiceSupervision() throws Exception {
985         putService("service1", 1, HttpStatus.CREATED);
986         addPolicyType("type1", "ric1");
987
988         String policyBody = putPolicyBody("service1", "ric1", "type1", "instance1");
989         restClient().put("/policies", policyBody).block();
990
991         assertThat(policies.size()).isEqualTo(1);
992         assertThat(services.size()).isEqualTo(1);
993
994         // Timeout after ~1 second
995         await().untilAsserted(() -> assertThat(policies.size()).isZero());
996         assertThat(services.size()).isZero();
997     }
998
999     @Test
1000     @DisplayName("test Get Policy Status")
1001     void testGetPolicyStatus() throws Exception {
1002         addPolicy("id", "typeName", "service1", "ric1");
1003         assertThat(policies.size()).isEqualTo(1);
1004
1005         String url = "/policies/id/status";
1006         String rsp = restClient().get(url).block();
1007         PolicyStatusInfo info = gson.fromJson(rsp, PolicyStatusInfo.class);
1008         assertThat(info.status).isEqualTo("OK");
1009
1010         // GET non existing policy status
1011         url = "/policies/XXX/status";
1012         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
1013
1014         // GET STATUS, the NearRT RIC returns error
1015         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client("ric1");
1016         url = "/policies/id/status";
1017         WebClientResponseException a1Exception = new WebClientResponseException(404, "", null, null, null);
1018         doReturn(Mono.error(a1Exception)).when(a1Client).getPolicyStatus(any());
1019         rsp = restClient().get(url).block();
1020         info = gson.fromJson(rsp, PolicyStatusInfo.class);
1021         assertThat(info.status).hasToString("{}");
1022     }
1023
1024     @Test
1025     @DisplayName("test Get Service Status")
1026     void testGetServiceStatus() throws Exception {
1027         String url = "/status";
1028         String rsp = restClient().get(url).block();
1029         assertThat(rsp).contains("hunky dory");
1030
1031         rsp = restClient(baseUrl(), false).get(url).block(); // V1 status is used by a readinessProbe
1032         assertThat(rsp).isEqualTo("hunky dory");
1033     }
1034
1035     @Test
1036     @DisplayName("test Service Notification")
1037     void testServiceNotification() throws Exception {
1038
1039         final String AUTH_TOKEN = "testToken";
1040         Path authFile = Files.createTempFile("pmsTestAuthToken", ".txt");
1041         Files.write(authFile, AUTH_TOKEN.getBytes());
1042         this.securityContext.setAuthTokenFilePath(authFile);
1043
1044         putService("junkService");
1045         Service junkService = this.services.get("junkService");
1046         junkService.setCallbackUrl("https://junk");
1047         putService("service");
1048
1049         Ric ric = addRic("ric1");
1050         ric.setState(Ric.RicState.UNAVAILABLE);
1051         supervision.checkAllRics();
1052         waitForRicState("ric1", RicState.AVAILABLE);
1053
1054         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
1055         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
1056         ServiceCallbackInfo callbackInfo = receivedCallbacks.getReceivedInfo().get(0);
1057         assertThat(callbackInfo.ricId).isEqualTo("ric1");
1058         assertThat(callbackInfo.eventType).isEqualTo(ServiceCallbackInfo.EventType.AVAILABLE);
1059
1060         var headers = receivedCallbacks.receivedHeaders.get(0);
1061         assertThat(headers).containsEntry("authorization", "Bearer " + AUTH_TOKEN);
1062
1063         Files.delete(authFile);
1064     }
1065
1066     private Policy addPolicy(String id, String typeName, String service, String ric) throws ServiceException {
1067         addRic(ric);
1068         Policy policy = Policy.builder() //
1069                 .id(id) //
1070                 .json(jsonString()) //
1071                 .ownerServiceId(service) //
1072                 .ric(rics.getRic(ric)) //
1073                 .type(addPolicyType(typeName, ric)) //
1074                 .lastModified(Instant.now()) //
1075                 .isTransient(false) //
1076                 .statusNotificationUri("/policy-status?id=XXX") //
1077                 .build();
1078         policies.put(policy);
1079         return policy;
1080     }
1081
1082     private Policy addPolicy(String id, String typeName, String service) throws ServiceException {
1083         return addPolicy(id, typeName, service, "ric");
1084     }
1085
1086     private String createServiceJson(String name, long keepAliveIntervalSeconds) {
1087         String callbackUrl = baseUrl() + RappSimulatorController.SERVICE_CALLBACK_URL;
1088         return createServiceJson(name, keepAliveIntervalSeconds, callbackUrl);
1089     }
1090
1091     private String createServiceJson(String name, long keepAliveIntervalSeconds, String url) {
1092         ServiceRegistrationInfo service = new ServiceRegistrationInfo(name, keepAliveIntervalSeconds, url);
1093
1094         String json = gson.toJson(service);
1095         return json;
1096     }
1097
1098     private void putService(String name) {
1099         putService(name, 0, null);
1100     }
1101
1102     private void putService(String name, long keepAliveIntervalSeconds, @Nullable HttpStatus expectedStatus) {
1103         String url = "/services";
1104         String body = createServiceJson(name, keepAliveIntervalSeconds);
1105         ResponseEntity<String> resp = restClient().putForEntity(url, body).block();
1106         if (expectedStatus != null) {
1107             assertEquals(expectedStatus, resp.getStatusCode(), "");
1108         }
1109     }
1110
1111     private String jsonString() {
1112         return "{\"servingCellNrcgi\":\"1\"}";
1113     }
1114
1115     @Test
1116     @DisplayName("test Concurrency")
1117     void testConcurrency() throws Exception {
1118         this.applicationConfig.setAuthProviderUrl("");
1119         logger.info("Concurrency test starting");
1120         final Instant startTime = Instant.now();
1121         List<Thread> threads = new ArrayList<>();
1122         List<ConcurrencyTestRunnable> tests = new ArrayList<>();
1123         a1ClientFactory.setResponseDelay(Duration.ofMillis(2));
1124         addRic("ric");
1125         addPolicyType("type1", "ric");
1126         addPolicyType("type2", "ric");
1127
1128         final String NON_RESPONDING_RIC = "NonRespondingRic";
1129         Ric nonRespondingRic = addRic(NON_RESPONDING_RIC);
1130         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client(NON_RESPONDING_RIC);
1131         a1Client.setErrorInject("errorInject");
1132
1133         for (int i = 0; i < 10; ++i) {
1134             AsyncRestClient restClient = restClient();
1135             ConcurrencyTestRunnable test =
1136                     new ConcurrencyTestRunnable(restClient, supervision, a1ClientFactory, rics, policyTypes);
1137             Thread thread = new Thread(test, "TestThread_" + i);
1138             thread.start();
1139             threads.add(thread);
1140             tests.add(test);
1141         }
1142         for (Thread t : threads) {
1143             t.join();
1144         }
1145         for (ConcurrencyTestRunnable test : tests) {
1146             assertThat(test.isFailed()).isFalse();
1147         }
1148         assertThat(policies.size()).isZero();
1149         logger.info("Concurrency test took " + Duration.between(startTime, Instant.now()));
1150
1151         assertThat(nonRespondingRic.getState()).isEqualTo(RicState.UNAVAILABLE);
1152         nonRespondingRic.setState(RicState.AVAILABLE);
1153     }
1154
1155     private AsyncRestClient restClient(String baseUrl, boolean useTrustValidation) {
1156         WebClientConfig config = this.applicationConfig.getWebClientConfig();
1157         config = WebClientConfig.builder() //
1158                 .keyStoreType(config.getKeyStoreType()) //
1159                 .keyStorePassword(config.getKeyStorePassword()) //
1160                 .keyStore(config.getKeyStore()) //
1161                 .keyPassword(config.getKeyPassword()) //
1162                 .isTrustStoreUsed(useTrustValidation) //
1163                 .trustStore(config.getTrustStore()) //
1164                 .trustStorePassword(config.getTrustStorePassword()) //
1165                 .httpProxyConfig(config.getHttpProxyConfig()) //
1166                 .build();
1167
1168         AsyncRestClientFactory f = new AsyncRestClientFactory(config, new SecurityContext(""));
1169         return f.createRestClientNoHttpProxy(baseUrl);
1170
1171     }
1172
1173     private String baseUrl() {
1174         return "https://localhost:" + port;
1175     }
1176
1177     private AsyncRestClient restClient(boolean useTrustValidation) {
1178         return restClient(baseUrl() + Consts.V2_API_ROOT, useTrustValidation);
1179     }
1180
1181     private AsyncRestClient restClient() {
1182         return restClient(false);
1183     }
1184
1185     private void testErrorCode(Mono<?> request, HttpStatus expStatus) {
1186         testErrorCode(request, expStatus, "", true);
1187     }
1188
1189     private void testErrorCode(Mono<?> request, HttpStatus expStatus, boolean expectApplicationProblemJsonMediaType) {
1190         testErrorCode(request, expStatus, "", expectApplicationProblemJsonMediaType);
1191     }
1192
1193     private void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains) {
1194         testErrorCode(request, expStatus, responseContains, true);
1195     }
1196
1197     private void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains,
1198             boolean expectApplicationProblemJsonMediaType) {
1199         StepVerifier.create(request) //
1200                 .expectSubscription() //
1201                 .expectErrorMatches(
1202                         t -> checkWebClientError(t, expStatus, responseContains, expectApplicationProblemJsonMediaType)) //
1203                 .verify();
1204     }
1205
1206     private void waitForRicState(String ricId, RicState state) throws ServiceException {
1207         Ric ric = rics.getRic(ricId);
1208         await().untilAsserted(() -> state.equals(ric.getState()));
1209     }
1210
1211     private boolean checkWebClientError(Throwable throwable, HttpStatus expStatus, String responseContains,
1212             boolean expectApplicationProblemJsonMediaType) {
1213         assertTrue(throwable instanceof WebClientResponseException);
1214         WebClientResponseException responseException = (WebClientResponseException) throwable;
1215         String body = responseException.getResponseBodyAsString();
1216         assertThat(body).contains(responseContains);
1217         assertThat(responseException.getStatusCode()).isEqualTo(expStatus);
1218
1219         if (expectApplicationProblemJsonMediaType) {
1220             assertThat(responseException.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_PROBLEM_JSON);
1221         }
1222         return true;
1223     }
1224
1225     private MockA1Client getA1Client(String ricId) throws ServiceException {
1226         return a1ClientFactory.getOrCreateA1Client(ricId);
1227     }
1228
1229     private PolicyType createPolicyType(String policyTypeName) {
1230         return PolicyType.builder() //
1231                 .id(policyTypeName) //
1232                 .schema("{\"title\":\"" + policyTypeName + "\"}") //
1233                 .build();
1234     }
1235
1236     private PolicyType addPolicyType(String policyTypeName, String ricId) {
1237         PolicyType type = createPolicyType(policyTypeName);
1238         policyTypes.put(type);
1239         addRic(ricId).addSupportedPolicyType(type);
1240         return type;
1241     }
1242
1243     private Ric addRic(String ricId) {
1244         return addRic(ricId, null);
1245     }
1246
1247     private RicConfig ricConfig(String ricId, String managedElement) {
1248         List<String> mes = new ArrayList<>();
1249         if (managedElement != null) {
1250             mes.add(managedElement);
1251         }
1252         return RicConfig.builder() //
1253                 .ricId(ricId) //
1254                 .baseUrl(ricId) //
1255                 .managedElementIds(mes) //
1256                 .build();
1257     }
1258
1259     private Ric addRic(String ricId, String managedElement) {
1260         if (rics.get(ricId) != null) {
1261             return rics.get(ricId);
1262         }
1263
1264         RicConfig conf = ricConfig(ricId, managedElement);
1265         Ric ric = new Ric(conf);
1266         ric.setState(Ric.RicState.AVAILABLE);
1267         this.rics.put(ric);
1268         return ric;
1269     }
1270
1271 }