gerrit.onap Code Review - ccsdk/oran.git/blob

   1 /*-
   2  * ========================LICENSE_START=================================
   3  * ONAP : ccsdk oran
   4  * ======================================================================
   5  * Copyright (C) 2019-2022 Nordix Foundation. All rights reserved.
   6  * ======================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ========================LICENSE_END===================================
  19  */
  20
  21 package org.onap.ccsdk.oran.a1policymanagementservice.controllers.v2;
  22
  23 import static org.assertj.core.api.Assertions.assertThat;
  24 import static org.awaitility.Awaitility.await;
  25 import static org.junit.jupiter.api.Assertions.assertEquals;
  26 import static org.junit.jupiter.api.Assertions.assertTrue;
  27 import static org.mockito.ArgumentMatchers.any;
  28 import static org.mockito.Mockito.doReturn;
  29
  30 import com.google.gson.Gson;
  31 import com.google.gson.GsonBuilder;
  32
  33 import java.io.FileOutputStream;
  34 import java.io.PrintStream;
  35 import java.lang.invoke.MethodHandles;
  36 import java.nio.charset.StandardCharsets;
  37 import java.nio.file.Files;
  38 import java.nio.file.Path;
  39 import java.nio.file.Paths;
  40 import java.time.Duration;
  41 import java.time.Instant;
  42 import java.util.ArrayList;
  43 import java.util.Collections;
  44 import java.util.List;
  45
  46 import org.json.JSONObject;
  47 import org.junit.jupiter.api.AfterAll;
  48 import org.junit.jupiter.api.AfterEach;
  49 import org.junit.jupiter.api.MethodOrderer;
  50 import org.junit.jupiter.api.Test;
  51 import org.junit.jupiter.api.TestMethodOrder;
  52 import org.onap.ccsdk.oran.a1policymanagementservice.clients.A1ClientFactory;
  53 import org.onap.ccsdk.oran.a1policymanagementservice.clients.AsyncRestClient;
  54 import org.onap.ccsdk.oran.a1policymanagementservice.clients.AsyncRestClientFactory;
  55 import org.onap.ccsdk.oran.a1policymanagementservice.clients.SecurityContext;
  56 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationConfig;
  57 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationConfig.RicConfigUpdate;
  58 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.RicConfig;
  59 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.WebClientConfig;
  60 import org.onap.ccsdk.oran.a1policymanagementservice.controllers.ServiceCallbackInfo;
  61 import org.onap.ccsdk.oran.a1policymanagementservice.exceptions.ServiceException;
  62 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Lock;
  63 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Lock.LockType;
  64 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policies;
  65 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policy;
  66 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyType;
  67 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyTypes;
  68 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric;
  69 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric.RicState;
  70 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Rics;
  71 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Service;
  72 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Services;
  73 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.RefreshConfigTask;
  74 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.RicSupervision;
  75 import org.onap.ccsdk.oran.a1policymanagementservice.tasks.ServiceSupervision;
  76 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1Client;
  77 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1ClientFactory;
  78 import org.slf4j.Logger;
  79 import org.slf4j.LoggerFactory;
  80 import org.springframework.beans.factory.annotation.Autowired;
  81 import org.springframework.boot.test.context.SpringBootTest;
  82 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
  83 import org.springframework.boot.test.context.TestConfiguration;
  84 import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
  85 import org.springframework.boot.web.server.LocalServerPort;
  86 import org.springframework.boot.web.servlet.server.ServletWebServerFactory;
  87 import org.springframework.context.ApplicationContext;
  88 import org.springframework.context.annotation.Bean;
  89 import org.springframework.http.HttpStatus;
  90 import org.springframework.http.MediaType;
  91 import org.springframework.http.ResponseEntity;
  92 import org.springframework.test.context.TestPropertySource;
  93 import org.springframework.util.FileSystemUtils;
  94 import org.springframework.web.reactive.function.client.WebClientRequestException;
  95 import org.springframework.web.reactive.function.client.WebClientResponseException;
  96
  97 import reactor.core.publisher.Mono;
  98 import reactor.test.StepVerifier;
  99 import reactor.util.annotation.Nullable;
 100
 101 @TestMethodOrder(MethodOrderer.MethodName.class)
 102 @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
 103 @TestPropertySource(properties = { //
 104         "server.ssl.key-store=./config/keystore.jks", //
 105         "app.webclient.trust-store=./config/truststore.jks", //
 106         "app.webclient.trust-store-used=true", //
 107         "app.vardata-directory=/tmp/pmstest", //
 108         "app.filepath=", //
 109         "app.s3.bucket=" // If this is set, S3 will be used to store data.
 110 })
 111 class ApplicationTest {
 112     private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 113
 114     @Autowired
 115     ApplicationContext context;
 116
 117     @Autowired
 118     private Rics rics;
 119
 120     @Autowired
 121     private Policies policies;
 122
 123     @Autowired
 124     private PolicyTypes policyTypes;
 125
 126     @Autowired
 127     MockA1ClientFactory a1ClientFactory;
 128
 129     @Autowired
 130     RicSupervision supervision;
 131
 132     @Autowired
 133     ApplicationConfig applicationConfig;
 134
 135     @Autowired
 136     Services services;
 137
 138     @Autowired
 139     RappSimulatorController rAppSimulator;
 140
 141     @Autowired
 142     RefreshConfigTask refreshConfigTask;
 143
 144     @Autowired
 145     SecurityContext securityContext;
 146
 147     private static Gson gson = new GsonBuilder().create();
 148
 149     /**
 150      * Overrides the BeanFactory.
 151      */
 152     @TestConfiguration
 153     static class TestBeanFactory {
 154
 155         @Bean
 156         A1ClientFactory getA1ClientFactory(@Autowired ApplicationConfig appConfig, @Autowired PolicyTypes types) {
 157             return new MockA1ClientFactory(appConfig, types);
 158         }
 159
 160         @Bean
 161         public ServiceSupervision getServiceSupervision(@Autowired Services services,
 162                 @Autowired A1ClientFactory a1ClientFactory, @Autowired Policies policies) {
 163             Duration checkInterval = Duration.ofMillis(1);
 164             return new ServiceSupervision(services, policies, a1ClientFactory, checkInterval);
 165         }
 166
 167         @Bean
 168         public ServletWebServerFactory servletContainer() {
 169             return new TomcatServletWebServerFactory();
 170         }
 171     }
 172
 173     @LocalServerPort
 174     private int port;
 175
 176     @AfterEach
 177     void reset() {
 178         rics.clear();
 179         policies.clear();
 180         policyTypes.clear();
 181         services.clear();
 182         a1ClientFactory.reset();
 183         this.rAppSimulator.getTestResults().clear();
 184         this.a1ClientFactory.setPolicyTypes(policyTypes); // Default same types in RIC and in this app
 185         this.securityContext.setAuthTokenFilePath(null);
 186     }
 187
 188     @AfterAll
 189     static void clearTestDir() {
 190         try {
 191             FileSystemUtils.deleteRecursively(Path.of("/tmp/pmstest"));
 192         } catch (Exception e) {
 193             logger.warn("Could test directory : {}", e.getMessage());
 194         }
 195     }
 196
 197     @AfterEach
 198     void verifyNoRicLocks() {
 199         for (Ric ric : this.rics.getRics()) {
 200             Lock.Grant grant = ric.getLock().lockBlocking(LockType.EXCLUSIVE, "verifyNoRicLocks");
 201             grant.unlockBlocking();
 202             assertThat(ric.getLock().getLockCounter()).isZero();
 203             assertThat(ric.getState()).isEqualTo(Ric.RicState.AVAILABLE);
 204         }
 205     }
 206
 207     @Test
 208     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 209     void testZZActuator() throws Exception {
 210         // The test must be run last, hence the "ZZ" in the name. All succeeding tests
 211         // will fail.
 212         AsyncRestClient client = restClient(baseUrl(), false);
 213
 214         client.post("/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice",
 215                 "{\"configuredLevel\":\"trace\"}").block();
 216
 217         String resp = client.get("/actuator/loggers/org.onap.ccsdk.oran.a1policymanagementservice").block();
 218         assertThat(resp).contains("TRACE");
 219
 220         client.post("/actuator/loggers/org.springframework.boot.actuate", "{\"configuredLevel\":\"trace\"}").block();
 221
 222         // This will stop the web server and all coming tests will fail.
 223         client.post("/actuator/shutdown", "").block();
 224
 225         Thread.sleep(1000);
 226
 227         StepVerifier.create(restClient().get("/rics")) // Any call
 228                 .expectSubscription() //
 229                 .expectErrorMatches(t -> t instanceof WebClientRequestException) //
 230                 .verify();
 231
 232     }
 233
 234     @Test
 235     void generateApiDoc() throws Exception {
 236         String url = "https://localhost:" + this.port + "/v3/api-docs";
 237         ResponseEntity<String> resp = restClient("", false).getForEntity(url).block();
 238         assertThat(resp.getStatusCode()).isEqualTo(HttpStatus.OK);
 239         JSONObject jsonObj = new JSONObject(resp.getBody());
 240         assertThat(jsonObj.remove("servers")).isNotNull();
 241
 242         String indented = (jsonObj).toString(4);
 243         String docDir = "api/";
 244         Files.createDirectories(Paths.get(docDir));
 245         try (PrintStream out = new PrintStream(new FileOutputStream(docDir + "pms-api.json"))) {
 246             out.print(indented);
 247         }
 248     }
 249
 250     @Test
 251     void testPersistencyPolicies() throws Exception {
 252         Ric ric = this.addRic("ric1");
 253         PolicyType type = this.addPolicyType("type1", ric.id());
 254
 255         final int noOfPolicies = 100;
 256         for (int i = 0; i < noOfPolicies; ++i) {
 257             addPolicy("id" + i, type.getId(), "service", ric.id());
 258         }
 259         waitforS3();
 260
 261         {
 262             Policies policies = new Policies(this.applicationConfig);
 263             policies.restoreFromDatabase(ric, this.policyTypes).blockLast();
 264             assertThat(policies.size()).isEqualTo(noOfPolicies);
 265         }
 266
 267         {
 268             restClient().delete("/policies/id2").block();
 269             Policies policies = new Policies(this.applicationConfig);
 270             policies.restoreFromDatabase(ric, this.policyTypes).blockLast();
 271             assertThat(policies.size()).isEqualTo(noOfPolicies - 1);
 272         }
 273     }
 274
 275     @Test
 276     void testPersistencyPolicyTypes() throws Exception {
 277         Ric ric = this.addRic("ric1");
 278         this.addPolicyType("type1", ric.id());
 279         waitforS3();
 280
 281         PolicyTypes types = new PolicyTypes(this.applicationConfig);
 282         types.restoreFromDatabase().blockLast();
 283         assertThat(types.size()).isEqualTo(1);
 284     }
 285
 286     @SuppressWarnings("squid:S2925") // "Thread.sleep" should not be used in tests.
 287     private void waitforS3() throws Exception {
 288         if (applicationConfig.isS3Enabled()) {
 289             Thread.sleep(1000);
 290         }
 291     }
 292
 293     @Test
 294     void testPersistencyService() throws Exception {
 295         final String SERVICE = "serviceName";
 296         putService(SERVICE, 1234, HttpStatus.CREATED);
 297         assertThat(this.services.size()).isEqualTo(1);
 298         Service service = this.services.getService(SERVICE);
 299         waitforS3();
 300
 301         Services servicesRestored = new Services(this.applicationConfig);
 302         servicesRestored.restoreFromDatabase().blockLast();
 303         Service serviceRestored = servicesRestored.getService(SERVICE);
 304         assertThat(servicesRestored.size()).isEqualTo(1);
 305         assertThat(serviceRestored.getCallbackUrl()).isEqualTo(service.getCallbackUrl());
 306         assertThat(serviceRestored.getKeepAliveInterval()).isEqualTo(service.getKeepAliveInterval());
 307
 308         // check that the service can be deleted
 309         this.services.remove(SERVICE);
 310         servicesRestored = new Services(this.applicationConfig);
 311         assertThat(servicesRestored.size()).isZero();
 312     }
 313
 314     @Test
 315     void testAddingRicFromConfiguration() throws Exception {
 316         // Test adding the RIC from configuration
 317
 318         final String RIC = "ric1";
 319         final String TYPE = "type123";
 320         PolicyTypes nearRtRicPolicyTypes = new PolicyTypes(this.applicationConfig);
 321         nearRtRicPolicyTypes.put(createPolicyType(TYPE));
 322         this.a1ClientFactory.setPolicyTypes(nearRtRicPolicyTypes);
 323
 324         putService("service");
 325
 326         refreshConfigTask.handleUpdatedRicConfig( //
 327                 new RicConfigUpdate(ricConfig(RIC, "me1"), RicConfigUpdate.Type.ADDED)) //
 328                 .block();
 329         waitForRicState(RIC, RicState.AVAILABLE);
 330
 331         // Test that the type has been synched
 332         Ric addedRic = this.rics.getRic(RIC);
 333         assertThat(addedRic.getSupportedPolicyTypes()).hasSize(1);
 334         assertThat(addedRic.getSupportedPolicyTypes().iterator().next().getId()).isEqualTo(TYPE);
 335
 336         // Check that a service callback for the AVAILABLE RIC is invoked
 337         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
 338         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
 339         ServiceCallbackInfo callbackInfo = receivedCallbacks.getReceivedInfo().get(0);
 340         assertThat(callbackInfo.ricId).isEqualTo(RIC);
 341         assertThat(callbackInfo.eventType).isEqualTo(ServiceCallbackInfo.EventType.AVAILABLE);
 342     }
 343
 344     @Test
 345     void testAddingRicFromConfiguration_nonRespondingRic() throws Exception {
 346         putService("service");
 347
 348         final String RIC = "NonRespondingRic";
 349         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client(RIC);
 350         doReturn(MockA1Client.monoError("error", HttpStatus.BAD_GATEWAY)).when(a1Client).getPolicyTypeIdentities();
 351
 352         refreshConfigTask.handleUpdatedRicConfig( //
 353                 new RicConfigUpdate(ricConfig(RIC, "me1"), RicConfigUpdate.Type.ADDED)) //
 354                 .block();
 355
 356         waitForRicState(RIC, RicState.UNAVAILABLE);
 357
 358         // Check that no service callback for the UNAVAILABLE RIC is invoked
 359         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
 360         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).isEmpty());
 361
 362         // Run a synch and check that the AVAILABLE notification is received
 363         a1ClientFactory.reset();
 364         supervision.checkAllRics();
 365         waitForRicState(RIC, RicState.AVAILABLE);
 366
 367         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
 368     }
 369
 370     @Test
 371     void testTrustValidation() {
 372         addRic("ric1");
 373
 374         String rsp = restClient(true).get("/rics").block(); // restClient(true) enables trust validation
 375         assertThat(rsp).contains("ric1");
 376
 377     }
 378
 379     @Test
 380     void testGetRics() throws Exception {
 381         addRic("ric1");
 382         this.addPolicyType("type1", "ric1");
 383         String url = "/rics?policytype_id=type1";
 384         String rsp = restClient().get(url).block();
 385         assertThat(rsp).contains("ric1");
 386
 387         // nameless type for ORAN A1 1.1
 388         addRic("ric2");
 389         this.addPolicyType("", "ric2");
 390         url = "/rics?policytype_id=";
 391         rsp = restClient().get(url).block();
 392         assertThat(rsp).contains("ric2") //
 393                 .doesNotContain("ric1") //
 394                 .contains("AVAILABLE");
 395
 396         // All RICs
 397         rsp = restClient().get("/rics").block();
 398         assertThat(rsp).contains("ric2") //
 399                 .contains("ric1");
 400
 401         // Non existing policy type
 402         url = "/rics?policytype_id=XXXX";
 403         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 404     }
 405
 406     @Test
 407     void testSynchronization() throws Exception {
 408         // Two polictypes will be put in the NearRT RICs
 409         PolicyTypes nearRtRicPolicyTypes = new PolicyTypes(this.applicationConfig);
 410         nearRtRicPolicyTypes.put(createPolicyType("typeName"));
 411         nearRtRicPolicyTypes.put(createPolicyType("typeName2"));
 412         this.a1ClientFactory.setPolicyTypes(nearRtRicPolicyTypes);
 413
 414         // One type and one instance added to the Policy Management Service's storage
 415         final String ric1Name = "ric1";
 416         Ric ric1 = addRic(ric1Name);
 417         Policy policy2 = addPolicy("policyId2", "typeName", "service", ric1Name);
 418         Ric ric2 = addRic("ric2");
 419
 420         getA1Client(ric1Name).putPolicy(policy2); // put it in the RIC (Near-RT RIC)
 421         policies.remove(policy2); // Remove it from the repo -> should be deleted in the RIC
 422
 423         String policyId = "policyId";
 424         Policy policy = addPolicy(policyId, "typeName", "service", ric1Name); // This should be created in the RIC
 425         supervision.checkAllRics(); // The created policy should be put in the RIC
 426
 427         // Wait until synch is completed
 428         waitForRicState(ric1Name, RicState.SYNCHRONIZING);
 429         waitForRicState(ric1Name, RicState.AVAILABLE);
 430         waitForRicState("ric2", RicState.AVAILABLE);
 431
 432         Policies ricPolicies = getA1Client(ric1Name).getPolicies();
 433         assertThat(ricPolicies.size()).isEqualTo(1);
 434         Policy ricPolicy = ricPolicies.get(policyId);
 435         assertThat(ricPolicy.getJson()).isEqualTo(policy.getJson());
 436
 437         // Both types should be in the Policy Management Service's storage after the
 438         // synch
 439         assertThat(ric1.getSupportedPolicyTypes()).hasSize(2);
 440         assertThat(ric2.getSupportedPolicyTypes()).hasSize(2);
 441     }
 442
 443     @Test
 444     void testGetRic() throws Exception {
 445         String ricId = "ric1";
 446         String managedElementId = "kista_1";
 447         addRic(ricId, managedElementId);
 448
 449         String url = "/rics/ric?managed_element_id=" + managedElementId;
 450         String rsp = restClient().get(url).block();
 451         RicInfo ricInfo = gson.fromJson(rsp, RicInfo.class);
 452         assertThat(ricInfo.ricId).isEqualTo(ricId);
 453
 454         url = "/rics/ric?ric_id=" + ricId;
 455         rsp = restClient().get(url).block();
 456         ricInfo = gson.fromJson(rsp, RicInfo.class);
 457         assertThat(ricInfo.ricId).isEqualTo(ricId);
 458
 459         // test GET RIC for ManagedElement that does not exist
 460         url = "/rics/ric?managed_element_id=" + "junk";
 461         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 462
 463         url = "/rics/ric";
 464         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST);
 465     }
 466
 467     private String putPolicyBody(String serviceName, String ricId, String policyTypeName, String policyInstanceId,
 468             boolean isTransient, String statusNotificationUri) {
 469         PolicyInfo info = new PolicyInfo();
 470         info.policyId = policyInstanceId;
 471         info.policyTypeId = policyTypeName;
 472         info.ricId = ricId;
 473         info.serviceId = serviceName;
 474         info.policyData = gson.fromJson(jsonString(), Object.class);
 475
 476         if (isTransient) {
 477             info.isTransient = isTransient;
 478         }
 479         info.statusNotificationUri = statusNotificationUri;
 480         return gson.toJson(info);
 481     }
 482
 483     private String putPolicyBody(String serviceName, String ricId, String policyTypeName, String policyInstanceId) {
 484         return putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId, false, "statusUri");
 485     }
 486
 487     @Test
 488     void testPutPolicy() throws Exception {
 489         String serviceName = "service.1";
 490         String ricId = "ric.1";
 491         String policyTypeName = "type1_1.2.3";
 492         String policyInstanceId = "instance_1.2.3";
 493
 494         putService(serviceName);
 495         addPolicyType(policyTypeName, ricId);
 496
 497         // PUT a transient policy
 498         String url = "/policies";
 499         String policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId, true, "statusNotif");
 500         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
 501
 502         restClient().put(url, policyBody).block();
 503
 504         Policy policy = policies.getPolicy(policyInstanceId);
 505         assertThat(policy).isNotNull();
 506         assertThat(policy.getId()).isEqualTo(policyInstanceId);
 507         assertThat(policy.getOwnerServiceId()).isEqualTo(serviceName);
 508         assertThat(policy.getRic().id()).isEqualTo(ricId);
 509         assertThat(policy.isTransient()).isTrue();
 510
 511         // Put a non transient policy
 512         policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId);
 513         restClient().put(url, policyBody).block();
 514         policy = policies.getPolicy(policyInstanceId);
 515         assertThat(policy.isTransient()).isFalse();
 516
 517         url = "/policy-instances";
 518         String rsp = restClient().get(url).block();
 519         assertThat(rsp).as("Response contains policy instance ID.").contains(policyInstanceId);
 520
 521         url = "/policies/" + policyInstanceId;
 522         rsp = restClient().get(url).block();
 523         assertThat(rsp).contains(policyBody);
 524
 525         // Test of error codes
 526         url = "/policies";
 527         policyBody = putPolicyBody(serviceName, ricId + "XX", policyTypeName, policyInstanceId);
 528         testErrorCode(restClient().put(url, policyBody), HttpStatus.NOT_FOUND);
 529
 530         policyBody = putPolicyBody(serviceName, ricId, policyTypeName + "XX", policyInstanceId);
 531         addPolicyType(policyTypeName + "XX", "otherRic");
 532         testErrorCode(restClient().put(url, policyBody), HttpStatus.NOT_FOUND);
 533
 534         policyBody = putPolicyBody(serviceName, ricId, policyTypeName, policyInstanceId);
 535         this.rics.getRic(ricId).setState(Ric.RicState.SYNCHRONIZING);
 536         testErrorCode(restClient().put(url, policyBody), HttpStatus.LOCKED);
 537         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
 538     }
 539
 540     @Test
 541     void testPutPolicy_NoServiceNoStatusUri() throws Exception {
 542         String ricId = "ric.1";
 543         String policyTypeName = "type1_1.2.3";
 544         String policyInstanceId = "instance_1.2.3";
 545
 546         addPolicyType(policyTypeName, ricId);
 547
 548         // PUT a transient policy
 549         String url = "/policies";
 550         String policyBody = putPolicyBody(null, ricId, policyTypeName, policyInstanceId, true, null);
 551         this.rics.getRic(ricId).setState(Ric.RicState.AVAILABLE);
 552
 553         restClient().put(url, policyBody).block();
 554
 555         Policy policy = policies.getPolicy(policyInstanceId);
 556         assertThat(policy).isNotNull();
 557         assertThat(policy.getOwnerServiceId()).isBlank();
 558         assertThat(policy.getStatusNotificationUri()).isBlank();
 559     }
 560
 561     @Test
 562     /**
 563      * Test that HttpStatus and body from failing REST call to A1 is passed on to
 564      * the caller.
 565      *
 566      * @throws ServiceException
 567      */
 568     void testErrorFromRic() throws ServiceException {
 569         putService("service1");
 570         addPolicyType("type1", "ric1");
 571
 572         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client("ric1");
 573         HttpStatus httpStatus = HttpStatus.INTERNAL_SERVER_ERROR;
 574         String responseBody = "Refused";
 575         byte[] responseBodyBytes = responseBody.getBytes(StandardCharsets.UTF_8);
 576
 577         WebClientResponseException a1Exception = new WebClientResponseException(httpStatus.value(), "statusText", null,
 578                 responseBodyBytes, StandardCharsets.UTF_8, null);
 579         doReturn(Mono.error(a1Exception)).when(a1Client).putPolicy(any());
 580
 581         // PUT Policy
 582         String putBody = putPolicyBody("service1", "ric1", "type1", "id1");
 583         String url = "/policies";
 584         testErrorCode(restClient().put(url, putBody), httpStatus, responseBody);
 585
 586         // DELETE POLICY
 587         this.addPolicy("instance1", "type1", "service1", "ric1");
 588         doReturn(Mono.error(a1Exception)).when(a1Client).deletePolicy(any());
 589         testErrorCode(restClient().delete("/policies/instance1"), httpStatus, responseBody);
 590
 591     }
 592
 593     @Test
 594     void testPutTypelessPolicy() throws Exception {
 595         putService("service1");
 596         addPolicyType("", "ric1");
 597         String body = putPolicyBody("service1", "ric1", "", "id1");
 598         restClient().put("/policies", body).block();
 599
 600         String rsp = restClient().get("/policy-instances").block();
 601         PolicyInfoList info = gson.fromJson(rsp, PolicyInfoList.class);
 602         assertThat(info.policies).hasSize(1);
 603         PolicyInfo policyInfo = info.policies.iterator().next();
 604         assertThat(policyInfo.policyId).isEqualTo("id1");
 605         assertThat(policyInfo.policyTypeId).isEmpty();
 606     }
 607
 608     @Test
 609     void testUpdateService() throws Exception {
 610         this.addRic("ric1");
 611         this.addPolicy("p", "type1", "", "ric1");
 612
 613         String url = "/policies?service_id=";
 614         String resp = restClient().get(url).block();
 615         assertThat(resp).contains("[\"p\"]");
 616
 617         this.addPolicy("p", "type1", "service", "ric1");
 618         url = "/policies?service_id=";
 619         resp = restClient().get(url).block();
 620         assertThat(resp).contains("[]");
 621
 622         url = "/policies?service_id=service";
 623         resp = restClient().get(url).block();
 624         assertThat(resp).contains("[\"p\"]");
 625     }
 626
 627     @Test
 628     void testRefuseToUpdatePolicy() throws Exception {
 629         // Test that only the json can be changed for a already created policy
 630         // In this case service is attempted to be changed
 631         this.addRic("ric1");
 632         this.addRic("ricXXX");
 633         this.addPolicy("instance1", "type1", "service1", "ric1");
 634         this.addPolicy("instance2", "type1", "service1", "ricXXX");
 635
 636         // Try change ric1 -> ricXXX
 637         String bodyWrongRic = putPolicyBody("service1", "ricXXX", "type1", "instance1");
 638         testErrorCode(restClient().put("/policies", bodyWrongRic), HttpStatus.CONFLICT);
 639     }
 640
 641     @Test
 642     void testGetPolicy() throws Exception {
 643         String url = "/policies/id";
 644         Policy policy = addPolicy("id", "typeName", "service1", "ric1");
 645         {
 646             String rsp = restClient().get(url).block();
 647             PolicyInfo info = gson.fromJson(rsp, PolicyInfo.class);
 648             String policyStr = gson.toJson(info.policyData);
 649             assertThat(policyStr).isEqualTo(policy.getJson());
 650         }
 651         {
 652             policies.remove(policy);
 653             testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 654         }
 655     }
 656
 657     @Test
 658     void testDeletePolicy() throws Exception {
 659         String policyId = "id.1";
 660         addPolicy(policyId, "typeName", "service1", "ric1");
 661         assertThat(policies.size()).isEqualTo(1);
 662
 663         String url = "/policies/" + policyId;
 664         ResponseEntity<String> entity = restClient().deleteForEntity(url).block();
 665
 666         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.NO_CONTENT);
 667         assertThat(policies.size()).isZero();
 668
 669         // Delete a non existing policy
 670         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 671     }
 672
 673     @Test
 674     void testGetPolicyType() throws Exception {
 675         String typeId = "AC.D";
 676         addPolicyType(typeId, "ric1");
 677
 678         waitForRicState("ric1", RicState.AVAILABLE);
 679
 680         String url = "/policy-types/" + typeId;
 681
 682         String rsp = this.restClient().get(url).block();
 683
 684         PolicyTypeInfo info = gson.fromJson(rsp, PolicyTypeInfo.class);
 685         assertThat(info.schema).isNotNull();
 686
 687         // Get non existing schema
 688         url = "/policy-types/JUNK";
 689         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 690     }
 691
 692     String createPolicyTypesJson(String... types) {
 693         List<String> list = new ArrayList<>();
 694         Collections.addAll(list, types);
 695         PolicyTypeIdList ids = new PolicyTypeIdList(list);
 696         return gson.toJson(ids);
 697     }
 698
 699     @Test
 700     void testGetPolicyTypes() throws Exception {
 701         String TYPE_ID_1 = "A_type1_1.9.0";
 702         String TYPE_ID_2 = "A_type1_2.0.0";
 703         String TYPE_ID_3 = "A_type1_1.5.0";
 704         String TYPE_ID_4 = "type3_1.9.0";
 705         addPolicyType(TYPE_ID_1, "ric1");
 706         addPolicyType(TYPE_ID_2, "ric2");
 707         addPolicyType(TYPE_ID_3, "ric2");
 708         addPolicyType(TYPE_ID_4, "ric2");
 709
 710         addPolicyType("junk", "ric2");
 711         addPolicyType("junk_a.b.c", "ric2");
 712
 713         String url = "/policy-types";
 714         String rsp = restClient().get(url).block();
 715         assertThat(rsp).contains(TYPE_ID_1, TYPE_ID_2);
 716
 717         url = "/policy-types?ric_id=ric1";
 718         rsp = restClient().get(url).block();
 719         String expResp = createPolicyTypesJson(TYPE_ID_1);
 720         assertThat(rsp).isEqualTo(expResp);
 721
 722         // Get policy types for non existing RIC
 723         url = "/policy-types?ric_id=ric1XXX";
 724         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 725
 726         // All types with a type_name
 727         url = "/policy-types?type_name=A_type1";
 728         rsp = restClient().get(url).block();
 729         assertThat(rsp).contains(TYPE_ID_1, TYPE_ID_2);
 730
 731         // All types compatible with type1_1.5.0 (which is type1_1.9.0)
 732         url = "/policy-types?type_name=A_type1&&compatible_with_version=1.5.0";
 733         rsp = restClient().get(url).block();
 734         expResp = createPolicyTypesJson(TYPE_ID_3, TYPE_ID_1);
 735         assertThat(rsp).isEqualTo(expResp);
 736
 737         url = "/policy-types?type_name=A_type1&&compatible_with_version=junk";
 738         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "Version must contain major.minor.patch code");
 739
 740         url = "/policy-types?type_name=A_type1&&compatible_with_version=a.b.c";
 741         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "Syntax error in");
 742
 743         url = "/policy-types?compatible_with_version=1.5.0";
 744         testErrorCode(restClient().get(url), HttpStatus.BAD_REQUEST, "type_name");
 745     }
 746
 747     @Test
 748     void testGetPolicyInstances() throws Exception {
 749         addPolicy("id1", "type1", "service1");
 750
 751         String url = "/policy-instances";
 752         String rsp = restClient().get(url).block();
 753         logger.info(rsp);
 754         PolicyInfoList info = gson.fromJson(rsp, PolicyInfoList.class);
 755         assertThat(info.policies).hasSize(1);
 756         PolicyInfo policyInfo = info.policies.iterator().next();
 757         assert (policyInfo.validate());
 758         assertThat(policyInfo.policyId).isEqualTo("id1");
 759         assertThat(policyInfo.policyTypeId).isEqualTo("type1");
 760         assertThat(policyInfo.serviceId).isEqualTo("service1");
 761     }
 762
 763     @Test
 764     void testGetPolicyInstancesFilter() throws Exception {
 765         addPolicy("id1", "type1", "service1");
 766         addPolicy("id2", "type1", "service2");
 767         addPolicy("id3", "type2", "service1");
 768         addPolicy("id4", "type1_1.0.0", "service1");
 769
 770         String url = "/policy-instances?policytype_id=type1";
 771         String rsp = restClient().get(url).block();
 772         logger.info(rsp);
 773         assertThat(rsp).contains("id1") //
 774                 .contains("id2") //
 775                 .doesNotContain("id3");
 776
 777         url = "/policy-instances?policytype_id=type1&service_id=service2";
 778         rsp = restClient().get(url).block();
 779         logger.info(rsp);
 780         assertThat(rsp).doesNotContain("id1") //
 781                 .contains("id2") //
 782                 .doesNotContain("id3");
 783
 784         url = "/policy-instances?type_name=type1";
 785         rsp = restClient().get(url).block();
 786         assertThat(rsp).contains("id1") //
 787                 .contains("id2") //
 788                 .doesNotContain("id3") //
 789                 .contains("id4");
 790
 791         // Test get policies for non existing type
 792         url = "/policy-instances?policytype_id=type1XXX";
 793         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 794
 795         // Test get policies for non existing RIC
 796         url = "/policy-instances?ric_id=XXX";
 797         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 798     }
 799
 800     @Test
 801     void testGetPolicyIdsFilter() throws Exception {
 802         addPolicy("id1", "type1", "service1", "ric1");
 803         addPolicy("id2", "type1", "service2", "ric1");
 804         addPolicy("id3", "type2", "service1", "ric1");
 805         addPolicy("id4", "type1_1.0.0", "service1");
 806
 807         String url = "/policies?policytype_id=type1";
 808         String rsp = restClient().get(url).block();
 809         logger.info(rsp);
 810         assertThat(rsp).contains("id1") //
 811                 .contains("id2") //
 812                 .doesNotContain("id3");
 813
 814         url = "/policies?policytype_id=type1&service_id=service1&ric=ric1";
 815         rsp = restClient().get(url).block();
 816         PolicyIdList respList = gson.fromJson(rsp, PolicyIdList.class);
 817         assertThat(respList.policyIds.iterator().next()).isEqualTo("id1");
 818
 819         url = "/policies?policytype_name=type1&service_id=service1";
 820         rsp = restClient().get(url).block();
 821         assertThat(rsp).contains("id1") //
 822                 .contains("id3") //
 823                 .contains("id4");
 824         assertThat(gson.fromJson(rsp, PolicyIdList.class).policyIds).hasSize(3);
 825
 826         // Test get policy ids for non existing type
 827         url = "/policies?policytype_id=type1XXX";
 828         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 829
 830         // Test get policy ids for non existing RIC
 831         url = "/policies?ric_id=XXX";
 832         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 833     }
 834
 835     @Test
 836     void testPutAndGetService() throws Exception {
 837         // PUT
 838         String serviceName = "ac.dc";
 839         putService(serviceName, 0, HttpStatus.CREATED);
 840         putService(serviceName, 0, HttpStatus.OK);
 841
 842         // GET one service
 843         String url = "/services?service_id=" + serviceName;
 844         String rsp = restClient().get(url).block();
 845         ServiceStatusList info = gson.fromJson(rsp, ServiceStatusList.class);
 846         assertThat(info.statusList).hasSize(1);
 847         ServiceStatus status = info.statusList.iterator().next();
 848         assertThat(status.keepAliveIntervalSeconds).isZero();
 849         assertThat(status.serviceId).isEqualTo(serviceName);
 850
 851         // GET (all)
 852         url = "/services";
 853         rsp = restClient().get(url).block();
 854         assertThat(rsp).as("Response contains service name").contains(serviceName);
 855         logger.info(rsp);
 856
 857         // Keep alive
 858         url = "/services/" + serviceName + "/keepalive";
 859         ResponseEntity<?> entity = restClient().putForEntity(url).block();
 860         assertThat(entity.getStatusCode()).isEqualTo(HttpStatus.OK);
 861
 862         // DELETE service
 863         assertThat(services.size()).isEqualTo(1);
 864         url = "/services/" + serviceName;
 865         restClient().delete(url).block();
 866         assertThat(services.size()).isZero();
 867
 868         // Keep alive, no registered service
 869         testErrorCode(restClient().put("/services/junk/keepalive", ""), HttpStatus.NOT_FOUND);
 870
 871         // PUT service with bad payload
 872         testErrorCode(restClient().put("/services", "crap"), HttpStatus.BAD_REQUEST, false);
 873         testErrorCode(restClient().put("/services", "{}"), HttpStatus.BAD_REQUEST, false);
 874         testErrorCode(restClient().put("/services", createServiceJson(serviceName, -123)), HttpStatus.BAD_REQUEST,
 875                 false);
 876         testErrorCode(restClient().put("/services", createServiceJson(serviceName, 0, "missing.portandprotocol.com")),
 877                 HttpStatus.BAD_REQUEST, false);
 878
 879         // GET non existing service
 880         testErrorCode(restClient().get("/services?service_id=XXX"), HttpStatus.NOT_FOUND);
 881     }
 882
 883     @Test
 884     void testServiceSupervision() throws Exception {
 885         putService("service1", 1, HttpStatus.CREATED);
 886         addPolicyType("type1", "ric1");
 887
 888         String policyBody = putPolicyBody("service1", "ric1", "type1", "instance1");
 889         restClient().put("/policies", policyBody).block();
 890
 891         assertThat(policies.size()).isEqualTo(1);
 892         assertThat(services.size()).isEqualTo(1);
 893
 894         // Timeout after ~1 second
 895         await().untilAsserted(() -> assertThat(policies.size()).isZero());
 896         assertThat(services.size()).isZero();
 897     }
 898
 899     @Test
 900     void testGetPolicyStatus() throws Exception {
 901         addPolicy("id", "typeName", "service1", "ric1");
 902         assertThat(policies.size()).isEqualTo(1);
 903
 904         String url = "/policies/id/status";
 905         String rsp = restClient().get(url).block();
 906         PolicyStatusInfo info = gson.fromJson(rsp, PolicyStatusInfo.class);
 907         assertThat(info.status).isEqualTo("OK");
 908
 909         // GET non existing policy status
 910         url = "/policies/XXX/status";
 911         testErrorCode(restClient().get(url), HttpStatus.NOT_FOUND);
 912
 913         // GET STATUS, the NearRT RIC returns error
 914         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client("ric1");
 915         url = "/policies/id/status";
 916         WebClientResponseException a1Exception = new WebClientResponseException(404, "", null, null, null);
 917         doReturn(Mono.error(a1Exception)).when(a1Client).getPolicyStatus(any());
 918         rsp = restClient().get(url).block();
 919         info = gson.fromJson(rsp, PolicyStatusInfo.class);
 920         assertThat(info.status).hasToString("{}");
 921     }
 922
 923     @Test
 924     void testGetServiceStatus() throws Exception {
 925         String url = "/status";
 926         String rsp = restClient().get(url).block();
 927         assertThat(rsp).contains("hunky dory");
 928
 929         rsp = restClient(baseUrl(), false).get(url).block(); // V1 status is used by a readinessProbe
 930         assertThat(rsp).isEqualTo("hunky dory");
 931     }
 932
 933     @Test
 934     void testServiceNotification() throws Exception {
 935
 936         final String AUTH_TOKEN = "testToken";
 937         Path authFile = Files.createTempFile("pmsTestAuthToken", ".txt");
 938         Files.write(authFile, AUTH_TOKEN.getBytes());
 939         this.securityContext.setAuthTokenFilePath(authFile);
 940
 941         putService("junkService");
 942         Service junkService = this.services.get("junkService");
 943         junkService.setCallbackUrl("https://junk");
 944         putService("service");
 945
 946         Ric ric = addRic("ric1");
 947         ric.setState(Ric.RicState.UNAVAILABLE);
 948         supervision.checkAllRics();
 949         waitForRicState("ric1", RicState.AVAILABLE);
 950
 951         final RappSimulatorController.TestResults receivedCallbacks = rAppSimulator.getTestResults();
 952         await().untilAsserted(() -> assertThat(receivedCallbacks.getReceivedInfo()).hasSize(1));
 953         ServiceCallbackInfo callbackInfo = receivedCallbacks.getReceivedInfo().get(0);
 954         assertThat(callbackInfo.ricId).isEqualTo("ric1");
 955         assertThat(callbackInfo.eventType).isEqualTo(ServiceCallbackInfo.EventType.AVAILABLE);
 956
 957         var headers = receivedCallbacks.receivedHeaders.get(0);
 958         assertThat(headers).containsEntry("authorization", "Bearer " + AUTH_TOKEN);
 959
 960         Files.delete(authFile);
 961     }
 962
 963     private Policy addPolicy(String id, String typeName, String service, String ric) throws ServiceException {
 964         addRic(ric);
 965         Policy policy = Policy.builder() //
 966                 .id(id) //
 967                 .json(jsonString()) //
 968                 .ownerServiceId(service) //
 969                 .ric(rics.getRic(ric)) //
 970                 .type(addPolicyType(typeName, ric)) //
 971                 .lastModified(Instant.now()) //
 972                 .isTransient(false) //
 973                 .statusNotificationUri("/policy-status?id=XXX") //
 974                 .build();
 975         policies.put(policy);
 976         return policy;
 977     }
 978
 979     private Policy addPolicy(String id, String typeName, String service) throws ServiceException {
 980         return addPolicy(id, typeName, service, "ric");
 981     }
 982
 983     private String createServiceJson(String name, long keepAliveIntervalSeconds) {
 984         String callbackUrl = baseUrl() + RappSimulatorController.SERVICE_CALLBACK_URL;
 985         return createServiceJson(name, keepAliveIntervalSeconds, callbackUrl);
 986     }
 987
 988     private String createServiceJson(String name, long keepAliveIntervalSeconds, String url) {
 989         ServiceRegistrationInfo service = new ServiceRegistrationInfo(name, keepAliveIntervalSeconds, url);
 990
 991         String json = gson.toJson(service);
 992         return json;
 993     }
 994
 995     private void putService(String name) {
 996         putService(name, 0, null);
 997     }
 998
 999     private void putService(String name, long keepAliveIntervalSeconds, @Nullable HttpStatus expectedStatus) {
1000         String url = "/services";
1001         String body = createServiceJson(name, keepAliveIntervalSeconds);
1002         ResponseEntity<String> resp = restClient().putForEntity(url, body).block();
1003         if (expectedStatus != null) {
1004             assertEquals(expectedStatus, resp.getStatusCode(), "");
1005         }
1006     }
1007
1008     private String jsonString() {
1009         return "{\"servingCellNrcgi\":\"1\"}";
1010     }
1011
1012     @Test
1013     void testConcurrency() throws Exception {
1014         logger.info("Concurrency test starting");
1015         final Instant startTime = Instant.now();
1016         List<Thread> threads = new ArrayList<>();
1017         List<ConcurrencyTestRunnable> tests = new ArrayList<>();
1018         a1ClientFactory.setResponseDelay(Duration.ofMillis(2));
1019         addRic("ric");
1020         addPolicyType("type1", "ric");
1021         addPolicyType("type2", "ric");
1022
1023         final String NON_RESPONDING_RIC = "NonRespondingRic";
1024         Ric nonRespondingRic = addRic(NON_RESPONDING_RIC);
1025         MockA1Client a1Client = a1ClientFactory.getOrCreateA1Client(NON_RESPONDING_RIC);
1026         a1Client.setErrorInject("errorInject");
1027
1028         for (int i = 0; i < 10; ++i) {
1029             AsyncRestClient restClient = restClient();
1030             ConcurrencyTestRunnable test =
1031                     new ConcurrencyTestRunnable(restClient, supervision, a1ClientFactory, rics, policyTypes);
1032             Thread thread = new Thread(test, "TestThread_" + i);
1033             thread.start();
1034             threads.add(thread);
1035             tests.add(test);
1036         }
1037         for (Thread t : threads) {
1038             t.join();
1039         }
1040         for (ConcurrencyTestRunnable test : tests) {
1041             assertThat(test.isFailed()).isFalse();
1042         }
1043         assertThat(policies.size()).isZero();
1044         logger.info("Concurrency test took " + Duration.between(startTime, Instant.now()));
1045
1046         assertThat(nonRespondingRic.getState()).isEqualTo(RicState.UNAVAILABLE);
1047         nonRespondingRic.setState(RicState.AVAILABLE);
1048     }
1049
1050     private AsyncRestClient restClient(String baseUrl, boolean useTrustValidation) {
1051         WebClientConfig config = this.applicationConfig.getWebClientConfig();
1052         config = WebClientConfig.builder() //
1053                 .keyStoreType(config.getKeyStoreType()) //
1054                 .keyStorePassword(config.getKeyStorePassword()) //
1055                 .keyStore(config.getKeyStore()) //
1056                 .keyPassword(config.getKeyPassword()) //
1057                 .isTrustStoreUsed(useTrustValidation) //
1058                 .trustStore(config.getTrustStore()) //
1059                 .trustStorePassword(config.getTrustStorePassword()) //
1060                 .httpProxyConfig(config.getHttpProxyConfig()) //
1061                 .build();
1062
1063         AsyncRestClientFactory f = new AsyncRestClientFactory(config, new SecurityContext(""));
1064         return f.createRestClientNoHttpProxy(baseUrl);
1065
1066     }
1067
1068     private String baseUrl() {
1069         return "https://localhost:" + port;
1070     }
1071
1072     private AsyncRestClient restClient(boolean useTrustValidation) {
1073         return restClient(baseUrl() + Consts.V2_API_ROOT, useTrustValidation);
1074     }
1075
1076     private AsyncRestClient restClient() {
1077         return restClient(false);
1078     }
1079
1080     private void testErrorCode(Mono<?> request, HttpStatus expStatus) {
1081         testErrorCode(request, expStatus, "", true);
1082     }
1083
1084     private void testErrorCode(Mono<?> request, HttpStatus expStatus, boolean expectApplicationProblemJsonMediaType) {
1085         testErrorCode(request, expStatus, "", expectApplicationProblemJsonMediaType);
1086     }
1087
1088     private void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains) {
1089         testErrorCode(request, expStatus, responseContains, true);
1090     }
1091
1092     private void testErrorCode(Mono<?> request, HttpStatus expStatus, String responseContains,
1093             boolean expectApplicationProblemJsonMediaType) {
1094         StepVerifier.create(request) //
1095                 .expectSubscription() //
1096                 .expectErrorMatches(
1097                         t -> checkWebClientError(t, expStatus, responseContains, expectApplicationProblemJsonMediaType)) //
1098                 .verify();
1099     }
1100
1101     private void waitForRicState(String ricId, RicState state) throws ServiceException {
1102         Ric ric = rics.getRic(ricId);
1103         await().untilAsserted(() -> state.equals(ric.getState()));
1104     }
1105
1106     private boolean checkWebClientError(Throwable throwable, HttpStatus expStatus, String responseContains,
1107             boolean expectApplicationProblemJsonMediaType) {
1108         assertTrue(throwable instanceof WebClientResponseException);
1109         WebClientResponseException responseException = (WebClientResponseException) throwable;
1110         assertThat(responseException.getStatusCode()).isEqualTo(expStatus);
1111         assertThat(responseException.getResponseBodyAsString()).contains(responseContains);
1112         if (expectApplicationProblemJsonMediaType) {
1113             assertThat(responseException.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_PROBLEM_JSON);
1114         }
1115         return true;
1116     }
1117
1118     private MockA1Client getA1Client(String ricId) throws ServiceException {
1119         return a1ClientFactory.getOrCreateA1Client(ricId);
1120     }
1121
1122     private PolicyType createPolicyType(String policyTypeName) {
1123         return PolicyType.builder() //
1124                 .id(policyTypeName) //
1125                 .schema("{\"title\":\"" + policyTypeName + "\"}") //
1126                 .build();
1127     }
1128
1129     private PolicyType addPolicyType(String policyTypeName, String ricId) {
1130         PolicyType type = createPolicyType(policyTypeName);
1131         policyTypes.put(type);
1132         addRic(ricId).addSupportedPolicyType(type);
1133         return type;
1134     }
1135
1136     private Ric addRic(String ricId) {
1137         return addRic(ricId, null);
1138     }
1139
1140     private RicConfig ricConfig(String ricId, String managedElement) {
1141         List<String> mes = new ArrayList<>();
1142         if (managedElement != null) {
1143             mes.add(managedElement);
1144         }
1145         return RicConfig.builder() //
1146                 .ricId(ricId) //
1147                 .baseUrl(ricId) //
1148                 .managedElementIds(mes) //
1149                 .build();
1150     }
1151
1152     private Ric addRic(String ricId, String managedElement) {
1153         if (rics.get(ricId) != null) {
1154             return rics.get(ricId);
1155         }
1156
1157         RicConfig conf = ricConfig(ricId, managedElement);
1158         Ric ric = new Ric(conf);
1159         ric.setState(Ric.RicState.AVAILABLE);
1160         this.rics.put(ric);
1161         return ric;
1162     }
1163
1164 }