nokiav2/driver/src/test/java/org/onap/vfc/nfvo/driver/vnfm/svnfm/nokia/vnfm/TestJobManager.java

   1 /*
   2  * Copyright 2016-2017, Nokia Corporation
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.onap.vfc.nfvo.driver.vnfm.svnfm.nokia.vnfm;
  17
  18 import com.google.common.collect.Lists;
  19 import com.google.gson.JsonElement;
  20 import com.google.gson.JsonParser;
  21 import com.nokia.cbam.lcm.v32.ApiException;
  22 import com.nokia.cbam.lcm.v32.model.*;
  23 import org.junit.Before;
  24 import org.junit.Test;
  25 import org.mockito.ArgumentCaptor;
  26 import org.mockito.InjectMocks;
  27 import org.mockito.Mock;
  28 import org.mockito.Mockito;
  29 import org.mockito.invocation.InvocationOnMock;
  30 import org.mockito.stubbing.Answer;
  31 import org.onap.vnfmdriver.model.JobDetailInfo;
  32 import org.onap.vnfmdriver.model.JobResponseInfo;
  33 import org.onap.vnfmdriver.model.JobStatus;
  34 import org.springframework.test.util.ReflectionTestUtils;
  35 import org.threeten.bp.OffsetDateTime;
  36
  37 import javax.servlet.http.HttpServletResponse;
  38 import java.util.ArrayList;
  39 import java.util.List;
  40 import java.util.NoSuchElementException;
  41 import java.util.UUID;
  42 import java.util.concurrent.ExecutorService;
  43 import java.util.concurrent.Executors;
  44 import java.util.concurrent.Future;
  45
  46 import static junit.framework.TestCase.*;
  47 import static org.mockito.Mockito.*;
  48 import static org.onap.vfc.nfvo.driver.vnfm.svnfm.nokia.util.CbamUtils.SEPARATOR;
  49 import static org.onap.vfc.nfvo.driver.vnfm.svnfm.nokia.vnfm.CbamRestApiProvider.NOKIA_LCM_API_VERSION;
  50 import static org.onap.vfc.nfvo.driver.vnfm.svnfm.nokia.vnfm.JobManager.extractOnapJobId;
  51 import static org.onap.vnfmdriver.model.JobStatus.*;
  52
  53 public class TestJobManager extends TestBase {
  54
  55     @Mock
  56     private HttpServletResponse httpResponse;
  57
  58     @InjectMocks
  59     private JobManager jobManager;
  60     private List<VnfInfo> vnfs = new ArrayList<>();
  61
  62     @Before
  63     public void initMocks() throws Exception {
  64         ReflectionTestUtils.setField(JobManager.class, "logger", logger);
  65         when(vnfApi.vnfsGet(NOKIA_LCM_API_VERSION)).thenReturn(vnfs);
  66         when(selfRegistrationManager.isReady()).thenReturn(true);
  67     }
  68
  69     /**
  70      * Only the _ can be used as separator
  71      * . / % & handled specially in URLs
  72      * - used in CBAM for separation
  73      */
  74     @Test
  75     public void testSeparator() {
  76         assertEquals("_", SEPARATOR);
  77     }
  78
  79     /**
  80      * The operation result must contain the ONAP job identifier under the jobId field
  81      */
  82     @Test
  83     public void testJobIdExtractionFromOperationResult() {
  84         assertEquals("1234", extractOnapJobId(new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"1234\"}}")));
  85         try {
  86             extractOnapJobId(new JsonParser().parse("{ }"));
  87             fail();
  88         } catch (NoSuchElementException e) {
  89             assertEquals("The operation result {} does not contain the mandatory additionalParams structure", e.getMessage());
  90         }
  91         try {
  92             extractOnapJobId(new JsonParser().parse("{ \"additionalParams\" : { } }"));
  93             fail();
  94         } catch (NoSuchElementException e) {
  95             assertEquals("The operation result {\"additionalParams\":{}} does not contain the mandatory jobId in the additionalParams structure", e.getMessage());
  96         }
  97     }
  98
  99     /**
 100      * If the VNF does not exists but the job manager still runs the VNF manipulation process the job is reported to be running
 101      */
 102     @Test
 103     public void testJobForNonExistingVnfReportedRunningIfJobIsOngoing() throws Exception {
 104         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 105         //when
 106         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 107         //verify
 108         assertResult(jobId, job, STARTED, "50", "Operation started");
 109     }
 110
 111     /**
 112      * If the VNF does not exists and the internal job is not running the job is reported to be finished
 113      */
 114     @Test
 115     public void testJobForExistingVnfReportedRunningIfJobIsFinished() throws Exception {
 116         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 117         jobManager.jobFinished(jobId);
 118         //when
 119         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 120         //verify
 121         assertResult(jobId, job, JobStatus.FINISHED, "100", "Operation finished");
 122     }
 123
 124     /**
 125      * Spawning jobs after preparing for shutdown results in error
 126      */
 127     @Test
 128     public void testNoMoreJobsAreAllowedAfterPrepareForShutdown() throws Exception {
 129         jobManager.prepareForShutdown();
 130         //when
 131         try {
 132             jobManager.spawnJob(JOB_ID, httpResponse);
 133             fail();
 134         } catch (Exception e) {
 135             verify(logger).error("The service is preparing to shut down");
 136         }
 137     }
 138
 139     /**
 140      * Verify if the jobId has valid format
 141      */
 142     @Test
 143     public void testJobIdValidation() throws Exception {
 144         try {
 145             //when
 146             jobManager.getJob(VNFM_ID, "bad");
 147             //verify
 148             fail();
 149         } catch (IllegalArgumentException e) {
 150             assertEquals("The jobId should be in the <vnfId>_<UUID> format, but was bad", e.getMessage());
 151         }
 152         try {
 153             //when
 154             jobManager.getJob(VNFM_ID, "vnfId_");
 155             //verify
 156             fail();
 157         } catch (IllegalArgumentException e) {
 158             assertEquals("The UUID in the jobId (vnfId_) can not be empty", e.getMessage());
 159         }
 160         try {
 161             //when
 162             jobManager.getJob(VNFM_ID, "_UUID");
 163             //verify
 164             fail();
 165         } catch (IllegalArgumentException e) {
 166             assertEquals("The vnfId in the jobId (_UUID) can not be empty", e.getMessage());
 167         }
 168     }
 169
 170     /**
 171      * If the VNF exists but no operation execution is present with given internalJobId, than the state of the
 172      * job is ongoing if the internal job is ongoing
 173      */
 174     @Test
 175     public void testExistingVnfWithNotYetStartedOperation() throws Exception {
 176         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 177         VnfInfo vnf = new VnfInfo();
 178         vnf.setId(VNF_ID);
 179         vnfs.add(vnf);
 180         VnfInfo detailedVnf = new VnfInfo();
 181         detailedVnf.setId(VNF_ID);
 182         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 183         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 184         //verify
 185         assertResult(jobId, job, STARTED, "50", "Operation started");
 186         assertTrue(jobManager.hasOngoingJobs());
 187     }
 188
 189     /**
 190      * If the VNF exists but no operation execution is present with given internalJobId, than the state of the
 191      * job is failed if the internal job is finished (the operation on CBAM was not able to start)
 192      */
 193     @Test
 194     public void testExistingVnfWithNotUnableToStartOperation() throws Exception {
 195         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 196         VnfInfo vnf = new VnfInfo();
 197         vnf.setId(VNF_ID);
 198         vnfs.add(vnf);
 199         VnfInfo detailedVnf = new VnfInfo();
 200         detailedVnf.setId(VNF_ID);
 201         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 202         jobManager.jobFinished(jobId);
 203         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 204         //verify
 205         assertResult(jobId, job, ERROR, "100", "Operation failed due to The requested operation was not able to start on CBAM");
 206         assertFalse(jobManager.hasOngoingJobs());
 207     }
 208
 209     /**
 210      * If the VNF exists but and the operation execution is present with given internalJobId, than the state of the
 211      * job is ongoing if the operation is ongoing
 212      */
 213     @Test
 214     public void testExistingVnfWithStartedOperation() throws Exception {
 215         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 216         VnfInfo vnf = new VnfInfo();
 217         vnf.setId(VNF_ID);
 218         vnfs.add(vnf);
 219         VnfInfo detailedVnf = new VnfInfo();
 220         detailedVnf.setId(VNF_ID);
 221         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 222         OperationExecution operation = new OperationExecution();
 223         operation.setId(UUID.randomUUID().toString());
 224         operation.setStartTime(OffsetDateTime.now());
 225         operation.setStatus(OperationStatus.STARTED);
 226         detailedVnf.setOperationExecutions(new ArrayList<>());
 227         detailedVnf.getOperationExecutions().add(operation);
 228         JsonElement operationParams = new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 229         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(operation.getId(), NOKIA_LCM_API_VERSION)).thenReturn(operationParams);
 230         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 231         //verify
 232         assertResult(jobId, job, STARTED, "50", "Operation started");
 233         assertTrue(jobManager.hasOngoingJobs());
 234     }
 235
 236     /**
 237      * If the VNF does not exists till the time the job queries the status of the operation
 238      */
 239     @Test
 240     public void testTerminatedVnf() throws Exception {
 241         //ddd
 242         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 243         VnfInfo vnf = new VnfInfo();
 244         vnf.setId(VNF_ID);
 245         vnfs.add(vnf);
 246         VnfInfo detailedVnf = new VnfInfo();
 247         detailedVnf.setId(VNF_ID);
 248         List<Integer> vnfQueryCallCounter = new ArrayList<>();
 249         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenAnswer(new Answer<VnfInfo>() {
 250             @Override
 251             public VnfInfo answer(InvocationOnMock invocation) throws Throwable {
 252                 vnfs.clear();
 253                 return detailedVnf;
 254             }
 255         });
 256
 257         jobManager.jobFinished(jobId);
 258
 259         OperationExecution operation = new OperationExecution();
 260         operation.setId(UUID.randomUUID().toString());
 261         operation.setStartTime(OffsetDateTime.now());
 262         operation.setStatus(OperationStatus.FINISHED);
 263         operation.setOperationType(OperationType.TERMINATE);
 264         detailedVnf.setOperationExecutions(new ArrayList<>());
 265         detailedVnf.getOperationExecutions().add(operation);
 266
 267         JsonElement operationParams = new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 268         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(operation.getId(), NOKIA_LCM_API_VERSION)).thenReturn(operationParams);
 269         //when
 270         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 271         //verify
 272         assertResult(jobId, job, FINISHED, "100", "Operation finished");
 273     }
 274
 275     /**
 276      * If the VNF exists but and the operation execution is present with given internalJobId, than the state of the
 277      * job is error if the operation is failed
 278      */
 279     @Test
 280     public void testExistingVnfWithFailedOperation() throws Exception {
 281         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 282         VnfInfo vnf = new VnfInfo();
 283         vnf.setId(VNF_ID);
 284         vnfs.add(vnf);
 285         VnfInfo detailedVnf = new VnfInfo();
 286         detailedVnf.setId(VNF_ID);
 287         List<Integer> vnfCounter = new ArrayList<>();
 288         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 289         OperationExecution operation = new OperationExecution();
 290         operation.setId(UUID.randomUUID().toString());
 291         operation.setStartTime(OffsetDateTime.now());
 292         operation.setStatus(OperationStatus.FAILED);
 293         ProblemDetails errorDetails = new ProblemDetails();
 294         errorDetails.setTitle("Title");
 295         errorDetails.setDetail("detail");
 296         operation.setError(errorDetails);
 297         detailedVnf.setOperationExecutions(new ArrayList<>());
 298         detailedVnf.getOperationExecutions().add(operation);
 299         JsonElement operationParams = new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 300         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(operation.getId(), NOKIA_LCM_API_VERSION)).thenReturn(operationParams);
 301         //when
 302         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 303         //verify
 304         assertResult(jobId, job, ERROR, "100", "Operation failed due to Title: detail");
 305         assertTrue(jobManager.hasOngoingJobs());
 306     }
 307
 308     private void assertResult(String jobId, JobDetailInfo job, JobStatus status, String progress, String descriptor) {
 309         assertEquals(jobId, job.getJobId());
 310         assertEquals(status, job.getResponseDescriptor().getStatus());
 311         assertEquals(progress, job.getResponseDescriptor().getProgress());
 312         assertNull(job.getResponseDescriptor().getErrorCode());
 313         boolean finalState = JobStatus.ERROR.equals(status) || JobStatus.FINISHED.equals(status);
 314         if (finalState) {
 315             assertEquals(2, job.getResponseDescriptor().getResponseHistoryList().size());
 316             JobResponseInfo startEvent = job.getResponseDescriptor().getResponseHistoryList().get(0);
 317             JobResponseInfo endEvent = job.getResponseDescriptor().getResponseHistoryList().get(1);
 318             assertNull(startEvent.getErrorCode());
 319             assertEquals("50", startEvent.getProgress());
 320             assertEquals(JobStatus.STARTED.name(), startEvent.getStatus());
 321             assertEquals("1", startEvent.getResponseId());
 322             assertEquals("Operation started", startEvent.getStatusDescription());
 323
 324             assertNull(endEvent.getErrorCode());
 325             assertEquals("100", endEvent.getProgress());
 326             assertEquals(job.getResponseDescriptor().getStatus().name(), endEvent.getStatus());
 327             assertEquals("2", endEvent.getResponseId());
 328             assertEquals(descriptor, endEvent.getStatusDescription());
 329         } else {
 330             assertEquals(1, job.getResponseDescriptor().getResponseHistoryList().size());
 331             assertNull(job.getResponseDescriptor().getResponseHistoryList().get(0).getErrorCode());
 332             assertEquals(progress, job.getResponseDescriptor().getResponseHistoryList().get(0).getProgress());
 333             assertEquals(job.getResponseDescriptor().getStatus().name(), job.getResponseDescriptor().getResponseHistoryList().get(0).getStatus());
 334             assertEquals("1", job.getResponseDescriptor().getResponseHistoryList().get(0).getResponseId());
 335             assertEquals(descriptor, job.getResponseDescriptor().getResponseHistoryList().get(0).getStatusDescription());
 336         }
 337         assertEquals(Integer.toString(job.getResponseDescriptor().getResponseHistoryList().size()), job.getResponseDescriptor().getResponseId());
 338         assertEquals(descriptor, job.getResponseDescriptor().getStatusDescription());
 339     }
 340
 341     /**
 342      * If the VNF exists but and the operation execution is present with given internalJobId, than the state of the
 343      * job is finished if the operation is finished, but is not a termination
 344      */
 345     @Test
 346     public void testExistingVnfWithFinishedOperation() throws Exception {
 347         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 348         VnfInfo vnf = new VnfInfo();
 349         vnf.setId(VNF_ID);
 350         vnfs.add(vnf);
 351         VnfInfo detailedVnf = new VnfInfo();
 352         detailedVnf.setId(VNF_ID);
 353         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 354         OperationExecution operation = new OperationExecution();
 355         operation.setId(UUID.randomUUID().toString());
 356         operation.setStartTime(OffsetDateTime.now());
 357         operation.setStatus(OperationStatus.FINISHED);
 358         operation.setOperationType(OperationType.SCALE);
 359         detailedVnf.setOperationExecutions(new ArrayList<>());
 360         detailedVnf.getOperationExecutions().add(operation);
 361         JsonElement operationParams = new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 362         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(operation.getId(), NOKIA_LCM_API_VERSION)).thenReturn(operationParams);
 363         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 364         //verify
 365         assertResult(jobId, job, JobStatus.FINISHED, "100", "Operation finished");
 366         assertTrue(jobManager.hasOngoingJobs());
 367     }
 368
 369     /**
 370      * If the VNF exists but and the operation execution is present with given internalJobId, than the state of the
 371      * job is ongoing if the termination operation is finished. In ONAP terminology the termination includes
 372      * delete, so the ONAP operation is ongoing since the VNF is not yet deleted
 373      */
 374     @Test
 375     public void testExistingVnfWithFinishedTerminationOperation() throws Exception {
 376         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 377         VnfInfo vnf = new VnfInfo();
 378         vnf.setId(VNF_ID);
 379         vnfs.add(vnf);
 380         VnfInfo detailedVnf = new VnfInfo();
 381         detailedVnf.setId(VNF_ID);
 382         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 383         OperationExecution operation = new OperationExecution();
 384         operation.setId(UUID.randomUUID().toString());
 385         operation.setStartTime(OffsetDateTime.now());
 386         operation.setStatus(OperationStatus.FINISHED);
 387         operation.setOperationType(OperationType.TERMINATE);
 388         detailedVnf.setOperationExecutions(new ArrayList<>());
 389         detailedVnf.getOperationExecutions().add(operation);
 390         JsonElement operationParams = new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 391         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(operation.getId(), NOKIA_LCM_API_VERSION)).thenReturn(operationParams);
 392         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 393         //verify
 394         assertResult(jobId, job, STARTED, "50", "Operation started");
 395         //when
 396         jobManager.jobFinished(jobId);
 397         job = jobManager.getJob(VNFM_ID, jobId);
 398         //verify
 399         assertResult(jobId, job, ERROR, "100", "Operation failed due to unable to delete VNF");
 400         assertFalse(jobManager.hasOngoingJobs());
 401
 402     }
 403
 404     /**
 405      * Failuire to retrieve operation parameters (CBAM REST API fail) is logged and propagated
 406      */
 407     @Test
 408     public void failuresDuringOperationExecutionRetrievalIsLoggedAndPropagated() throws Exception {
 409         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 410         VnfInfo vnf = new VnfInfo();
 411         vnf.setId(VNF_ID);
 412         vnfs.add(vnf);
 413         VnfInfo detailedVnf = new VnfInfo();
 414         detailedVnf.setId(VNF_ID);
 415         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 416         OperationExecution operation = new OperationExecution();
 417         operation.setId(UUID.randomUUID().toString());
 418         detailedVnf.setOperationExecutions(new ArrayList<>());
 419         detailedVnf.getOperationExecutions().add(operation);
 420         ApiException expectedException = new ApiException();
 421         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(operation.getId(), NOKIA_LCM_API_VERSION)).thenThrow(expectedException);
 422         //verify
 423         try {
 424             JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 425             fail();
 426         } catch (RuntimeException e) {
 427             assertEquals(expectedException, e.getCause());
 428             verify(logger).error("Unable to retrieve operation parameters", expectedException);
 429         }
 430         assertTrue(jobManager.hasOngoingJobs());
 431     }
 432
 433     /**
 434      * Failure to retrieve VNF (CBAM REST API fail) is logged and propagated
 435      */
 436     @Test
 437     public void failuresDuringVnfRetrievalIsLoggedAndPropagated() throws Exception {
 438         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 439         VnfInfo vnf = new VnfInfo();
 440         vnf.setId(VNF_ID);
 441         vnfs.add(vnf);
 442         ApiException expectedException = new ApiException();
 443         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenThrow(expectedException);
 444         //verify
 445         try {
 446             JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 447             fail();
 448         } catch (RuntimeException e) {
 449             assertEquals(expectedException, e.getCause());
 450             verify(logger).error("Unable to retrieve VNF", expectedException);
 451         }
 452         assertTrue(jobManager.hasOngoingJobs());
 453     }
 454
 455     /**
 456      * When searching for the ONAP job by iterating the operation executions. The newest jobs
 457      * are inspected first (performance optimalization)
 458      */
 459     @Test
 460     public void testNewestOperationAreInspectedFirst() throws Exception {
 461         String jobId = jobManager.spawnJob(VNF_ID, httpResponse);
 462         VnfInfo vnf = new VnfInfo();
 463         vnf.setId(VNF_ID);
 464         vnfs.add(vnf);
 465         VnfInfo detailedVnf = new VnfInfo();
 466         detailedVnf.setId(VNF_ID);
 467         when(vnfApi.vnfsVnfInstanceIdGet(VNF_ID, NOKIA_LCM_API_VERSION)).thenReturn(detailedVnf);
 468         OperationExecution olderOperation = new OperationExecution();
 469         olderOperation.setId(UUID.randomUUID().toString());
 470         olderOperation.setStartTime(OffsetDateTime.now());
 471         olderOperation.setStatus(OperationStatus.FINISHED);
 472         olderOperation.setOperationType(OperationType.TERMINATE);
 473         OperationExecution newerOperation = new OperationExecution();
 474         newerOperation.setId(UUID.randomUUID().toString());
 475         newerOperation.setStartTime(OffsetDateTime.now().plusDays(1));
 476         newerOperation.setStatus(OperationStatus.FINISHED);
 477         newerOperation.setOperationType(OperationType.TERMINATE);
 478         detailedVnf.setOperationExecutions(new ArrayList<>());
 479         detailedVnf.getOperationExecutions().add(olderOperation);
 480         detailedVnf.getOperationExecutions().add(newerOperation);
 481         JsonElement operationParams = new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 482         List<String> queriedOperaionsInOrder = new ArrayList<>();
 483         when(operationExecutionApi.operationExecutionsOperationExecutionIdOperationParamsGet(Mockito.anyString(), Mockito.eq(NOKIA_LCM_API_VERSION)))
 484                 .then(new Answer<Object>() {
 485                     @Override
 486                     public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
 487                         queriedOperaionsInOrder.add(invocationOnMock.getArguments()[0].toString());
 488                         if (invocationOnMock.getArguments()[0].equals(olderOperation.getId())) {
 489                             return new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + jobId + "\"}}");
 490                         } else {
 491                             return new JsonParser().parse("{ \"additionalParams\" : { \"jobId\" : \"" + "nonMatching" + "\"}}");
 492                         }
 493                     }
 494                 });
 495         JobDetailInfo job = jobManager.getJob(VNFM_ID, jobId);
 496         //verify
 497         assertEquals(Lists.newArrayList(newerOperation.getId(), olderOperation.getId()), queriedOperaionsInOrder);
 498         assertTrue(jobManager.hasOngoingJobs());
 499     }
 500
 501     /**
 502      * if the registration process has not finished it is prevented to spawn jobs
 503      */
 504     @Test
 505     public void noJobCanBeStartedIfRegistrationNotFinished() throws Exception {
 506         //given
 507         when(selfRegistrationManager.isReady()).thenReturn(false);
 508         //when
 509         try {
 510             jobManager.spawnJob(VNF_ID, httpResponse);
 511             fail();
 512         } catch (RuntimeException e) {
 513             assertEquals("The service is not yet ready", e.getMessage());
 514         }
 515     }
 516
 517     /**
 518      * Ongoing job are out waited during the the preparation for shutdown
 519      */
 520     @Test
 521     //need to wait for an asynchronous execution to finish
 522     //this is the most optimal way to do it
 523     @SuppressWarnings("squid:S2925")
 524     public void onGoingJobsAreOutwaitedDuringShutdown() throws Exception {
 525         String firstJobId = jobManager.spawnJob(VNF_ID, httpResponse);
 526         ExecutorService executorService = Executors.newCachedThreadPool();
 527         ArgumentCaptor<Integer> sleeps = ArgumentCaptor.forClass(Integer.class);
 528         doNothing().when(systemFunctions).sleep(sleeps.capture());
 529         //when prepare job manager for shutdown
 530         Future<?> shutDown = executorService.submit(() -> jobManager.prepareForShutdown());
 531         while (sleeps.getAllValues().size() == 0) {
 532             try {
 533                 Thread.sleep(1);
 534             } catch (InterruptedException e) {
 535             }
 536         }
 537         assertFalse(shutDown.isDone());
 538         jobManager.jobFinished(firstJobId);
 539         //verify
 540         shutDown.get();
 541         verify(systemFunctions, times(sleeps.getAllValues().size())).sleep(500L);
 542     }
 543 }