From 0fa2fabeec18763bab060d85f5123bceff8ee34c Mon Sep 17 00:00:00 2001 From: ToineSiebelink Date: Thu, 19 Jan 2023 16:45:58 +0000 Subject: [PATCH] Introduce Instrumentation - Add instrumentation related dependency - Added Timed Instrumentation - CPS-Service Crud methods - CPS Yang parsing - NCMP Registration methods - NCMP Events handling - Remove manual Gauge for YanResources Cache as (better!) instrumentation is already built into the 3PP - Sorted dependecies alphabetically (as we used to enforce, to prevent duplicates) - Added ## P E R F O R M A N C E T E S T R E S U L T S ### mini report - (unrelated) test improvement (because of bug that turned out to be invalid) Reviewers: Sourabh,Priyank, Luke Issue-ID: CPS-1457 Signed-off-by: ToineSiebelink Change-Id: I34b20bece2f59488b022b8effa9470704c57be4d --- .../src/main/java/org/onap/cps/Application.java | 1 - .../java/org/onap/cps/config/MicroMeterConfig.java | 36 ++++++++++ .../onap/cps/config/MicroMeterConfigSpec.groovy | 35 +++++++++ .../rest/controller/ControllerSecuritySpec.groovy | 29 ++++---- .../onap/cps/rest/controller/TestController.java | 10 +++ .../NetworkCmProxyInventoryController.java | 5 +- .../lcm/LcmEventsCmHandleStateHandlerImpl.java | 7 +- .../ncmp/api/impl/event/lcm/LcmEventsService.java | 5 +- .../api/inventory/InventoryPersistenceImpl.java | 9 ++- .../cps/rest/controller/AdminRestController.java | 5 +- .../exceptions/CpsRestExceptionHandlerSpec.groovy | 14 +++- .../cps/spi/impl/CpsPersistencePerfSpecBase.groovy | 25 +++++++ .../CpsDataPersistenceServiceDeletePerfTest.groovy | 24 ++++--- .../CpsDataPersistenceServicePerfTest.groovy | 18 +++-- cps-service/pom.xml | 82 ++++++++++++---------- .../org/onap/cps/api/impl/CpsDataServiceImpl.java | 48 +++++++++++-- .../onap/cps/api/impl/CpsModuleServiceImpl.java | 12 ++-- .../cps/api/impl/YangTextSchemaSourceSetCache.java | 10 ++- .../java/org/onap/cps/utils/TimedYangParser.java | 61 ++++++++++++++++ .../main/java/org/onap/cps/utils/YangUtils.java | 6 +- .../yang/TimedYangTextSchemaSourceSetBuilder.java | 37 ++++++++++ .../cps/yang/YangTextSchemaSourceSetBuilder.java | 22 +++++- .../cps/api/impl/CpsDataServiceImplSpec.groovy | 6 +- .../cps/api/impl/CpsModuleServiceImplSpec.groovy | 6 +- .../onap/cps/api/impl/E2ENetworkSliceSpec.groovy | 12 +++- docker-compose/docker-compose.yml | 2 +- .../org/onap/cps/integration/TestConfig.groovy | 19 +++-- 27 files changed, 431 insertions(+), 115 deletions(-) create mode 100644 cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java create mode 100644 cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy create mode 100644 cps-service/src/main/java/org/onap/cps/utils/TimedYangParser.java create mode 100644 cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java diff --git a/cps-application/src/main/java/org/onap/cps/Application.java b/cps-application/src/main/java/org/onap/cps/Application.java index 79d5950f6..053139fcc 100644 --- a/cps-application/src/main/java/org/onap/cps/Application.java +++ b/cps-application/src/main/java/org/onap/cps/Application.java @@ -27,7 +27,6 @@ import org.springframework.retry.annotation.EnableRetry; @EnableRetry @SpringBootApplication public class Application { - public static void main(final String[] args) { SpringApplication.run(Application.class, args); } diff --git a/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java new file mode 100644 index 000000000..22194f3ad --- /dev/null +++ b/cps-application/src/main/java/org/onap/cps/config/MicroMeterConfig.java @@ -0,0 +1,36 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.config; + +import io.micrometer.core.aop.TimedAspect; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MicroMeterConfig { + + @Bean + public TimedAspect timedAspect(final MeterRegistry registry) { + return new TimedAspect(registry); + } + +} diff --git a/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy new file mode 100644 index 000000000..61bc2cf02 --- /dev/null +++ b/cps-application/src/test/groovy/org/onap/cps/config/MicroMeterConfigSpec.groovy @@ -0,0 +1,35 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.config + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import spock.lang.Specification + +class MicroMeterConfigSpec extends Specification { + + def objectUnderTest = new MicroMeterConfig() + + def 'Creating a tined aspect.'() { + expect: ' a timed aspect can be created' + assert objectUnderTest.timedAspect(new SimpleMeterRegistry()) != null + } + +} diff --git a/cps-application/src/test/groovy/org/onap/cps/rest/controller/ControllerSecuritySpec.groovy b/cps-application/src/test/groovy/org/onap/cps/rest/controller/ControllerSecuritySpec.groovy index 01d373582..5c255f1da 100755 --- a/cps-application/src/test/groovy/org/onap/cps/rest/controller/ControllerSecuritySpec.groovy +++ b/cps-application/src/test/groovy/org/onap/cps/rest/controller/ControllerSecuritySpec.groovy @@ -1,20 +1,21 @@ /* - * ============LICENSE_START======================================================= - * Copyright (C) 2020 Pantheon.tech - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * ============LICENSE_START======================================================= + * Copyright (C) 2020 Pantheon.tech + * Modifications Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= */ package org.onap.cps.rest.controller @@ -27,7 +28,7 @@ import org.springframework.http.HttpStatus import org.springframework.test.web.servlet.MockMvc import spock.lang.Specification -@WebMvcTest(controllers = TestController.class) +@WebMvcTest(TestController) class ControllerSecuritySpec extends Specification { @Autowired diff --git a/cps-application/src/test/java/org/onap/cps/rest/controller/TestController.java b/cps-application/src/test/java/org/onap/cps/rest/controller/TestController.java index 34325c71a..75aab83bd 100644 --- a/cps-application/src/test/java/org/onap/cps/rest/controller/TestController.java +++ b/cps-application/src/test/java/org/onap/cps/rest/controller/TestController.java @@ -1,6 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2021 Pantheon.tech + * Modifications Copyright (C) 2023 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +20,9 @@ package org.onap.cps.rest.controller; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.springframework.context.annotation.Bean; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -27,8 +31,14 @@ import org.springframework.web.bind.annotation.RestController; @RestController public class TestController { + @Bean + public MeterRegistry meterRegistry() { + return new SimpleMeterRegistry(); + } + @GetMapping("/test") ResponseEntity test() { return new ResponseEntity<>(HttpStatus.OK); } + } diff --git a/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyInventoryController.java b/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyInventoryController.java index 98da15093..0c27d3efb 100755 --- a/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyInventoryController.java +++ b/cps-ncmp-rest/src/main/java/org/onap/cps/ncmp/rest/controller/NetworkCmProxyInventoryController.java @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2021-2022 Bell Canada - * Modifications Copyright (C) 2022 Nordix Foundation + * Modifications Copyright (C) 2022-2023 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ package org.onap.cps.ncmp.rest.controller; +import io.micrometer.core.annotation.Timed; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -78,6 +79,8 @@ public class NetworkCmProxyInventoryController implements NetworkCmProxyInventor * @param restDmiPluginRegistration the registration data */ @Override + @Timed(value = "cps.ncmp.inventory.controller.update", + description = "Time taken to handle registration request") public ResponseEntity updateDmiPluginRegistration( final @Valid RestDmiPluginRegistration restDmiPluginRegistration) { final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsCmHandleStateHandlerImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsCmHandleStateHandlerImpl.java index eba0389f2..6485fdf76 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsCmHandleStateHandlerImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsCmHandleStateHandlerImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation + * Copyright (C) 2022-2023 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import static org.onap.cps.ncmp.api.inventory.CmHandleState.DELETED; import static org.onap.cps.ncmp.api.inventory.CmHandleState.LOCKED; import static org.onap.cps.ncmp.api.inventory.CmHandleState.READY; +import io.micrometer.core.annotation.Timed; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; @@ -74,6 +75,8 @@ public class LcmEventsCmHandleStateHandlerImpl implements LcmEventsCmHandleState } @Override + @Timed(value = "cps.ncmp.cmhandle.state.update.batch", + description = "Time taken to update a batch of cm handle states") public void updateCmHandleStateBatch(final Map cmHandleStatePerCmHandle) { final Collection cmHandleTransitionPairs = prepareCmHandleTransitionBatch(cmHandleStatePerCmHandle); @@ -164,7 +167,6 @@ public class LcmEventsCmHandleStateHandlerImpl implements LcmEventsCmHandleState } - private void updateToSpecifiedCmHandleState(final YangModelCmHandle yangModelCmHandle, final CmHandleState targetCmHandleState) { @@ -219,7 +221,6 @@ public class LcmEventsCmHandleStateHandlerImpl implements LcmEventsCmHandleState @Setter @NoArgsConstructor static class CmHandleTransitionPair { - private YangModelCmHandle currentYangModelCmHandle; private YangModelCmHandle targetYangModelCmHandle; } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsService.java index d6857d36a..a94d664de 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsService.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/event/lcm/LcmEventsService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation + * Copyright (C) 2022-2023 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.onap.cps.ncmp.api.impl.event.lcm; +import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.ncmp.cmhandle.event.lcm.LcmEvent; @@ -50,6 +51,8 @@ public class LcmEventsService { * @param cmHandleId Cm Handle Id * @param lcmEvent Lcm Event */ + @Timed(value = "cps.ncmp.lcm.events.publish", + description = "Time taken to publish a LCM event") public void publishLcmEvent(final String cmHandleId, final LcmEvent lcmEvent) { if (notificationsEnabled) { try { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java index 5b0b5eafd..dd165853d 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation + * Copyright (C) 2022-2023 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,7 @@ import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NO_TIMES import static org.onap.cps.spi.CascadeDeleteAllowed.CASCADE_DELETE_ALLOWED; import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS; +import io.micrometer.core.annotation.Timed; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; @@ -140,6 +141,8 @@ public class InventoryPersistenceImpl implements InventoryPersistence { } @Override + @Timed(value = "cps.ncmp.inventory.persistence.schemaset.delete", + description = "Time taken to delete a schemaset") public void deleteSchemaSetWithCascade(final String schemaSetName) { try { cpsValidator.validateNameCharacters(schemaSetName); @@ -156,6 +159,8 @@ public class InventoryPersistenceImpl implements InventoryPersistence { } @Override + @Timed(value = "cps.ncmp.inventory.persistence.datanode.get", + description = "Time taken to get a data node (from ncmp dmi registry)") public DataNode getDataNode(final String xpath, final FetchDescendantsOption fetchDescendantsOption) { return cpsDataService.getDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, xpath, fetchDescendantsOption); @@ -182,4 +187,4 @@ public class InventoryPersistenceImpl implements InventoryPersistence { cpsDataService.deleteDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpath, NO_TIMESTAMP); } -} \ No newline at end of file +} diff --git a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java index b8ba08915..993c5a3b5 100755 --- a/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/controller/AdminRestController.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2022 Nordix Foundation + * Copyright (C) 2020-2023 Nordix Foundation * Modifications Copyright (C) 2020-2021 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -26,6 +26,7 @@ package org.onap.cps.rest.controller; import static org.onap.cps.rest.utils.MultipartFileUtil.extractYangResourcesMap; import static org.onap.cps.spi.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED; +import io.micrometer.core.annotation.Timed; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -116,6 +117,8 @@ public class AdminRestController implements CpsAdminApi { * @return a {@Link ResponseEntity} of created schema set without any response body & {@link HttpStatus} CREATED */ @Override + @Timed(value = "cps.rest.admin.controller.schemaset.create", + description = "Time taken to create schemaset from controller") public ResponseEntity createSchemaSetV2(@NotNull @Valid final String schemaSetName, final String dataspaceName, @Valid final MultipartFile multipartFile) { cpsModuleService.createSchemaSet(dataspaceName, schemaSetName, extractYangResourcesMap(multipartFile)); diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/exceptions/CpsRestExceptionHandlerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/exceptions/CpsRestExceptionHandlerSpec.groovy index 0821b6beb..535b83df7 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/exceptions/CpsRestExceptionHandlerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/exceptions/CpsRestExceptionHandlerSpec.groovy @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech - * Modifications Copyright (C) 2021-2022 Nordix Foundation + * Modifications Copyright (C) 2021-2023 Nordix Foundation * Modifications Copyright (C) 2021 Bell Canada. * Modifications Copyright (C) 2022 TechMahindra Ltd. * Modifications Copyright (C) 2022 Deutsche Telekom AG @@ -136,8 +136,18 @@ class CpsRestExceptionHandlerSpec extends Specification { "Anchor with name ${existingObjectName} already exists for ${dataspaceName}.") } + def 'Request with a schema set in use exception returns HTTP Status Conflict.'() { + when: 'Schema set in use exception is thrown by the service' + setupTestException(new SchemaSetInUseException(dataspaceName, existingObjectName)) + def response = performTestRequest() + then: 'a HTTP conflict response is returned with correct message an details' + assertTestResponse(response, CONFLICT, + "Schema Set is being used.", + "Schema Set with name ${existingObjectName} in dataspace ${dataspaceName} is having Anchor records associated.") + } + def 'Get request with a #exceptionThrown.class.simpleName returns HTTP Status Bad Request'() { - when: 'CPS validation exception is thrown by the service' + when: '#exceptionThrown.class.simpleName is thrown by the service' setupTestException(exceptionThrown) def response = performTestRequest() then: 'an HTTP Bad Request response is returned with correct message and details' diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistencePerfSpecBase.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistencePerfSpecBase.groovy index 3bbae2d08..b67a5cc68 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistencePerfSpecBase.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsPersistencePerfSpecBase.groovy @@ -22,6 +22,7 @@ package org.onap.cps.spi.impl import org.onap.cps.spi.model.DataNode import org.onap.cps.spi.model.DataNodeBuilder +import org.springframework.util.StopWatch class CpsPersistencePerfSpecBase extends CpsPersistenceSpecBase { @@ -32,6 +33,18 @@ class CpsPersistencePerfSpecBase extends CpsPersistenceSpecBase { static def xpathsToAllGrandChildren = [] + static def PERFORMANCE_RECORD = [] + + def stopWatch = new StopWatch() + + def cleanupSpec() { + println('#############################################################################') + println('## P E R F O R M A N C E T E S T R E S U L T S ##') + println('#############################################################################') + PERFORMANCE_RECORD.sort().each { println(it) } + PERFORMANCE_RECORD.clear() + } + def createLineage(cpsDataPersistenceService, numberOfChildren, numberOfGrandChildren, createLists) { xpathsToAllGrandChildren = [] (1..numberOfChildren).each { @@ -71,4 +84,16 @@ class CpsPersistencePerfSpecBase extends CpsPersistenceSpecBase { } return nodeCount } + + def recordAndAssertPerformance(String shortTitle, thresholdInMs, recordedTimeInMs) { + def pass = recordedTimeInMs <= thresholdInMs + if (shortTitle.length()>40) { + shortTitle = shortTitle.substring(0,40) + } + def record = String.format('%2d.%-40s limit%,7d took %,7d ms ', PERFORMANCE_RECORD.size()+1, shortTitle, thresholdInMs, recordedTimeInMs) + record += pass?'PASS':'FAIL' + PERFORMANCE_RECORD.add(record) + assert recordedTimeInMs <= thresholdInMs + return true + } } diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy index 4dd4823c9..91da53d2e 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy @@ -25,6 +25,7 @@ import org.onap.cps.spi.impl.CpsPersistencePerfSpecBase import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.context.jdbc.Sql import org.springframework.util.StopWatch +import spock.lang.Shared import java.util.concurrent.TimeUnit @@ -43,13 +44,13 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() { - given: 'a node with a large number of descendants is created' + when: 'a node with a large number of descendants is created' stopWatch.start() createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() - and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' - assert setupDurationInMillis < ALLOWED_SETUP_TIME_MS + then: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' + recordAndAssertPerformance('Setup',ALLOWED_SETUP_TIME_MS, setupDurationInMillis) } def 'Delete 5 children with grandchildren'() { @@ -62,7 +63,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 300 milliseconds' - assert deleteDurationInMillis < 300 + recordAndAssertPerformance('Delete 5 children', 300, deleteDurationInMillis) } def 'Delete 50 grandchildren (that have no descendants)'() { @@ -75,7 +76,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 350 milliseconds' - assert deleteDurationInMillis < 350 + recordAndAssertPerformance('Delete 50 grandchildren', 350, deleteDurationInMillis) } def 'Delete 1 large data node with many descendants'() { @@ -85,7 +86,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 250 milliseconds' - assert deleteDurationInMillis < 250 + recordAndAssertPerformance('Delete one large node', 250, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) @@ -96,7 +97,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' - assert setupDurationInMillis < ALLOWED_SETUP_TIME_MS + recordAndAssertPerformance('Create node with many list elements', ALLOWED_SETUP_TIME_MS, setupDurationInMillis) } def 'Delete 5 whole lists with many elements'() { @@ -109,7 +110,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 1000 milliseconds' - assert deleteDurationInMillis < 1000 + recordAndAssertPerformance('Delete 5 whole lists', 1500, deleteDurationInMillis) } def 'Delete 10 list elements with keys'() { @@ -123,7 +124,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 1200 milliseconds' - assert deleteDurationInMillis < 1200 + recordAndAssertPerformance('Delete 10 lists elements', 1500, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) @@ -136,7 +137,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 250 milliseconds' - assert deleteDurationInMillis < 250 + recordAndAssertPerformance('Delete root node', 250, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) @@ -149,6 +150,7 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() then: 'delete duration is under 250 milliseconds' - assert deleteDurationInMillis < 250 + recordAndAssertPerformance('Delete data nodes for anchor', 250, deleteDurationInMillis) } + } diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy index 040749027..0c4f5ec41 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServicePerfTest.groovy @@ -50,9 +50,6 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { static def NUMBER_OF_GRAND_CHILDREN = 50 static def TOTAL_NUMBER_OF_NODES = 1 + NUMBER_OF_CHILDREN + (NUMBER_OF_CHILDREN * NUMBER_OF_GRAND_CHILDREN) // Parent + Children + Grand-children - def stopWatch = new StopWatch() - def readStopWatch = new StopWatch() - @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() { given: 'a node with a large number of descendants is created' @@ -61,7 +58,7 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() and: 'setup duration is under 10 seconds' - assert setupDurationInMillis < 10000 + recordAndAssertPerformance('Setup', 10000, setupDurationInMillis) } def 'Get data node with many descendants by xpath #scenario'() { @@ -71,7 +68,7 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { stopWatch.stop() def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'read duration is under 500 milliseconds' - assert readDurationInMillis < 500 + recordAndAssertPerformance("Get ${scenario}", 500, readDurationInMillis) and: 'data node is returned with all the descendants populated' assert countDataNodes(result) == TOTAL_NUMBER_OF_NODES where: 'the following xPaths are used' @@ -87,7 +84,7 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { stopWatch.stop() def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'read duration is under 500 milliseconds' - assert readDurationInMillis < 500 + recordAndAssertPerformance('Query with many descendants', 500, readDurationInMillis) and: 'data node is returned with all the descendants populated' assert countDataNodes(result) == TOTAL_NUMBER_OF_NODES } @@ -95,14 +92,14 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { def 'Performance of finding multiple xpaths'() { when: 'we query for all grandchildren (except 1 for fun) with the new native method' xpathsToAllGrandChildren.remove(0) - readStopWatch.start() + stopWatch.start() def result = objectUnderTest.getDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToAllGrandChildren, INCLUDE_ALL_DESCENDANTS) - readStopWatch.stop() - def readDurationInMillis = readStopWatch.getTotalTimeMillis() + stopWatch.stop() + def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'the returned number of entities equal to the number of children * number of grandchildren' assert result.size() == xpathsToAllGrandChildren.size() and: 'it took less then 4000ms' - assert readDurationInMillis < 4000 + recordAndAssertPerformance('Find multiple xpaths', 4000, readDurationInMillis) } def 'Query many descendants by cps-path with #scenario'() { @@ -113,6 +110,7 @@ class CpsDataPersistenceServicePerfTest extends CpsPersistencePerfSpecBase { def readDurationInMillis = stopWatch.getTotalTimeMillis() then: 'read duration is under #allowedDuration milliseconds' assert readDurationInMillis < allowedDuration + recordAndAssertPerformance("Query many descendants by cpspath (${scenario})", allowedDuration, readDurationInMillis) and: 'data node is returned with all the descendants populated' assert result.size() == NUMBER_OF_CHILDREN where: 'the following options are used' diff --git a/cps-service/pom.xml b/cps-service/pom.xml index 888ef463f..d9babf530 100644 --- a/cps-service/pom.xml +++ b/cps-service/pom.xml @@ -1,7 +1,7 @@ + com.google.code.gson + gson + + + + com.hazelcast + hazelcast-spring + + + io.micrometer + micrometer-core + + + net.logstash.logback + logstash-logback-encoder + + + org.codehaus.janino + janino + org.onap.cps cps-events + + org.onap.cps + cps-path-parser + org.opendaylight.yangtools yang-model-api @@ -57,72 +87,46 @@ yang-parser-impl + org.opendaylight.yangtools - yang-model-util + yang-data-codec-gson - org.opendaylight.yangtools - yang-data-codec-gson + yang-data-codec-xml org.opendaylight.yangtools - yang-data-codec-xml + yang-model-util org.projectlombok lombok - - org.springframework.boot - spring-boot-starter-cache - - - com.github.ben-manes.caffeine - caffeine - - - org.springframework.kafka - spring-kafka - - - org.springframework - spring-messaging - org.slf4j slf4j-api - org.springframework.boot - spring-boot-starter-validation + org.springframework + spring-messaging - - com.google.code.gson - gson + org.springframework.boot + spring-boot-starter-cache org.springframework.boot spring-boot-starter-aop - net.logstash.logback - logstash-logback-encoder - - - org.codehaus.janino - janino - - - org.onap.cps - cps-path-parser + org.springframework.boot + spring-boot-starter-validation - - com.hazelcast - hazelcast-spring + org.springframework.kafka + spring-kafka diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java index 38fa92a09..53fab2916 100755 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2022 Nordix Foundation + * Copyright (C) 2021-2023 Nordix Foundation * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -28,6 +28,7 @@ import static org.onap.cps.notification.Operation.CREATE; import static org.onap.cps.notification.Operation.DELETE; import static org.onap.cps.notification.Operation.UPDATE; +import io.micrometer.core.annotation.Timed; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; @@ -48,7 +49,7 @@ import org.onap.cps.spi.model.DataNode; import org.onap.cps.spi.model.DataNodeBuilder; import org.onap.cps.spi.utils.CpsValidator; import org.onap.cps.utils.ContentType; -import org.onap.cps.utils.YangUtils; +import org.onap.cps.utils.TimedYangParser; import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; import org.opendaylight.yangtools.yang.model.api.SchemaContext; import org.springframework.stereotype.Service; @@ -66,6 +67,7 @@ public class CpsDataServiceImpl implements CpsDataService { private final YangTextSchemaSourceSetCache yangTextSchemaSourceSetCache; private final NotificationService notificationService; private final CpsValidator cpsValidator; + private final TimedYangParser timedYangParser; @Override public void saveData(final String dataspaceName, final String anchorName, final String nodeData, @@ -74,6 +76,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.root.save", + description = "Time taken to save a root data node") public void saveData(final String dataspaceName, final String anchorName, final String nodeData, final OffsetDateTime observedTimestamp, final ContentType contentType) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -90,6 +94,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.child.save", + description = "Time taken to save a child data node") public void saveData(final String dataspaceName, final String anchorName, final String parentNodeXpath, final String nodeData, final OffsetDateTime observedTimestamp, final ContentType contentType) { @@ -101,6 +107,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.list.element.save", + description = "Time taken to save a list element") public void saveListElements(final String dataspaceName, final String anchorName, final String parentNodeXpath, final String jsonData, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -112,6 +120,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.list.element.batch.save", + description = "Time taken to save a batch of list elements") public void saveListElementsBatch(final String dataspaceName, final String anchorName, final String parentNodeXpath, final Collection jsonDataList, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -123,6 +133,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.get", + description = "Time taken to get a data node") public DataNode getDataNode(final String dataspaceName, final String anchorName, final String xpath, final FetchDescendantsOption fetchDescendantsOption) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -130,6 +142,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.batch.get", + description = "Time taken to get a batch of data nodes") public Collection getDataNodes(final String dataspaceName, final String anchorName, final Collection xpaths, final FetchDescendantsOption fetchDescendantsOption) { @@ -138,6 +152,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.leaves.update", + description = "Time taken to get a batch of data nodes") public void updateNodeLeaves(final String dataspaceName, final String anchorName, final String parentNodeXpath, final String jsonData, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -148,6 +164,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.leaves.descendants.leaves.update", + description = "Time taken to update data node leaves and existing descendants leaves") public void updateNodeLeavesAndExistingDescendantLeaves(final String dataspaceName, final String anchorName, final String parentNodeXpath, final String dataNodeUpdatesAsJson, @@ -184,6 +202,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.descendants.update", + description = "Time taken to update a data node and descendants") public void updateDataNodeAndDescendants(final String dataspaceName, final String anchorName, final String parentNodeXpath, final String jsonData, final OffsetDateTime observedTimestamp) { @@ -196,6 +216,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.descendants.batch.update", + description = "Time taken to update a batch of data nodes and descendants") public void updateDataNodesAndDescendants(final String dataspaceName, final String anchorName, final Map nodesJsonData, final OffsetDateTime observedTimestamp) { @@ -208,6 +230,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.list.update", + description = "Time taken to update a list") public void replaceListContent(final String dataspaceName, final String anchorName, final String parentNodeXpath, final String jsonData, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -217,6 +241,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.list.batch.update", + description = "Time taken to update a batch of lists") public void replaceListContent(final String dataspaceName, final String anchorName, final String parentNodeXpath, final Collection dataNodes, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -225,6 +251,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.delete", + description = "Time taken to delete a datanode") public void deleteDataNode(final String dataspaceName, final String anchorName, final String dataNodeXpath, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -233,6 +261,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.datanode.batch.delete", + description = "Time taken to delete a batch of datanodes") public void deleteDataNodes(final String dataspaceName, final String anchorName, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -241,6 +271,8 @@ public class CpsDataServiceImpl implements CpsDataService { } @Override + @Timed(value = "cps.data.service.list.delete", + description = "Time taken to delete a list or list element") public void deleteListOrListElement(final String dataspaceName, final String anchorName, final String listNodeXpath, final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); @@ -256,11 +288,12 @@ public class CpsDataServiceImpl implements CpsDataService { final SchemaContext schemaContext = getSchemaContext(dataspaceName, anchor.getSchemaSetName()); if (ROOT_NODE_XPATH.equals(parentNodeXpath)) { - final ContainerNode containerNode = YangUtils.parseData(contentType, nodeData, schemaContext); + final ContainerNode containerNode = timedYangParser.parseData(contentType, nodeData, schemaContext); return new DataNodeBuilder().withContainerNode(containerNode).build(); } - final ContainerNode containerNode = YangUtils.parseData(contentType, nodeData, schemaContext, parentNodeXpath); + final ContainerNode containerNode = + timedYangParser.parseData(contentType, nodeData, schemaContext, parentNodeXpath); return new DataNodeBuilder() .withParentNodeXpath(parentNodeXpath) @@ -283,8 +316,9 @@ public class CpsDataServiceImpl implements CpsDataService { final Anchor anchor = cpsAdminService.getAnchor(dataspaceName, anchorName); final SchemaContext schemaContext = getSchemaContext(dataspaceName, anchor.getSchemaSetName()); + if (ROOT_NODE_XPATH.equals(parentNodeXpath)) { - final ContainerNode containerNode = YangUtils.parseData(contentType, nodeData, schemaContext); + final ContainerNode containerNode = timedYangParser.parseData(contentType, nodeData, schemaContext); final Collection dataNodes = new DataNodeBuilder() .withContainerNode(containerNode) .buildCollection(); @@ -293,7 +327,8 @@ public class CpsDataServiceImpl implements CpsDataService { } return dataNodes; } - final ContainerNode containerNode = YangUtils.parseData(contentType, nodeData, schemaContext, parentNodeXpath); + final ContainerNode containerNode = + timedYangParser.parseData(contentType, nodeData, schemaContext, parentNodeXpath); final Collection dataNodes = new DataNodeBuilder() .withParentNodeXpath(parentNodeXpath) .withContainerNode(containerNode) @@ -302,7 +337,6 @@ public class CpsDataServiceImpl implements CpsDataService { throw new DataValidationException("Invalid data.", "No data nodes provided"); } return dataNodes; - } private Collection> buildDataNodes(final String dataspaceName, final String anchorName, diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java index a04dd2af5..ccd0fcc28 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsModuleServiceImpl.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2022 Nordix Foundation + * Copyright (C) 2020-2023 Nordix Foundation * Modifications Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2022 TechMahindra Ltd @@ -23,6 +23,7 @@ package org.onap.cps.api.impl; +import io.micrometer.core.annotation.Timed; import java.util.Collection; import java.util.Map; import lombok.RequiredArgsConstructor; @@ -36,8 +37,8 @@ import org.onap.cps.spi.model.ModuleDefinition; import org.onap.cps.spi.model.ModuleReference; import org.onap.cps.spi.model.SchemaSet; import org.onap.cps.spi.utils.CpsValidator; +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder; import org.onap.cps.yang.YangTextSchemaSourceSet; -import org.onap.cps.yang.YangTextSchemaSourceSetBuilder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -49,13 +50,16 @@ public class CpsModuleServiceImpl implements CpsModuleService { private final YangTextSchemaSourceSetCache yangTextSchemaSourceSetCache; private final CpsAdminService cpsAdminService; private final CpsValidator cpsValidator; + private final TimedYangTextSchemaSourceSetBuilder timedYangTextSchemaSourceSetBuilder; @Override + @Timed(value = "cps.module.service.schemaset.create", + description = "Time taken to create (and store) a schemaset") public void createSchemaSet(final String dataspaceName, final String schemaSetName, final Map yangResourcesNameToContentMap) { cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); - final var yangTextSchemaSourceSet - = YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap); + final YangTextSchemaSourceSet yangTextSchemaSourceSet = + timedYangTextSchemaSourceSetBuilder.getYangTextSchemaSourceSet(yangResourcesNameToContentMap); cpsModulePersistenceService.storeSchemaSet(dataspaceName, schemaSetName, yangResourcesNameToContentMap); yangTextSchemaSourceSetCache.updateCache(dataspaceName, schemaSetName, yangTextSchemaSourceSet); } diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/YangTextSchemaSourceSetCache.java b/cps-service/src/main/java/org/onap/cps/api/impl/YangTextSchemaSourceSetCache.java index 0f620b0dd..4fdae5a30 100644 --- a/cps-service/src/main/java/org/onap/cps/api/impl/YangTextSchemaSourceSetCache.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/YangTextSchemaSourceSetCache.java @@ -2,7 +2,7 @@ * ============LICENSE_START======================================================= * Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 Bell Canada - * Modifications Copyright (C) 2022 Nordix Foundation + * Modifications Copyright (C) 2022-2023 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,9 @@ package org.onap.cps.api.impl; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.micrometer.core.instrument.Metrics; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import lombok.RequiredArgsConstructor; import org.onap.cps.spi.CpsModulePersistenceService; import org.onap.cps.spi.utils.CpsValidator; @@ -46,6 +48,9 @@ public class YangTextSchemaSourceSetCache { private final CpsModulePersistenceService cpsModulePersistenceService; private final CpsValidator cpsValidator; + private final AtomicInteger yangSchemaCacheCounter = Metrics.gauge("cps.yangschema.cache.gauge", + new AtomicInteger(0)); + /** * Cache YangTextSchemaSourceSet. * @@ -74,10 +79,10 @@ public class YangTextSchemaSourceSetCache { public YangTextSchemaSourceSet updateCache(final String dataspaceName, final String schemaSetName, final YangTextSchemaSourceSet yangTextSchemaSourceSet) { cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + yangSchemaCacheCounter.incrementAndGet(); return yangTextSchemaSourceSet; } - /** * Remove the cached value for the given dataspace and schema-set. * @@ -87,6 +92,7 @@ public class YangTextSchemaSourceSetCache { @CacheEvict(key = "#p0.concat('-').concat(#p1)") public void removeFromCache(final String dataspaceName, final String schemaSetName) { cpsValidator.validateNameCharacters(dataspaceName, schemaSetName); + yangSchemaCacheCounter.decrementAndGet(); // Spring provides implementation for removing object from cache } diff --git a/cps-service/src/main/java/org/onap/cps/utils/TimedYangParser.java b/cps-service/src/main/java/org/onap/cps/utils/TimedYangParser.java new file mode 100644 index 000000000..4640593dc --- /dev/null +++ b/cps-service/src/main/java/org/onap/cps/utils/TimedYangParser.java @@ -0,0 +1,61 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.utils; + +import io.micrometer.core.annotation.Timed; +import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; +import org.opendaylight.yangtools.yang.model.api.SchemaContext; +import org.springframework.stereotype.Service; + +@Service +public class TimedYangParser { + + /** + * Parses data into Collection of NormalizedNode according to given schema context. + * + * @param nodeData data string + * @param schemaContext schema context describing associated data model + * @return the NormalizedNode object + */ + @Timed(value = "cps.utils.yangparser.nodedata.parse", + description = "Time taken to parse node data without a parent") + public ContainerNode parseData(final ContentType contentType, + final String nodeData, + final SchemaContext schemaContext) { + return YangUtils.parseData(contentType, nodeData, schemaContext); + } + + /** + * Parses data into NormalizedNode according to given schema context. + * + * @param nodeData data string + * @param schemaContext schema context describing associated data model + * @return the NormalizedNode object + */ + @Timed(value = "cps.utils.yangparser.nodedata.with.parent.parse", + description = "Time taken to parse node data with a parent") + public ContainerNode parseData(final ContentType contentType, + final String nodeData, + final SchemaContext schemaContext, + final String parentNodeXpath) { + return YangUtils.parseData(contentType, nodeData, schemaContext, parentNodeXpath); + } +} diff --git a/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java b/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java index c0dfe5205..7da402415 100644 --- a/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java +++ b/cps-service/src/main/java/org/onap/cps/utils/YangUtils.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2022 Nordix Foundation + * Copyright (C) 2020-2023 Nordix Foundation * Modifications Copyright (C) 2021 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -87,7 +87,7 @@ public class YangUtils { * @param schemaContext schema context describing associated data model * @return the NormalizedNode object */ - public static ContainerNode parseData(final ContentType contentType, + static ContainerNode parseData(final ContentType contentType, final String nodeData, final SchemaContext schemaContext) { if (contentType == ContentType.JSON) { @@ -103,7 +103,7 @@ public class YangUtils { * @param schemaContext schema context describing associated data model * @return the NormalizedNode object */ - public static ContainerNode parseData(final ContentType contentType, + static ContainerNode parseData(final ContentType contentType, final String nodeData, final SchemaContext schemaContext, final String parentNodeXpath) { diff --git a/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java b/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java new file mode 100644 index 000000000..013faff0c --- /dev/null +++ b/cps-service/src/main/java/org/onap/cps/yang/TimedYangTextSchemaSourceSetBuilder.java @@ -0,0 +1,37 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.yang; + +import io.micrometer.core.annotation.Timed; +import java.util.Map; +import org.springframework.stereotype.Service; + +@Service +public class TimedYangTextSchemaSourceSetBuilder { + + @Timed(value = "cps.yangtextschemasourceset.build", + description = "Time taken to build a yang text schema source set") + public YangTextSchemaSourceSet getYangTextSchemaSourceSet( + final Map yangResourcesNameToContentMap) { + return YangTextSchemaSourceSetBuilder.of(yangResourcesNameToContentMap); + } + +} diff --git a/cps-service/src/main/java/org/onap/cps/yang/YangTextSchemaSourceSetBuilder.java b/cps-service/src/main/java/org/onap/cps/yang/YangTextSchemaSourceSetBuilder.java index e0f24f315..deb5b0575 100644 --- a/cps-service/src/main/java/org/onap/cps/yang/YangTextSchemaSourceSetBuilder.java +++ b/cps-service/src/main/java/org/onap/cps/yang/YangTextSchemaSourceSetBuilder.java @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2020 Pantheon.tech - * Modifications Copyright (C) 2022 Nordix Foundation. + * Modifications Copyright (C) 2022-2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; +import io.micrometer.core.annotation.Timed; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -58,16 +59,35 @@ public final class YangTextSchemaSourceSetBuilder { private final ImmutableMap.Builder yangModelMap = new ImmutableMap.Builder<>(); + /** + * Add Yang resource context. + * + * @param yangResourceNameToContent the resource content + * @return this builder + */ public YangTextSchemaSourceSetBuilder putAll(final Map yangResourceNameToContent) { this.yangModelMap.putAll(yangResourceNameToContent); return this; } + /** + * Build a YangTextSchemaSourceSet. + * + * @return the YangTextSchemaSourceSet + */ public YangTextSchemaSourceSet build() { final var schemaContext = generateSchemaContext(yangModelMap.build()); return new YangTextSchemaSourceSetImpl(schemaContext); } + /** + * Add yangResourceNameToContent and build a YangTextSchemaSourceSet. + * + * @param yangResourceNameToContent the resource content + * @return the YangTextSchemaSourceSet + */ + + @Timed(value = "cps.yang.schemasourceset.build", description = "Time taken to build a ODL yang Model") public static YangTextSchemaSourceSet of(final Map yangResourceNameToContent) { return new YangTextSchemaSourceSetBuilder().putAll(yangResourceNameToContent).build(); } diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy index c81a50ea7..01dc0bde4 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2022 Nordix Foundation + * Copyright (C) 2021-2023 Nordix Foundation * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2021-2022 Bell Canada. * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -34,6 +34,7 @@ import org.onap.cps.spi.model.Anchor import org.onap.cps.spi.model.DataNode import org.onap.cps.spi.model.DataNodeBuilder import org.onap.cps.utils.ContentType +import org.onap.cps.utils.TimedYangParser import org.onap.cps.yang.YangTextSchemaSourceSet import org.onap.cps.yang.YangTextSchemaSourceSetBuilder import spock.lang.Specification @@ -48,9 +49,10 @@ class CpsDataServiceImplSpec extends Specification { def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache) def mockNotificationService = Mock(NotificationService) def mockCpsValidator = Mock(CpsValidator) + def timedYangParser = new TimedYangParser() def objectUnderTest = new CpsDataServiceImpl(mockCpsDataPersistenceService, mockCpsAdminService, - mockYangTextSchemaSourceSetCache, mockNotificationService, mockCpsValidator) + mockYangTextSchemaSourceSetCache, mockNotificationService, mockCpsValidator, timedYangParser) def setup() { mockCpsAdminService.getAnchor(dataspaceName, anchorName) >> anchor diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy index 358a6fb3f..2bb0e63ab 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsModuleServiceImplSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2022 Nordix Foundation + * Copyright (C) 2020-2023 Nordix Foundation * Modifications Copyright (C) 2020-2021 Pantheon.tech * Modifications Copyright (C) 2020-2022 Bell Canada. * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -32,6 +32,7 @@ import org.onap.cps.spi.utils.CpsValidator import org.onap.cps.spi.model.Anchor import org.onap.cps.spi.model.ModuleReference import org.onap.cps.spi.model.SchemaSet +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder import org.onap.cps.yang.YangTextSchemaSourceSet import org.onap.cps.yang.YangTextSchemaSourceSetBuilder import spock.lang.Specification @@ -44,8 +45,9 @@ class CpsModuleServiceImplSpec extends Specification { def mockCpsAdminService = Mock(CpsAdminService) def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache) def mockCpsValidator = Mock(CpsValidator) + def timedYangTextSchemaSourceSetBuilder = new TimedYangTextSchemaSourceSetBuilder() - def objectUnderTest = new CpsModuleServiceImpl(mockCpsModulePersistenceService, mockYangTextSchemaSourceSetCache, mockCpsAdminService, mockCpsValidator) + def objectUnderTest = new CpsModuleServiceImpl(mockCpsModulePersistenceService, mockYangTextSchemaSourceSetCache, mockCpsAdminService, mockCpsValidator,timedYangTextSchemaSourceSetBuilder) def 'Create schema set.'() { given: 'Valid yang resource as name-to-content map' diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy index ccfb23b44..8ed7aede6 100755 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/E2ENetworkSliceSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2022 Nordix Foundation. + * Copyright (C) 2021-2023 Nordix Foundation. * Modifications Copyright (C) 2021-2022 Bell Canada. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -30,7 +30,9 @@ import org.onap.cps.spi.CpsDataPersistenceService import org.onap.cps.spi.CpsModulePersistenceService import org.onap.cps.spi.model.Anchor import org.onap.cps.spi.utils.CpsValidator +import org.onap.cps.utils.TimedYangParser import org.onap.cps.utils.YangUtils +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder import org.onap.cps.yang.YangTextSchemaSourceSetBuilder import spock.lang.Specification @@ -41,10 +43,14 @@ class E2ENetworkSliceSpec extends Specification { def mockNotificationService = Mock(NotificationService) def mockYangTextSchemaSourceSetCache = Mock(YangTextSchemaSourceSetCache) def mockCpsValidator = Mock(CpsValidator) + def timedYangTextSchemaSourceSetBuilder = new TimedYangTextSchemaSourceSetBuilder() + def timedYangParser = new TimedYangParser() + def cpsModuleServiceImpl = new CpsModuleServiceImpl(mockModuleStoreService, - mockYangTextSchemaSourceSetCache, mockCpsAdminService, mockCpsValidator) + mockYangTextSchemaSourceSetCache, mockCpsAdminService, mockCpsValidator,timedYangTextSchemaSourceSetBuilder) + def cpsDataServiceImpl = new CpsDataServiceImpl(mockDataStoreService, mockCpsAdminService, - mockYangTextSchemaSourceSetCache, mockNotificationService, mockCpsValidator) + mockYangTextSchemaSourceSetCache, mockNotificationService, mockCpsValidator, timedYangParser) def dataspaceName = 'someDataspace' def anchorName = 'someAnchor' diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index fe41fc4ff..7d6379ac2 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -119,4 +119,4 @@ services: - ../../../cps-ncmp-service/src/main/resources/model/:/model depends_on: cps-and-ncmp: - condition: service_started \ No newline at end of file + condition: service_started diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy index 33283f1f8..0e04d62dd 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy @@ -36,6 +36,8 @@ import org.onap.cps.spi.repository.SchemaSetRepository import org.onap.cps.spi.repository.YangResourceRepository import org.onap.cps.spi.utils.SessionManager import org.onap.cps.utils.JsonObjectMapper +import org.onap.cps.utils.TimedYangParser +import org.onap.cps.yang.TimedYangTextSchemaSourceSetBuilder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -76,10 +78,6 @@ class TestConfig extends Specification{ @Lazy JsonObjectMapper jsonObjectMapper - @Autowired - @Lazy - NotificationService stubbedNotificationService - @Autowired @Lazy SessionManager stubbedSessionManager @@ -113,4 +111,15 @@ class TestConfig extends Specification{ SessionManager sessionManager() { return Stub(SessionManager) } -} \ No newline at end of file + + @Bean + TimedYangParser timedYangParser() { + return new TimedYangParser() + } + + @Bean + TimedYangTextSchemaSourceSetBuilder textSchemaSourceSetBuilder() { + return new TimedYangTextSchemaSourceSetBuilder() + } + +} -- 2.16.6