def 'CPS Path Processing Performance Test.'() {
when: '200,000 paths are processed'
- def setupStopWatch = new StopWatch()
- setupStopWatch.start()
+ def stopWatch = new StopWatch()
+ stopWatch.start()
(1..100000).each {
CpsPathUtil.getNormalizedXpath('/long/path/to/see/if/it/adds/paring/time/significantly/parent/child[@common-leaf-name="123"]')
CpsPathUtil.getNormalizedXpath('//child[@other-leaf=1]/leaf-name[text()="search"]/ancestor::parent')
}
- setupStopWatch.stop()
+ stopWatch.stop()
then: 'it takes less then 10,000 milliseconds'
// In CI this actually takes about 3-5 sec which is approx. 50+ parser executions per millisecond!
- assert setupStopWatch.getTotalTimeMillis() < 10000
+ assert stopWatch.getTotalTimeMillis() < 10000
}
}
*/
public interface ModuleReferenceQuery {
- Collection<ModuleReference> identifyNewModuleReferences(
- final Collection<ModuleReference> moduleReferencesToCheck);
+ Collection<ModuleReference> identifyNewModuleReferences(final Collection<ModuleReference> moduleReferencesToCheck);
}
package org.onap.cps.spi.repository;
-import java.util.Collection;
import org.onap.cps.spi.entities.YangResourceEntity;
-import org.onap.cps.spi.model.ModuleReference;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
-public interface ModuleReferenceRepository extends JpaRepository<YangResourceEntity, Long>, ModuleReferenceQuery {
+public interface ModuleReferenceRepository extends JpaRepository<YangResourceEntity, Long>, ModuleReferenceQuery {}
- Collection<ModuleReference> identifyNewModuleReferences(
- final Collection<ModuleReference> moduleReferencesToCheck);
-
-}
\ No newline at end of file
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
-import java.util.UUID;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import lombok.AllArgsConstructor;
@PersistenceContext
private EntityManager entityManager;
+ private TempTableCreator tempTableCreator;
+
@Override
@SneakyThrows
public Collection<ModuleReference> identifyNewModuleReferences(
return Collections.emptyList();
}
- final String tempTableName = "moduleReferencesToCheckTemp"
- + UUID.randomUUID().toString().replace("-", "");
-
- createTemporaryTable(tempTableName);
- insertDataIntoTable(tempTableName, moduleReferencesToCheck);
-
- return identifyNewModuleReferencesForCmHandle(tempTableName);
- }
-
- private void createTemporaryTable(final String tempTableName) {
- final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE " + tempTableName + "(");
- sqlStringBuilder.append(" id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,");
- sqlStringBuilder.append(" module_name varchar NOT NULL,");
- sqlStringBuilder.append(" revision varchar NOT NULL");
- sqlStringBuilder.append(");");
-
- entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate();
- }
-
- private void insertDataIntoTable(final String tempTableName, final Collection<ModuleReference> moduleReferences) {
- final StringBuilder sqlStringBuilder = new StringBuilder("INSERT INTO " + tempTableName);
- sqlStringBuilder.append(" (module_name, revision) ");
- sqlStringBuilder.append(" VALUES ");
-
- for (final ModuleReference moduleReference : moduleReferences) {
- sqlStringBuilder.append("('");
- sqlStringBuilder.append(moduleReference.getModuleName());
- sqlStringBuilder.append("', '");
- sqlStringBuilder.append(moduleReference.getRevision());
- sqlStringBuilder.append("'),");
+ final Collection<List<String>> sqlData = new HashSet<>(moduleReferencesToCheck.size());
+ for (final ModuleReference moduleReference : moduleReferencesToCheck) {
+ final List<String> row = new ArrayList<>(2);
+ row.add(moduleReference.getModuleName());
+ row.add(moduleReference.getRevision());
+ sqlData.add(row);
}
- // replace last ',' with ';'
- sqlStringBuilder.replace(sqlStringBuilder.length() - 1, sqlStringBuilder.length(), ";");
+ final String tempTableName = tempTableCreator.createTemporaryTable(
+ "moduleReferencesToCheckTemp", sqlData, "module_name", "revision");
- entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate();
+ return identifyNewModuleReferencesForCmHandle(tempTableName);
}
private Collection<ModuleReference> identifyNewModuleReferencesForCmHandle(final String tempTableName) {
--- /dev/null
+/*-
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2022 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.spi.repository;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+@Slf4j
+@Transactional
+@AllArgsConstructor
+@Component
+public class TempTableCreator {
+
+ @PersistenceContext
+ private EntityManager entityManager;
+
+ /**
+ * Create a uniquely named temporary table.
+ *
+ * @param prefix prefix for the table name (so you can recognize it)
+ * @param sqlData data to insert (strings only) the inner List present a row of data
+ * @param columnNames column names (in same order as data in rows in sqlData)
+ * @return a unique temporary table name with given prefix
+ */
+ public String createTemporaryTable(final String prefix,
+ final Collection<List<String>> sqlData,
+ final String... columnNames) {
+ final String tempTableName = prefix + UUID.randomUUID().toString().replace("-", "");
+ final StringBuilder sqlStringBuilder = new StringBuilder("CREATE TEMPORARY TABLE ");
+ sqlStringBuilder.append(tempTableName);
+ defineColumns(sqlStringBuilder, columnNames);
+ insertData(sqlStringBuilder, tempTableName, columnNames, sqlData);
+ entityManager.createNativeQuery(sqlStringBuilder.toString()).executeUpdate();
+ return tempTableName;
+ }
+
+ private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) {
+ sqlStringBuilder.append('(');
+ final Iterator<String> it = Arrays.stream(columnNames).iterator();
+ while (it.hasNext()) {
+ final String columnName = it.next();
+ sqlStringBuilder.append(" ");
+ sqlStringBuilder.append(columnName);
+ sqlStringBuilder.append(" varchar NOT NULL");
+ if (it.hasNext()) {
+ sqlStringBuilder.append(",");
+ }
+ }
+ sqlStringBuilder.append(");");
+ }
+
+ private static void insertData(final StringBuilder sqlStringBuilder,
+ final String tempTableName,
+ final String[] columnNames,
+ final Collection<List<String>> sqlData) {
+ final Collection<String> sqlInserts = new HashSet<>(sqlData.size());
+ for (final Collection<String> row : sqlData) {
+ sqlInserts.add("('" + String.join("','", row) + "')");
+ }
+ sqlStringBuilder.append("INSERT INTO ");
+ sqlStringBuilder.append(tempTableName);
+ sqlStringBuilder.append(" (");
+ sqlStringBuilder.append(String.join(",", columnNames));
+ sqlStringBuilder.append(") VALUES ");
+ sqlStringBuilder.append(String.join(",", sqlInserts));
+ sqlStringBuilder.append(";");
+ }
+
+}
def 'Identifying new module references where #scenario'() {
when: 'identifyNewModuleReferences is called'
def result = objectUnderTest.identifyNewModuleReferences(moduleReferences)
- then: 'the correct module reference collection is returned'
- assert result == expectedResult
+ then: 'the correct module references are returned'
+ assert result.size() == expectedResult.size()
+ assert result.containsAll(expectedResult)
where: 'the following data is used'
scenario | moduleReferences || expectedResult
'new module references exist' | toModuleReference([['some module 1' : 'some revision 1'], ['some module 2' : 'some revision 2']]) || toModuleReference([['some module 1' : 'some revision 1'], ['some module 2' : 'some revision 2']])
def moduleReferences = [].withDefault { [:] }
moduleReferenceAsMap.forEach(property ->
property.forEach((moduleName, revision) -> {
- moduleReferences.add(new ModuleReference('moduleName' : moduleName, 'revision' : revision))
+ moduleReferences.add(new ModuleReference(moduleName, revision))
}))
return moduleReferences
}
--- /dev/null
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2022 Nordix Foundation
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.spi.performance
+
+import org.onap.cps.spi.CpsModulePersistenceService
+import org.onap.cps.spi.entities.SchemaSetEntity
+import org.onap.cps.spi.impl.CpsPersistenceSpecBase
+import org.onap.cps.spi.model.ModuleReference
+import org.onap.cps.spi.repository.DataspaceRepository
+import org.onap.cps.spi.repository.ModuleReferenceRepository
+import org.onap.cps.spi.repository.SchemaSetRepository
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.test.context.jdbc.Sql
+import org.springframework.util.StopWatch
+
+import java.util.concurrent.ThreadLocalRandom
+
+class CpsModuleReferenceRepositoryPerfTest extends CpsPersistenceSpecBase {
+
+ static final String PERF_TEST_DATA = '/data/perf-test.sql'
+
+ def NEW_RESOURCE_CONTENT = 'module stores {\n' +
+ ' yang-version 1.1;\n' +
+ ' namespace "org:onap:ccsdk:sample";\n' +
+ '\n' +
+ ' prefix book-store;\n' +
+ '\n' +
+ ' revision "2020-09-15" {\n' +
+ ' description\n' +
+ ' "Sample Model";\n' +
+ ' }' +
+ '}'
+
+ @Autowired
+ CpsModulePersistenceService objectUnderTest
+
+ @Autowired
+ DataspaceRepository dataspaceRepository
+
+ @Autowired
+ SchemaSetRepository schemaSetRepository
+
+ @Autowired
+ ModuleReferenceRepository moduleReferenceRepository
+
+ @Sql([CLEAR_DATA, PERF_TEST_DATA])
+ def 'Store new schema set with many modules'() {
+ when: 'a new schema set with 200 modules is stored'
+ def newYangResourcesNameToContentMap = [:]
+ (1..200).each {
+ def year = 2000 + it
+ def resourceName = "module${it}".toString()
+ def moduleName = "stores${it}"
+ def content = NEW_RESOURCE_CONTENT.replace('2020',String.valueOf(year)).replace('stores',moduleName)
+ newYangResourcesNameToContentMap.put(resourceName, content)
+ }
+ objectUnderTest.storeSchemaSet('PERF-DATASPACE', 'perfSchemaSet', newYangResourcesNameToContentMap)
+ then: 'the schema set is persisted correctly'
+ def dataspaceEntity = dataspaceRepository.getByName('PERF-DATASPACE')
+ SchemaSetEntity result = schemaSetRepository.getByDataspaceAndName(dataspaceEntity, 'perfSchemaSet')
+ result.yangResources.size() == 200
+ and: 'identification of new module resources is fast enough (1,000 executions less then 5,000 milliseconds)'
+ def stopWatch = new StopWatch()
+ 1000.times() {
+ def moduleReferencesToCheck = createModuleReferencesWithRandomMatchingExistingModuleReferences()
+ stopWatch.start()
+ def newModuleReferences = moduleReferenceRepository.identifyNewModuleReferences(moduleReferencesToCheck)
+ stopWatch.stop()
+ assert newModuleReferences.size() > 0 && newModuleReferences.size() < 300
+ }
+ assert stopWatch.getTotalTimeMillis() < 5000
+ }
+
+ def createModuleReferencesWithRandomMatchingExistingModuleReferences() {
+ def moduleReferences = []
+ (1..250).each {
+ def randomNumber = ThreadLocalRandom.current().nextInt(1, 300)
+ def year = 2000 + randomNumber
+ def moduleName = "stores${randomNumber}"
+ moduleReferences.add(new ModuleReference(moduleName, "${year}-09-15"))
+ }
+ return moduleReferences
+ }
+
+}
static def ALLOWED_SETUP_TIME_MS = TimeUnit.SECONDS.toMillis(10)
static def ALLOWED_READ_TIME_AL_NODES_MS = 500
- def readStopWatch = new StopWatch()
+ def stopWatch = new StopWatch()
@Sql([CLEAR_DATA, PERF_TEST_DATA])
def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() {
given: 'a node with a large number of descendants is created'
- def setupStopWatch = new StopWatch()
- setupStopWatch.start()
+ stopWatch.start()
createLineage()
- setupStopWatch.stop()
- def setupDurationInMillis = setupStopWatch.getTotalTimeMillis()
+ stopWatch.stop()
+ def setupDurationInMillis = stopWatch.getTotalTimeMillis()
and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds'
assert setupDurationInMillis < ALLOWED_SETUP_TIME_MS
}
def 'Get data node with many descendants by xpath #scenario'() {
when: 'get parent is executed with all descendants'
- readStopWatch.start()
+ stopWatch.start()
def result = objectUnderTest.getDataNode('PERF-DATASPACE', 'PERF-ANCHOR', xpath, INCLUDE_ALL_DESCENDANTS)
- readStopWatch.stop()
- def readDurationInMillis = readStopWatch.getTotalTimeMillis()
+ stopWatch.stop()
+ def readDurationInMillis = stopWatch.getTotalTimeMillis()
then: 'read duration is under 500 milliseconds'
assert readDurationInMillis < ALLOWED_READ_TIME_AL_NODES_MS
and: 'data node is returned with all the descendants populated'
def 'Query parent data node with many descendants by cps-path'() {
when: 'query is executed with all descendants'
- readStopWatch.start()
+ stopWatch.start()
def result = objectUnderTest.queryDataNodes('PERF-DATASPACE', 'PERF-ANCHOR', '//perf-parent-1' , INCLUDE_ALL_DESCENDANTS)
- readStopWatch.stop()
- def readDurationInMillis = readStopWatch.getTotalTimeMillis()
+ stopWatch.stop()
+ def readDurationInMillis = stopWatch.getTotalTimeMillis()
then: 'read duration is under 500 milliseconds'
assert readDurationInMillis < ALLOWED_READ_TIME_AL_NODES_MS
and: 'data node is returned with all the descendants populated'
def 'Query many descendants by cps-path with #scenario'() {
when: 'query is executed with all descendants'
- readStopWatch.start()
+ stopWatch.start()
def result = objectUnderTest.queryDataNodes('PERF-DATASPACE', 'PERF-ANCHOR', '//perf-test-grand-child-1', descendantsOption)
- readStopWatch.stop()
- def readDurationInMillis = readStopWatch.getTotalTimeMillis()
+ stopWatch.stop()
+ def readDurationInMillis = stopWatch.getTotalTimeMillis()
then: 'read duration is under 500 milliseconds'
assert readDurationInMillis < alowedDuration
and: 'data node is returned with all the descendants populated'
this.revision = revision;
this.namespace = "";
}
+
}
Code Review / cps.git / commitdiff