-<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2014, Oracle America, Inc.
All rights reserved.
THE POSSIBILITY OF SUCH DAMAGE.
-->
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
<artifactId>mdbc</artifactId>
- <groupId>org.onap.music.mdbc</groupId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>mdbc</artifactId>
+ <groupId>org.onap.music.mdbc</groupId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
- <artifactId>benchmark</artifactId>
- <packaging>jar</packaging>
+ <artifactId>mdbc-benchmark</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>mdbc-benchmark</name>
+ <description>MDBC Benchmark</description>
+ <packaging>jar</packaging>
- <name>JMH benchmark sample: Java</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20160810</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.att.eelf</groupId>
+ <artifactId>eelf-core</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-server</artifactId>
+ <version>1.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>1.1.7</version>
+ </dependency>
+ <dependency>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>9.1-901-1.jdbc4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>3.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>19.0</version>
+ </dependency>
+ <dependency>
+ <!-- The oparent dependency managed version 9.4.12.v20180830 will not work -->
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <version>9.2.19.v20160908</version>
+ </dependency>
+ <dependency>
+ <!-- The oparent dependency managed version 9.4.12.v20180830 will not work -->
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>9.2.19.v20160908</version>
+ </dependency>
+ <dependency>
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <version>9.2.19.v20160908</version>
+ </dependency>
+ <dependency>
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <version>9.2.19.v20160908</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.7</version>
+ </dependency>
+ <!-- These two dependencies pull in optional libraries for Cassandra -->
+ <dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>1.1.2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.jsqlparser</groupId>
+ <artifactId>jsqlparser</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>1.2.3</version>
+ </dependency>
- <!--
- This is the demo/sample template build script for building Java benchmarks with JMH.
- Edit as needed.
- -->
+ <!-- https://mvnrepository.com/artifact/javax.servlet/servlet-api -->
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <version>2.5</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.apache.calcite.avatica/avatica-server -->
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-server</artifactId>
+ <version>1.12.0</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-plus -->
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-plus</artifactId>
+ <version>1.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ <version>1.72</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ <version>4.1.30.Final</version>
+ </dependency>
+ <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>3.6.1</version>
+ </dependency>
+ </dependencies>
- <dependencies>
- <dependency>
- <groupId>org.json</groupId>
- <artifactId>json</artifactId>
- <version>20160810</version>
- </dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-core</artifactId>
- <version>${jmh.version}</version>
- </dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-generator-annprocess</artifactId>
- <version>${jmh.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.att.eelf</groupId>
- <artifactId>eelf-core</artifactId>
- <version>1.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica-server</artifactId>
- <version>1.12.0</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.8.5</version>
- </dependency>
- <dependency>
- <groupId>org.mariadb.jdbc</groupId>
- <artifactId>mariadb-java-client</artifactId>
- <version>1.1.7</version>
- </dependency>
- <dependency>
- <groupId>postgresql</groupId>
- <artifactId>postgresql</artifactId>
- <version>9.1-901-1.jdbc4</version>
- </dependency>
- </dependencies>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-
- <!--
+ <!--
JMH version to use with this project.
-->
<jmh.version>1.21</jmh.version>
<!--
Name of the benchmark Uber-JAR to generate.
-->
- <uberjar.name>benchmarks</uberjar.name>
+ <!--
+ Name of the benchmark Uber-JAR to generate.
+ -->
+ <uberjar.name>benchmarks</uberjar.name>
</properties>
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <compilerVersion>${javac.target}</compilerVersion>
- <source>${javac.target}</source>
- <target>${javac.target}</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.2</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <finalName>${uberjar.name}</finalName>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.openjdk.jmh.Main</mainClass>
- </transformer>
- </transformers>
- <filters>
- <filter>
- <!--
- Shading signed JARs will fail without this.
- http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
- -->
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <version>2.5</version>
- </plugin>
- <plugin>
- <artifactId>maven-deploy-plugin</artifactId>
- <version>2.8.1</version>
- </plugin>
- <plugin>
- <artifactId>maven-install-plugin</artifactId>
- <version>2.5.1</version>
- </plugin>
- <plugin>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.4</version>
- </plugin>
- <plugin>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.9.1</version>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- </plugin>
- <plugin>
- <artifactId>maven-site-plugin</artifactId>
- <version>3.3</version>
- </plugin>
- <plugin>
- <artifactId>maven-source-plugin</artifactId>
- <version>2.2.1</version>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.17</version>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <compilerVersion>${javac.target}</compilerVersion>
+ <source>${javac.target}</source>
+ <target>${javac.target}</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${uberjar.name}</finalName>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <!--
+ Shading signed JARs will fail without this.
+ http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
+ -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.17</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+</build>
</project>
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+public class BenchmarkUtils {
+ public static final String driver = "org.apache.calcite.avatica.remote.Driver";
+ public static final String mariaDriver = "org.mariadb.jdbc.Driver";
+ public static final String postgresDriver = "org.postgresql.Driver";
+ public static String TABLE;
+ public static String updateBuilder;
+
+ public enum ExecutionType {
+ MARIA_DB, COCKROACH_DB, METRIC,POSTGRES
+ }
+
+ public static void SetupTable(String table){
+ TABLE=table;
+ updateBuilder = new StringBuilder()
+ .append("UPDATE ")
+ .append(table)
+ .append(" SET Counter = Counter + 1,")
+ .append("City = 'Sandy Springs'")
+ .append(";").toString();
+ }
+
+ public static void setupCreateTables(Connection connection){
+ createTable(connection);
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static void setupCreateRows(Connection testConnection, int rows){
+ //Empty database
+ boolean cleanResult = BenchmarkUtils.cleanTable(testConnection);
+ //Add new lines
+ BenchmarkUtils.addRowsToTable(Integer.valueOf(rows),testConnection);
+
+ //Commit
+ try {
+ testConnection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static Connection getConnection(String ip, ExecutionType type, String user, String password) {
+ final String connectionUrl = "jdbc:avatica:remote:url=http://" + ip + ":30000;serialization=protobuf";
+ final String mariaConnectionUrl = "jdbc:mariadb://" + ip + ":3306/test";
+ final String cockroachUrl = "jdbc:postgresql://" + ip + ":26257/test";
+ final String postgresUrl = "jdbc:postgresql://" + ip + ":5432/test";
+
+ try {
+ switch (type) {
+ case MARIA_DB:
+ Class.forName(mariaDriver);
+ break;
+ case METRIC:
+ Class.forName(driver);
+ break;
+ case COCKROACH_DB:
+ case POSTGRES:
+ Class.forName(postgresDriver);
+ break;
+ }
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ Connection connection = null;
+ try {
+ if (type == ExecutionType.METRIC) {
+ connection = DriverManager.getConnection(connectionUrl);
+ } else {
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", user);
+ if (type == ExecutionType.COCKROACH_DB) {
+ connectionProps.setProperty("sslmode", "disable");
+ } else {
+ connectionProps.put("password", password);
+ }
+ final String url = (type == ExecutionType.MARIA_DB) ? mariaConnectionUrl :
+ (type == ExecutionType.COCKROACH_DB)?cockroachUrl:postgresUrl;
+ connection = DriverManager.getConnection(url, connectionProps);
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ connection.setAutoCommit(false);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ return connection;
+ }
+
+ public static void createTable(Connection connection) {
+ final String sql = "CREATE TABLE IF NOT EXISTS "+TABLE+" (\n" +
+ " PersonID int,\n" +
+ " Counter int,\n" +
+ " LastName varchar(255),\n" +
+ " FirstName varchar(255),\n" +
+ " Address varchar(255),\n" +
+ " City varchar(255),\n" +
+ " PRIMARY KEY(PersonID)"+
+ ");";
+
+ Statement stmt = null;
+ try {
+ stmt = connection.createStatement();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ Boolean execute = null;
+ try {
+ execute = stmt.execute(sql);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ try {
+ connection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static boolean cleanTable(Connection testConnection) {
+ String cleanCmd = "DELETE FROM "+TABLE+";";
+ Statement stmt = null;
+ try {
+ stmt = testConnection.createStatement();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ Boolean execute = null;
+ try {
+ execute = stmt.execute(cleanCmd);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ testConnection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ return execute;
+ }
+
+ public static void addRowsToTable(int totalNumberOfRows, Connection testConnection) {
+ for (int i = 0; i < totalNumberOfRows; i++) {
+ final StringBuilder insertSQLBuilder = new StringBuilder()
+ .append("INSERT INTO "+TABLE+" VALUES (")
+ .append(i)
+ .append(", ")
+ .append(0)
+ .append(", '")
+ .append("Last-")
+ .append(i)
+ .append("', '")
+ .append("First-")
+ .append(i)
+ .append("', 'KACB', 'ATLANTA');");
+ Statement stmt = null;
+ try {
+ stmt = testConnection.createStatement();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ Boolean execute = null;
+ try {
+ execute = stmt.execute(insertSQLBuilder.toString());
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+ try {
+ testConnection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+
+}
* ============LICENSE_START====================================================
* org.onap.music.mdbc
* =============================================================================
- * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
* =============================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
*/
package org.onap.music.mdbc;
+import static org.onap.music.mdbc.BenchmarkUtils.setupCreateRows;
+import static org.onap.music.mdbc.BenchmarkUtils.setupCreateTables;
+
+import org.onap.music.mdbc.BenchmarkUtils.ExecutionType;
import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import java.util.Properties;
import java.util.concurrent.TimeUnit;
@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
public class MetricBenchmark {
- public StringBuilder updateBuilder = new StringBuilder()
- .append("UPDATE PERSONS ")
- .append("SET Counter = Counter + 1,")
- .append("City = 'Sandy Springs'")
- .append(";");
- public String update = updateBuilder.toString();
public static void main(String[] args) throws RunnerException {
Options opt = new OptionsBuilder()
.include(MetricBenchmark.class.getSimpleName())
.param("type", ExecutionType.METRIC.name())
+ .forks(0)
+ .threads(1)
.build();
new Runner(opt).run();
}
@Benchmark
- public boolean testMethod(MyState state) {
+ public boolean testMethod(MyState state, Blackhole blackhole) {
Statement stmt = null;
try {
stmt = state.testConnection.createStatement();
Boolean execute = null;
try {
- execute = stmt.execute(update);
+ execute = stmt.execute(state.update);
} catch (SQLException e) {
e.printStackTrace();
System.exit(1);
e.printStackTrace();
System.exit(1);
}
+ blackhole.consume(execute);
return execute;
}
- public static enum ExecutionType {
- MARIA_DB, COCKROACH_DB, METRIC
- }
@State(Scope.Benchmark)
public static class MyState {
+ public String update;
public final String driver = "org.apache.calcite.avatica.remote.Driver";
public final String mariaDriver = "org.mariadb.jdbc.Driver";
public final String cockroachDriver = "org.postgresql.Driver";
final String password = "metriccluster";
@Param({"104.209.240.219"})
public String ip;
+ @Param({"PERSONS"})
+ public String table;
@Param({"1", "10", "50", "80", "100", "200", "300", "400"})
public int rows;
- @Param({"MARIA_DB", "COCKROACH_DB", "METRIC"})
+ @Param({"MARIA_DB", "COCKROACH_DB", "METRIC", "POSTGRES"})
public ExecutionType type;
public Connection testConnection;
private Connection createConnection() {
- final String connectionUrl = "jdbc:avatica:remote:url=http://" + ip + ":30000;serialization=protobuf";
- final String mariaConnectionUrl = "jdbc:mariadb://" + ip + ":3306/test";
- final String cockroachUrl = "jdbc:postgresql://" + ip + ":26257/test";
-
- try {
- switch (type) {
- case MARIA_DB:
- Class.forName(this.mariaDriver);
- break;
- case METRIC:
- Class.forName(this.driver);
- break;
- case COCKROACH_DB:
- Class.forName(this.cockroachDriver);
- break;
- }
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- System.exit(1);
- }
- Connection connection = null;
- try {
- if (type == ExecutionType.METRIC) {
- connection = DriverManager.getConnection(connectionUrl);
- } else {
- Properties connectionProps = new Properties();
- connectionProps.put("user", user);
- if(type == ExecutionType.COCKROACH_DB){
- connectionProps.setProperty("sslmode", "disable");
- }
- else{
- connectionProps.put("password", password);
- }
- final String url = (type == ExecutionType.MARIA_DB) ? mariaConnectionUrl : cockroachUrl;
- connection = DriverManager.getConnection(url, connectionProps);
- }
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- try {
- connection.setAutoCommit(false);
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- return connection;
+ return BenchmarkUtils.getConnection(ip,type,user,password);
}
- private void createTable(Connection connection) {
- final String sql = "CREATE TABLE IF NOT EXISTS PERSONS (\n" +
- " PersonID int,\n" +
- " Counter int,\n" +
- " LastName varchar(255),\n" +
- " FirstName varchar(255),\n" +
- " Address varchar(255),\n" +
- " City varchar(255)\n" +
- ");";
-
- Statement stmt = null;
- try {
- stmt = connection.createStatement();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- Boolean execute = null;
- try {
- execute = stmt.execute(sql);
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- try {
- connection.commit();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- try {
- stmt.close();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- }
-
- private boolean cleanTable() {
- String cleanCmd = "DELETE FROM PERSONS;";
- Statement stmt = null;
- try {
- stmt = testConnection.createStatement();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- Boolean execute = null;
- try {
- execute = stmt.execute(cleanCmd);
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- try {
- testConnection.commit();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- try {
- stmt.close();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- return execute;
- }
-
- private void addRowsToTable(int totalNumberOfRows) {
- for (int i = 0; i < totalNumberOfRows; i++) {
- final StringBuilder insertSQLBuilder = new StringBuilder()
- .append("INSERT INTO PERSONS VALUES (")
- .append(i)
- .append(", ")
- .append(0)
- .append(", '")
- .append("Last-")
- .append(i)
- .append("', '")
- .append("First-")
- .append(i)
- .append("', 'KACB', 'ATLANTA');");
- Statement stmt = null;
- try {
- stmt = testConnection.createStatement();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- Boolean execute = null;
- try {
- execute = stmt.execute(insertSQLBuilder.toString());
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- try {
- stmt.close();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
- }
- try {
- testConnection.commit();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ BenchmarkUtils.SetupTable(table);
+ update = BenchmarkUtils.updateBuilder;
}
@Setup(Level.Iteration)
public void doSetup() {
Connection connection = createConnection();
- createTable(connection);
- try {
- connection.close();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
+ setupCreateTables(connection);
//Setup connection
testConnection = createConnection();
- //Empty database
- boolean cleanResult = cleanTable();
- //Add new lines
- addRowsToTable(Integer.valueOf(rows));
+ setupCreateRows(testConnection,rows);
- //Commit
- try {
- testConnection.commit();
- } catch (SQLException e) {
- e.printStackTrace();
- System.exit(1);
- }
}
@TearDown(Level.Iteration)
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc;
+
+import static org.onap.music.mdbc.BenchmarkUtils.setupCreateRows;
+import static org.onap.music.mdbc.BenchmarkUtils.setupCreateTables;
+
+import org.onap.music.mdbc.BenchmarkUtils.ExecutionType;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricCommitBenchmark {
+
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricBenchmark.class.getSimpleName())
+ .param("type", ExecutionType.METRIC.name())
+ .forks(1)
+ .threads(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state) {
+ try {
+ state.testConnection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+
+ String update;
+ final String user = "root";
+ final String password = "metriccluster";
+ @Param({"104.209.240.219"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ @Param({"MARIA_DB", "COCKROACH_DB", "METRIC","POSTGRES"})
+ public ExecutionType type;
+ @Param({"PERSONS"})
+ public String table;
+
+ public Connection testConnection;
+
+ public Statement stmt;
+
+ private Connection createConnection() {
+ return BenchmarkUtils.getConnection(ip,type,user,password);
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ BenchmarkUtils.SetupTable(table);
+ update=BenchmarkUtils.updateBuilder;
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ try {
+ stmt = testConnection.createStatement();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ Boolean execute = null;
+ try {
+ execute = stmt.execute(update);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @TearDown(Level.Invocation)
+ public void doInvocationTearDown(){
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @Setup(Level.Iteration)
+ public void doSetup() {
+ Connection connection = createConnection();
+ setupCreateTables(connection);
+ //Setup connection
+ testConnection = createConnection();
+ setupCreateRows(testConnection,rows);
+ }
+
+ @TearDown(Level.Iteration)
+ public void doTearDown() {
+ System.out.println("Do TearDown");
+ try {
+ testConnection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc;
+
+import static org.onap.music.mdbc.BenchmarkUtils.setupCreateRows;
+import static org.onap.music.mdbc.BenchmarkUtils.setupCreateTables;
+
+import org.onap.music.mdbc.BenchmarkUtils.ExecutionType;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricStatementBenchmark {
+
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricBenchmark.class.getSimpleName())
+ .param("type", ExecutionType.METRIC.name())
+ .forks(1)
+ .threads(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public boolean testMethod(MyState state, Blackhole blackhole) {
+ Boolean execute = null;
+ try {
+ execute = state.stmt.execute(state.update);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ blackhole.consume(execute);
+ return execute;
+ }
+
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+
+ public String update;
+ final String user = "root";
+ final String password = "metriccluster";
+ @Param({"104.209.240.219"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ @Param({"MARIA_DB", "COCKROACH_DB", "METRIC","POSTGRES"})
+ public ExecutionType type;
+ @Param({"PERSONS"})
+ public String table;
+
+ public Connection testConnection;
+
+ public Statement stmt;
+
+ private Connection createConnection() {
+ return BenchmarkUtils.getConnection(ip,type,user,password);
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ BenchmarkUtils.SetupTable(table);
+ update=BenchmarkUtils.updateBuilder;
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ try {
+ stmt = testConnection.createStatement();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @TearDown(Level.Invocation)
+ public void doInvocationTearDown(){
+ try {
+ //TODO: check if state need to be consumed by blackhole to guarantee execution
+ testConnection.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ stmt.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @Setup(Level.Iteration)
+ public void doSetup() {
+ Connection connection = createConnection();
+ setupCreateTables(connection);
+ //Setup connection
+ testConnection = createConnection();
+ setupCreateRows(testConnection,rows);
+ }
+
+ @TearDown(Level.Iteration)
+ public void doTearDown() {
+ System.out.println("Do TearDown");
+ try {
+ testConnection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+ }
+}
-->
-<configuration scan="true" scanPeriod="3 seconds" debug="true">
+<configuration scan="false" scanPeriod="3 seconds" debug="true">
<!--
Logback files for the mdbc Driver "mdbc"
are created in directory ${catalina.base}/logs/mdbc;
--- /dev/null
+<!--
+Copyright (c) 2014, Oracle America, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ * Neither the name of Oracle nor the names of its contributors may be used
+ to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>mdbc</artifactId>
+ <groupId>org.onap.music.mdbc</groupId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>mdbc-own-benchmark</artifactId>
+ <packaging>jar</packaging>
+
+ <name>JMH benchmark sample: Java</name>
+
+ <!--
+ This is the demo/sample template build script for building Java benchmarks with JMH.
+ Edit as needed.
+ -->
+
+ <prerequisites>
+ <maven>3.0</maven>
+ </prerequisites>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>1.1.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.onap.music.mdbc</groupId>
+ <artifactId>mdbc-server</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.onap.music</groupId>
+ <artifactId>dev-MUSIC-cassandra</artifactId>
+ <version>3.2.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>3.3.0</version>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <jmh.version>1.9.3</jmh.version>
+ <javac.target>1.6</javac.target>
+ <uberjar.name>benchmarks</uberjar.name>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <compilerVersion>${javac.target}</compilerVersion>
+ <source>7</source>
+ <target>7</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${uberjar.name}</finalName>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <!--
+ Shading signed JARs will fail without this.
+ http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
+ -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.17</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+</project>
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.onap.music.mdbc.tables.MusicTxDigestId;
+import org.onap.music.mdbc.tables.StagingTable;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricAddTxDigestBenchmark {
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricAddTxDigestBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state, Blackhole blackhole) {
+ OwnUtils.addTransactionDigest(state.copy, state.copyTxDigestId,
+ (MusicMixin) state.getManager().getMusicInterface());
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ public MusicTxDigestId musicTxDigestId,copyTxDigestId;
+ public StagingTable copy,current;
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"127.0.0.1"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ private void assignManager() {
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ OwnUtils.dropAll(ip);
+ setupServer();
+ assignManager();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1);
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute(OwnUtils.UPDATE);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest());
+ final DatabasePartition partition = conn.getPartition();
+ OwnUtils.appendToRedo((MusicMixin)getManager().getMusicInterface(),partition.getMRIIndex(),partition.getLockId(),
+ musicTxDigestId);
+ current=conn.getTransactionDigest();
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ meta=null;
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ try {
+ copy = new StagingTable(current);
+ } catch (CloneNotSupportedException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ copyTxDigestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1);
+ }
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricCommitBenchmark {
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricCommitBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state, Blackhole blackhole) {
+ try {
+ state.conn.commit();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"104.209.240.219"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ private void assignManager() {
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ OwnUtils.dropAll(ip);
+ setupServer();
+ assignManager();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1);
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ meta=null;
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ Statement stmt = null;
+ try {
+ stmt = conn.createStatement();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ stmt.execute(OwnUtils.UPDATE);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.onap.music.mdbc.tables.MusicTxDigestId;
+import org.onap.music.mdbc.tables.StagingTable;
+import org.onap.music.mdbc.tables.TxCommitProgress;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricIsolatedMixinCommBenchmark {
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricIsolatedMixinCommBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state, Blackhole blackhole) {
+ try {
+ state.musicMixin.commitLog(state.conn.getPartition(),null,state.copy,
+ state.copyTxDigestId.transactionId.toString(),state.progressKeeper);
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ public TxCommitProgress progressKeeper;
+ public MusicTxDigestId musicTxDigestId,copyTxDigestId;
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"104.209.240.219"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+ public StagingTable copy,current;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ private void assignManager() {
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ OwnUtils.dropAll(ip);
+ setupServer();
+ assignManager();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1);
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+ Statement stmt = null;
+ try {
+ stmt = conn.createStatement();
+ stmt.execute(OwnUtils.UPDATE);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest());
+ current=conn.getTransactionDigest();
+ progressKeeper = new TxCommitProgress();
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ meta=null;
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ try {
+ copy = new StagingTable(current);
+ } catch (CloneNotSupportedException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ copyTxDigestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1);
+ progressKeeper.createNewTransactionTracker(copyTxDigestId.transactionId.toString(),conn);
+ }
+
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import java.sql.SQLException;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricOwnBenchmark{
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricOwnBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state, Blackhole blackhole) {
+ try {
+ state.conn.preStatementHook(state.update);
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ public String update="SELECT * FROM PERSONS;";
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"104.209.240.219"})
+ public String ip;
+ //Rows per transaction (e.g. size of each tx digest)
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ //Transaction before each ownership transition (e.g. size of redo log)
+ @Param({"1","10","100"})
+ public int updates;
+ //Number of ownership transitions before measurement (e.g. number of mri rows"
+ @Param({"1","10","100"})
+ public int transitions;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ OwnUtils.dropAll(ip);
+ setupServer();
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows, updates, transitions);
+ meta=null;
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ meta=null;
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ setupServer();
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+ try {
+ OwnUtils.dropAndCreateTable(conn.getDBInterface());
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @TearDown(Level.Invocation)
+ public void doInvocationTearDown(){
+ OwnUtils.deleteLastMriRow((MusicMixin) musicMixin);
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ getManager().closeConnection(id);
+ }
+
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.ExecutionInfo;
+import com.datastax.driver.core.QueryTrace;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.SimpleStatement;
+import org.onap.music.datastore.MusicDataStore;
+import org.onap.music.datastore.MusicDataStoreHandle;
+import org.onap.music.datastore.PreparedQueryObject;
+import org.onap.music.exceptions.MusicQueryException;
+import org.onap.music.exceptions.MusicServiceException;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.onap.music.mdbc.tables.MusicRangeInformationRow;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricPeekLockBenchmark {
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricPeekLockBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state, Blackhole blackhole) {
+ String table = state.table_prepend_name+OwnUtils.MRI_TABLE_NAME;
+ String selectQuery = "select * from "+OwnUtils.KEYSPACE+"."+table+" where key='"+state.key+"' LIMIT 1;";
+ PreparedQueryObject queryObject = new PreparedQueryObject();
+ queryObject.appendQueryString(selectQuery);
+ ResultSet results=null;
+ SimpleStatement statement = new SimpleStatement(queryObject.getQuery(), queryObject.getValues().toArray());
+ statement.setConsistencyLevel(ConsistencyLevel.ONE);
+ statement.enableTracing();
+ results = state.dsHandle.getSession().execute(statement);
+ blackhole.consume(results);
+ }
+
+ public void printInfo(ResultSet results){
+ ExecutionInfo executionInfo = results.getExecutionInfo();
+ System.out.println(executionInfo.getQueriedHost().getAddress());
+ System.out.println(executionInfo.getQueriedHost().getDatacenter());
+ System.out.println(executionInfo.getQueriedHost().getRack());
+ final QueryTrace trace = executionInfo.getQueryTrace();
+ System.out.printf(
+ "'%s' to %s took %dμs%n",
+ trace.getRequestType(), trace.getCoordinator(), trace.getDurationMicros());
+ for (QueryTrace.Event event : trace.getEvents()) {
+ System.out.printf(
+ " %d - %s - %s%n",
+ event.getSourceElapsedMicros(), event.getSource(), event.getDescription());
+ }
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ public MusicDataStore dsHandle;
+ public String key;
+ private String table_prepend_name = "lockQ_";
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"104.209.240.219"})
+ public String ip;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+ public MusicRangeInformationRow lastRow;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ private void assignManager() {
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ OwnUtils.dropAll(ip);
+ setupServer();
+ try {
+ dsHandle=MusicDataStoreHandle.getDSHandle();
+ } catch (MusicServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ assignManager();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, 1,1,1);
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+ lastRow = OwnUtils.getLastRow((MusicMixin) musicMixin);
+ key = lastRow.getPartitionIndex().toString();
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ System.exit(1);
+ }
+ meta=null;
+ }
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.onap.music.mdbc.tables.MusicTxDigestId;
+import org.onap.music.mdbc.tables.StagingTable;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricRedoLogBenchmark {
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricRedoLogBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ public void testMethod(MyState state, Blackhole blackhole) {
+ final DatabasePartition partition = state.conn.getPartition();
+ OwnUtils.appendToRedo((MusicMixin)state.getManager().getMusicInterface(),partition.getMRIIndex(),
+ partition.getLockId(),state.musicTxDigestId);
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ public MusicTxDigestId musicTxDigestId;
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"104.209.240.219"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ private void assignManager() {
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ OwnUtils.dropAll(ip);
+ setupServer();
+ assignManager();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,1,1);
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+ Statement stmt = null;
+ try {
+ stmt = conn.createStatement();
+ stmt.execute(OwnUtils.UPDATE);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest());
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ meta=null;
+ }
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+
+package org.onap.music.mdbc;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.exceptions.MusicLockingException;
+import org.onap.music.logging.EELFLoggerDelegate;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.onap.music.mdbc.tables.MusicTxDigestId;
+import org.onap.music.mdbc.tables.StagingTable;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@BenchmarkMode({Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MetricThreadJoinBenchmark {
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt = new OptionsBuilder()
+ .include(MetricThreadJoinBenchmark.class.getSimpleName())
+ .threads(1)
+ .forks(1)
+ .build();
+ new Runner(opt).run();
+ }
+
+ public void testMethod1(final MyState state) {
+ final String lockId = state.conn.getPartition().getLockId();
+ final UUID MRIIndex = state.conn.getPartition().getMRIIndex();
+ Thread t1=null;
+ Thread t2=null;
+ if(state.runTxDigest) {
+ final Runnable insertDigestCallable = new Runnable() {
+ @Override
+ public void run() {
+ OwnUtils.hardcodedAddtransaction(110);
+ }
+ };
+ t1 = new Thread(insertDigestCallable);
+ t1.start();
+ }
+ if(state.runRedo) {
+ final Runnable appendCallable = new Runnable() {
+ @Override
+ public void run() {
+ OwnUtils.hardcodedAppendToRedo(MRIIndex,lockId);
+ }
+ };
+ t2 = new Thread(appendCallable);
+ t2.start();
+ }
+
+ try {
+ if(state.runTxDigest) {
+ t1.join();
+ }
+ if(state.runRedo) {
+ t2.join();
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ @Benchmark
+ public void testMethod2(final MyState state) {
+ final String lockId = state.conn.getPartition().getLockId();
+ final UUID MRIIndex = state.conn.getPartition().getMRIIndex();
+ OwnUtils.hardcodedAddtransaction(110);
+ OwnUtils.hardcodedAppendToRedo(MRIIndex,lockId);
+ }
+
+ @State(Scope.Benchmark)
+ public static class MyState {
+ public MusicTxDigestId musicTxDigestId,copyTxDigestId;
+ public StagingTable copy,current;
+ final String user = OwnUtils.SQL_USER;
+ final String password = OwnUtils.SQL_PASSWORD;
+ public final Range range = new Range(OwnUtils.TABLE);
+ @Param({"127.0.0.1"})
+ public String ip;
+ @Param({"1", "10", "50", "80", "100", "200", "300", "400"})
+ public int rows;
+ @Param({"true","false"})
+ public boolean runRedo;
+ @Param({"true","false"})
+ public boolean runTxDigest;
+ private MusicInterface musicMixin = null;
+ private MdbcConnection conn;
+ private DBInterface dbMixin;
+ private MdbcServerLogic meta;
+ private String id;
+ public ExecutorService commitExecutorThreads;
+
+ private void setupServer(){
+ meta = OwnUtils.setupServer(user, password);
+ }
+
+ private StateManager getManager(){
+ return meta.getStateManager();
+ }
+
+ private void assignManager() {
+ StateManager manager = getManager();
+ musicMixin=manager.getMusicInterface();
+ }
+
+ @Setup(Level.Trial)
+ public void doTrialSetup(){
+ commitExecutorThreads = Executors.newFixedThreadPool(4);
+ OwnUtils.dropAll(ip);
+ setupServer();
+ assignManager();
+ OwnUtils.initMri((MusicMixin) musicMixin,range,meta, rows,0,0);
+ id = UUID.randomUUID().toString();
+ conn = (MdbcConnection) getManager().getConnection(id);
+ try {
+ Statement stmt = conn.createStatement();
+ stmt.execute(OwnUtils.UPDATE);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ musicTxDigestId = OwnUtils.setupCommit(conn.getPartition(), conn.getTransactionDigest());
+ current=conn.getTransactionDigest();
+ }
+
+ @TearDown(Level.Trial)
+ public void doTrialTearDown(){
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ meta=null;
+ commitExecutorThreads.shutdown();
+ }
+
+ @Setup(Level.Invocation)
+ public void doInvocationSetup(){
+ try {
+ copy = new StagingTable(current);
+ } catch (CloneNotSupportedException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ copyTxDigestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1);
+ }
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Cluster.Builder;
+import com.datastax.driver.core.Session;
+
+import java.nio.ByteBuffer;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.datastax.driver.core.SocketOptions;
+import org.onap.music.datastore.PreparedQueryObject;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.exceptions.MusicLockingException;
+import org.onap.music.exceptions.MusicServiceException;
+import org.onap.music.lockingservice.cassandra.MusicLockState;
+import org.onap.music.main.MusicCore;
+import org.onap.music.main.ResultType;
+import org.onap.music.main.ReturnType;
+import org.onap.music.mdbc.configurations.NodeConfiguration;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+import org.onap.music.mdbc.mixins.MusicMixin;
+import org.onap.music.mdbc.tables.MriRowComparator;
+import org.onap.music.mdbc.tables.MusicRangeInformationRow;
+import org.onap.music.mdbc.tables.MusicTxDigestId;
+import org.onap.music.mdbc.tables.StagingTable;
+
+public class OwnUtils {
+
+ public static final int SQL_PORT= 3306;
+
+ public static final String SQL_USER = "root";
+ public static final String SQL_PASSWORD = "metriccluster";
+ public static final String CASSANDRA_USER = "metric";
+ public static final String CASSANDRA_PASSWORD = "metriccluster";
+ public static final String KEYSPACE ="namespace";
+ public static final String MRI_TABLE_NAME = "musicrangeinformation";
+ public static final String MTD_TABLE_NAME = "musictxdigest";
+ public static final String MDBC_SERVER_NAME = "name";
+ public static final String DATABASE = "test";
+ public static final String TABLE= "PERSONS";
+ public static final int REPLICATION_FACTOR = 3;
+ public static final String SQL_URL="jdbc:mariadb://localhost:"+OwnUtils.SQL_PORT;
+ final static String CREATE_TABLE = "CREATE TABLE IF NOT EXISTS "+TABLE+" (\n" +
+ " PersonID int,\n" +
+ " Counter int,\n" +
+ " LastName varchar(255),\n" +
+ " FirstName varchar(255),\n" +
+ " Address varchar(255),\n" +
+ " City varchar(255),\n" +
+ " PRIMARY KEY(PersonID)"+
+ ");";
+
+ public static final String UPDATE = new StringBuilder()
+ .append("UPDATE PERSONS ")
+ .append("SET Counter = Counter + 1,")
+ .append("City = 'Sandy Springs'")
+ .append(";").toString();
+ public static final String PURGE = "PURGE BINARY LOGS BEFORE '";
+ public static final String DROP_TABLE = "DROP TABLE IF EXISTS " + TABLE + ";";
+ private static Lock sessionLock = new ReentrantLock();
+ private static Boolean sessionReady= false;
+ private static Cluster cluster;
+ private static Session session;
+
+ private static String getPurgeCommand(){
+ String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date());
+ return PURGE+timeStamp+"';";
+ }
+
+ public static Session getSession(String CASSANDRA_HOST){
+ sessionLock.lock();
+ try {
+ if(!sessionReady){
+ SocketOptions options = new SocketOptions();
+ options.setConnectTimeoutMillis(30000);
+ options.setReadTimeoutMillis(300000);
+ options.setTcpNoDelay(true);
+ final Builder builder = Cluster.builder();
+ builder.addContactPoint(CASSANDRA_HOST);
+ builder.withCredentials(CASSANDRA_USER, CASSANDRA_PASSWORD);
+ builder.withSocketOptions(options);
+ cluster = builder.build();
+ session=cluster.newSession();
+ }
+ return session;
+ // access the resource protected by this lock
+ } finally {
+ sessionLock.unlock();
+ }
+ }
+
+ public static Connection getConnection() throws ClassNotFoundException, SQLException {
+ Class.forName("org.mariadb.jdbc.Driver");
+ Properties connectionProps = new Properties();
+ connectionProps.put("user", SQL_USER);
+ connectionProps.put("password", SQL_PASSWORD);
+ Connection connection = DriverManager.getConnection(SQL_URL+"/"+DATABASE, connectionProps);
+ connection.setAutoCommit(false);
+ return connection;
+ }
+
+ public static void purgeBinLogs(Connection conn) throws SQLException{
+ final Statement dropStatement = conn.createStatement();
+ dropStatement.execute(getPurgeCommand());
+ dropStatement.close();
+ }
+
+ public static void dropTable(Connection conn) throws SQLException {
+ final Statement dropStatement = conn.createStatement();
+ dropStatement.execute(DROP_TABLE);
+ dropStatement.close();
+ }
+
+ private static void createTable(Connection conn) throws SQLException {
+ final Statement createStatement = conn.createStatement();
+ createStatement.execute(CREATE_TABLE);
+ createStatement.close();
+ }
+
+ public static void dropAndCreateTable(DBInterface dbMixin) throws SQLException {
+ dbMixin.dropSQLTriggers(TABLE);
+ dropTable(dbMixin.getSQLConnection());
+ createTable(dbMixin.getSQLConnection());
+ dbMixin.createSQLTriggers(TABLE);
+ }
+
+ public static void unlockRow(String keyspace, String mriTableName, DatabasePartition partition)
+ throws MusicLockingException {
+ String fullyQualifiedMriKey = keyspace+"."+ mriTableName+"."+partition.getMRIIndex().toString();
+ MusicLockState musicLockState = MusicCore.voluntaryReleaseLock(fullyQualifiedMriKey, partition.getLockId());
+ }
+
+ public static DatabasePartition createBasicRow(Range range, MusicInterface mixin, String mdbcServerName)
+ throws MDBCServiceException {
+ List<Range> ranges = new ArrayList<>();
+ ranges.add(range);
+ final UUID uuid = MDBCUtils.generateTimebasedUniqueKey();
+ DatabasePartition dbPartition = new DatabasePartition(ranges,uuid,null);
+ MusicRangeInformationRow newRow = new MusicRangeInformationRow(uuid,dbPartition, new ArrayList<MusicTxDigestId>(), "",
+ mdbcServerName, true);
+ DatabasePartition partition=mixin.createMusicRangeInformation(newRow);
+ return partition;
+ }
+
+ public static void initMriTable(MusicMixin musicMixin, Range range)
+ throws MDBCServiceException, SQLException, MusicLockingException {
+ final DatabasePartition partition = createBasicRow(range, musicMixin, MDBC_SERVER_NAME);
+ unlockRow(KEYSPACE,MRI_TABLE_NAME,partition);
+ }
+
+ public static MusicRangeInformationRow getLastRow(MusicMixin musicMixin){
+ List<MusicRangeInformationRow> allMriRows;
+ try {
+ allMriRows = musicMixin.getAllMriRows();
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ allMriRows=null;//Just to avoid IDE annoyance
+ }
+ Collections.sort(allMriRows, new MriRowComparator());
+ MusicRangeInformationRow musicRangeInformationRow = allMriRows.get(allMriRows.size() - 1);
+ return musicRangeInformationRow;
+ }
+
+ public static void deleteLastMriRow(MusicMixin musicMixin){
+ MusicRangeInformationRow musicRangeInformationRow = getLastRow(musicMixin);
+ try {
+ musicMixin.deleteMriRow(musicRangeInformationRow);
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ private static void changeRows(Connection conn) throws SQLException {
+ Statement stmt = conn.createStatement();
+ Boolean execute = stmt.execute(UPDATE);
+ conn.commit();
+ stmt.close();
+ }
+
+ private static void addRowsToTable(int totalNumberOfRows, Connection testConnection) throws SQLException {
+ for (int i = 0; i < totalNumberOfRows; i++) {
+ final StringBuilder insertSQLBuilder = new StringBuilder()
+ .append("INSERT INTO PERSONS VALUES (")
+ .append(i)
+ .append(", ")
+ .append(0)
+ .append(", '")
+ .append("Last-")
+ .append(i)
+ .append("', '")
+ .append("First-")
+ .append(i)
+ .append("', 'KACB', 'ATLANTA');");
+ Statement stmt = testConnection.createStatement();
+ Boolean execute = stmt.execute(insertSQLBuilder.toString());
+ stmt.close();
+ }
+ testConnection.commit();
+ }
+
+ public static void createHistory(MdbcServerLogic meta,int rows, int updates,int transitions)
+ throws SQLException {
+ final StateManager stateManager = meta.getStateManager();
+ String id = UUID.randomUUID().toString();
+ Connection connection = stateManager.getConnection(id);
+ createTable(connection);
+ addRowsToTable(rows, connection);
+ connection.close();
+ stateManager.closeConnection(id);
+ for(int mriRow=0;mriRow<transitions;mriRow++) {
+ final String finalId = UUID.randomUUID().toString();
+ connection = stateManager.getConnection(finalId);
+ for(int depth=0;depth<updates;depth++){
+ changeRows(connection);
+ }
+ connection.close();
+ stateManager.closeConnection(finalId);
+ }
+ }
+
+ public static MdbcServerLogic setupServer(String user, String password){
+ MdbcServerLogic meta;
+ NodeConfiguration config = new NodeConfiguration("","",null,OwnUtils.DATABASE,
+ OwnUtils.MDBC_SERVER_NAME);
+ //\TODO Add configuration file with Server Info
+ Properties connectionProps = new Properties();
+ connectionProps.setProperty("user", user);
+ connectionProps.setProperty("password", password);
+ connectionProps.setProperty(MusicMixin.KEY_MUSIC_RFACTOR,Integer.toString(OwnUtils.REPLICATION_FACTOR));
+ connectionProps.setProperty(MusicMixin.KEY_MUSIC_NAMESPACE,OwnUtils.KEYSPACE);
+ try {
+ meta = new MdbcServerLogic(OwnUtils.SQL_URL,connectionProps,config);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ meta=null;
+ System.exit(1);
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ meta=null;
+ System.exit(1);
+ }
+ return meta;
+ }
+
+ public static void dropAll(String ip){
+ Session session=OwnUtils.getSession(ip);
+ session.execute("DROP KEYSPACE IF EXISTS "+OwnUtils.KEYSPACE+";");
+ try {
+ Connection connection = OwnUtils.getConnection();
+ OwnUtils.dropTable(connection);
+ connection.close();
+ } catch (SQLException|ClassNotFoundException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static void initMri(MusicMixin musicMixin,Range range,MdbcServerLogic meta, int rows, int updates, int transitions){
+ try {
+ OwnUtils.initMriTable(musicMixin,range);
+ } catch (MusicLockingException|SQLException|MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ try {
+ OwnUtils.createHistory(meta, rows, updates, transitions);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static void hardcodedAddtransaction(int size){
+ final UUID uuid = MDBCUtils.generateTimebasedUniqueKey();
+ ByteBuffer serializedTransactionDigest = ByteBuffer.allocate(size);
+ for(int i=0;i<size;i++){
+ serializedTransactionDigest.put((byte)i);
+ }
+ PreparedQueryObject query = new PreparedQueryObject();
+ String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest,compressed ) VALUES (?,?,?);",KEYSPACE,
+ MTD_TABLE_NAME);
+ query.appendQueryString(cql);
+ query.addValue(uuid);
+ query.addValue(serializedTransactionDigest);
+ query.addValue(false);
+ //\TODO check if I am not shooting on my own foot
+ try {
+ MusicCore.nonKeyRelatedPut(query,"critical");
+ } catch (MusicServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static void hardcodedAppendToRedo(UUID mriId, String lockId){
+ final UUID uuid = MDBCUtils.generateTimebasedUniqueKey();
+ PreparedQueryObject query = new PreparedQueryObject();
+ StringBuilder appendBuilder = new StringBuilder();
+ appendBuilder.append("UPDATE ")
+ .append(KEYSPACE)
+ .append(".")
+ .append(MRI_TABLE_NAME)
+ .append(" SET txredolog = txredolog +[('")
+ .append(MTD_TABLE_NAME)
+ .append("',")
+ .append(uuid)
+ .append(")] WHERE rangeid = ")
+ .append(mriId)
+ .append(";");
+ query.appendQueryString(appendBuilder.toString());
+ ReturnType returnType = MusicCore.criticalPut(KEYSPACE, MRI_TABLE_NAME, mriId.toString(),
+ query, lockId, null);
+ //returnType.getExecutionInfo()
+ if (returnType.getResult().compareTo(ResultType.SUCCESS) != 0) {
+ System.exit(1);
+ }
+ }
+
+ public static void addTransactionDigest(StagingTable transactionDigest, MusicTxDigestId digestId, MusicMixin music){
+ try {
+ music.createAndAddTxDigest(transactionDigest, digestId.transactionId);
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static void appendToRedo(MusicMixin music, UUID MRIIndex, String lockId, MusicTxDigestId digestId){
+ try {
+ music.appendToRedoLog(KEYSPACE,MRIIndex,digestId.transactionId,lockId,OwnUtils.MTD_TABLE_NAME,
+ OwnUtils.MRI_TABLE_NAME);
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ public static MusicTxDigestId setupCommit(DatabasePartition partition, StagingTable transactionDigest){
+ UUID mriIndex = partition.getMRIIndex();
+
+ if(transactionDigest == null || transactionDigest.isEmpty()) {
+ System.err.println("Transaction digest is empty");
+ System.exit(1);
+ }
+
+ MusicTxDigestId digestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1);
+ return digestId;
+ }
+}
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ============LICENSE_START==========================================
+ mdbc
+ ===================================================================
+ Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ ===================================================================
+
+ Unless otherwise specified, all software contained herein is licensed
+ under the Apache License, Version 2.0 (the “License”);
+ you may not use this software except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Unless otherwise specified, all documentation contained herein is licensed
+ under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+ you may not use this documentation except in compliance with the License.
+ You may obtain a copy of the License at
+
+ https://creativecommons.org/licenses/by/4.0/
+
+ Unless required by applicable law or agreed to in writing, documentation
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ ============LICENSE_END============================================
+
+
+ -->
+
+<configuration scan="false" scanPeriod="3 seconds" debug="true">
+ <!--
+ Logback files for the mdbc Driver "mdbc"
+ are created in directory ${catalina.base}/logs/mdbc;
+ e.g., apache-tomcat-8.0.35/logs/mdbc/application.log
+ -->
+ <!--<jmxConfigurator /> -->
+
+ <!-- specify the component name -->
+ <property name="catalina.home" value="/var/log/metric/"/>
+ <property name="componentName" value="mdbc"></property>
+
+ <!-- specify the base path of the log directory -->
+ <property name="logDirPrefix" value="${catalina.base}/logs"></property>
+
+ <!-- The directories where logs are written -->
+ <property name="logDirectory" value="${logDirPrefix}/${componentName}"/>
+ <!-- Can easily relocate debug logs by modifying this path. -->
+ <property name="debugLogDirectory" value="${logDirPrefix}/${componentName}"/>
+
+ <!-- log file names -->
+ <property name="generalLogName" value="application"/>
+ <property name="errorLogName" value="error"/>
+ <property name="metricsLogName" value="metrics"/>
+ <property name="auditLogName" value="audit"/>
+ <property name="debugLogName" value="debug"/>
+ <!--
+ These loggers are not used in code (yet).
+ <property name="securityLogName" value="security" />
+ <property name="policyLogName" value="policy" />
+ <property name="performanceLogName" value="performance" />
+ <property name="serverLogName" value="server" />
+ -->
+
+ <!-- 1610 Logging Fields Format Revisions -->
+ <property name="auditLoggerPattern"
+ value="%X{AuditLogBeginTimestamp}|%X{AuditLogEndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{ClientIPAddress}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| %msg%n"/>
+
+ <property name="metricsLoggerPattern"
+ value="%X{MetricsLogBeginTimestamp}|%X{MetricsLogEndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{ClientIPAddress}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{TargetVisualEntity}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| %msg%n"/>
+
+ <property name="errorLoggerPattern"
+ value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{ClassName}|%X{AlertSeverity}|%X{ErrorCode}|%X{ErrorDescription}| %msg%n"/>
+
+ <property name="defaultLoggerPattern"
+ value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ClassName}| %msg%n"/>
+
+ <!-- use %class so library logging calls yield their class name -->
+ <property name="applicationLoggerPattern"
+ value="%date{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%class{36}| %msg%n"/>
+
+ <!-- Example evaluator filter applied against console appender -->
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>${defaultLoggerPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <!-- ============================================================================ -->
+ <!-- EELF Appenders -->
+ <!-- ============================================================================ -->
+
+ <!-- The EELFAppender is used to record events to the general application
+ log -->
+
+
+ <appender name="EELF"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${generalLogName}.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!-- daily rollover -->
+ <fileNamePattern>${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern>
+
+ <!-- keep 30 days' worth of history capped at 3GB total size -->
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>3GB</totalSizeCap>
+
+ </rollingPolicy>
+ <encoder>
+ <pattern>${applicationLoggerPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <!-- Class name is part of caller data -->
+ <includeCallerData>true</includeCallerData>
+ <appender-ref ref="EELF"/>
+ </appender>
+
+ <!-- EELF Security Appender. This appender is used to record security events
+ to the security log file. Security events are separate from other loggers
+ in EELF so that security log records can be captured and managed in a secure
+ way separate from the other logs. This appender is set to never discard any
+ events. -->
+ <!--
+ <appender name="EELFSecurity"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${securityLogName}.log</file>
+ <rollingPolicy
+ class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${logDirectory}/${securityLogName}.%i.log.zip
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>5MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>${defaultPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="asyncEELFSecurity" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <discardingThreshold>0</discardingThreshold>
+ <appender-ref ref="EELFSecurity" />
+ </appender>
+ -->
+
+ <!-- EELF Performance Appender. This appender is used to record performance
+ records. -->
+ <!--
+ <appender name="EELFPerformance"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${performanceLogName}.log</file>
+ <rollingPolicy
+ class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${logDirectory}/${performanceLogName}.%i.log.zip
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>5MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <outputPatternAsHeader>true</outputPatternAsHeader>
+ <pattern>${defaultPattern}</pattern>
+ </encoder>
+ </appender>
+ <appender name="asyncEELFPerformance" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFPerformance" />
+ </appender>
+ -->
+
+ <!-- EELF Server Appender. This appender is used to record Server related
+ logging events. The Server logger and appender are specializations of the
+ EELF application root logger and appender. This can be used to segregate Server
+ events from other components, or it can be eliminated to record these events
+ as part of the application root log. -->
+ <!--
+ <appender name="EELFServer"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${serverLogName}.log</file>
+ <rollingPolicy
+ class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${logDirectory}/${serverLogName}.%i.log.zip
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>5MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>${defaultPattern}</pattern>
+ </encoder>
+ </appender>
+ <appender name="asyncEELFServer" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFServer" />
+ </appender>
+ -->
+
+ <!-- EELF Policy Appender. This appender is used to record Policy engine
+ related logging events. The Policy logger and appender are specializations
+ of the EELF application root logger and appender. This can be used to segregate
+ Policy engine events from other components, or it can be eliminated to record
+ these events as part of the application root log. -->
+ <!--
+ <appender name="EELFPolicy"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${policyLogName}.log</file>
+ <rollingPolicy
+ class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${logDirectory}/${policyLogName}.%i.log.zip
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>5MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>${defaultPattern}</pattern>
+ </encoder>
+ </appender>
+ <appender name="asyncEELFPolicy" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFPolicy" />
+ </appender>
+ -->
+
+ <!-- EELF Audit Appender. This appender is used to record audit engine
+ related logging events. The audit logger and appender are specializations
+ of the EELF application root logger and appender. This can be used to segregate
+ Policy engine events from other components, or it can be eliminated to record
+ these events as part of the application root log. -->
+
+ <appender name="EELFAudit"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${auditLogName}.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!-- daily rollover -->
+ <fileNamePattern>${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern>
+
+ <!-- keep 30 days' worth of history capped at 3GB total size -->
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>3GB</totalSizeCap>
+
+ </rollingPolicy>
+ <encoder>
+ <pattern>${auditLoggerPattern}</pattern>
+ </encoder>
+ </appender>
+ <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFAudit"/>
+ </appender>
+
+ <appender name="EELFMetrics"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${metricsLogName}.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!-- daily rollover -->
+ <fileNamePattern>${logDirectory}/${metricsLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern>
+
+ <!-- keep 30 days' worth of history capped at 3GB total size -->
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>3GB</totalSizeCap>
+
+ </rollingPolicy>
+ <encoder>
+ <pattern>${metricsLoggerPattern}</pattern>
+ </encoder>
+ </appender>
+
+
+ <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFMetrics"/>
+ </appender>
+
+ <appender name="EELFError"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${errorLogName}.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!-- daily rollover -->
+ <fileNamePattern>${logDirectory}/${errorLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern>
+
+ <!-- keep 30 days' worth of history capped at 3GB total size -->
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>3GB</totalSizeCap>
+
+ </rollingPolicy>
+ <encoder>
+ <pattern>${errorLoggerPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="asyncEELFError" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFError"/>
+ </appender>
+
+ <appender name="EELFDebug"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${debugLogDirectory}/${debugLogName}.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!-- daily rollover -->
+ <fileNamePattern>${logDirectory}/${debugLogName}.%d{yyyy-MM-dd}.log.zip</fileNamePattern>
+
+ <!-- keep 30 days' worth of history capped at 3GB total size -->
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>3GB</totalSizeCap>
+
+ </rollingPolicy>
+ <encoder>
+ <pattern>${defaultLoggerPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="EELFDebug"/>
+ </appender>
+
+
+ <logger name="com.att.eelf" level="error" additivity="false">
+ <appender-ref ref="asyncEELF"/>
+ </logger>
+
+ <logger name="com.att.eelf" level="error" additivity="false">
+ <appender-ref ref="asyncEELFAudit"/>
+ </logger>
+
+ <logger name="com.att.eelf" level="error" additivity="false">
+ <appender-ref ref="asyncEELFDebug"/>
+ </logger>
+
+ <logger name="com.att.eelf.error" level="error" additivity="false">
+ <appender-ref ref="asyncEELFError"/>
+ </logger>
+
+ <logger name="com.att.eelf.metrics" level="error" additivity="false">
+ <appender-ref ref="asyncEELFMetrics"/>
+ </logger>
+
+ <root level="ERROR">
+ <appender-ref ref="asyncEELF"/>
+ </root>
+
+</configuration>
--- /dev/null
+#
+# A list of all Mixins that should be checked by MDBC
+#
+MIXINS= \
+ org.onap.music.mdbc.mixins.MySQLMixin \
+ org.onap.music.mdbc.mixins.MusicMixin \
+ org.onap.music.mdbc.mixins.Music2Mixin
+
+DEFAULT_DRIVERS=\
+ org.h2.Driver \
+ com.mysql.jdbc.Driver
+
+txdaemonsleeps=15
\ No newline at end of file
--- /dev/null
+cassandra.host =\
+ 10.0.0.5
+cassandra.user =\
+ metric
+cassandra.password =\
+ metriccluster
+zookeeper.host =\
+ localhost
</parent>
<dependencies>
- <dependency>
- <groupId>org.apache.calcite</groupId>
-
<artifactId>calcite</artifactId>
- <version>1.11.0</version>
- <type>pom</type>
- </dependency>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite</artifactId>
+ <version>1.11.0</version>
+ <type>pom</type>
+ </dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
- <!-- The oparent dependency managed version 26.0-jre will not work -->
- <!-- 19.0 is needed by cassandra-driver-core 3.3.0 -->
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>19.0</version>
+ <!-- The oparent dependency managed version 26.0-jre will not work -->
+ <!-- 19.0 is needed by cassandra-driver-core 3.3.0 -->
+ <groupId>com.google.guava</groupId>
+
<artifactId>guava</artifactId>
+ <version>19.0</version>
</dependency>
<dependency>
- <!-- The oparent dependency managed version 9.4.12.v20180830 will not work -->
- <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-http</artifactId>
- <version>9.2.19.v20160908</version>
+ <!-- The oparent dependency managed version 9.4.12.v20180830 will not work -->
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+
<artifactId>jetty-http</artifactId>
+ <version>9.2.19.v20160908</version>
</dependency>
<dependency>
- <!-- The oparent dependency managed version 9.4.12.v20180830 will not work -->
- <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <version>9.2.19.v20160908</version>
+ <!-- The oparent dependency managed version 9.4.12.v20180830 will not work -->
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+
<artifactId>jetty-server</artifactId>
+ <version>9.2.19.v20160908</version>
</dependency>
<dependency>
- <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-security</artifactId>
- <version>9.2.19.v20160908</version>
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+
<artifactId>jetty-security</artifactId>
+ <version>9.2.19.v20160908</version>
</dependency>
<dependency>
- <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- <version>9.2.19.v20160908</version>
+ <!-- 9.2.19.v20160908 is needed by avatica-server 1.12.0 -->
+ <groupId>org.eclipse.jetty</groupId>
+
<artifactId>jetty-util</artifactId>
+ <version>9.2.19.v20160908</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<version>20160810</version>
</dependency>
<dependency>
- <groupId>org.mariadb.jdbc</groupId>
-
<artifactId>mariadb-java-client</artifactId>
- <version>2.3.0</version>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
</configuration>
<executions>
<execution>
- <id>make-assembly</id> <!-- this is used for inheritance merges -->
- <phase>package</phase> <!-- bind to the packaging phase -->
- <goals>
- <goal>single</goal>
- </goals>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+
<phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
</execution>
<execution>
<id>node-configuration</id>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
+ <execution>
+ <id>cluster-setup</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.onap.music.mdbc.tools.ClusterSetup</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <finalName>cluster-setup</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ </execution>
<execution>
<id>mdbc-server</id>
<phase>package</phase>
import org.onap.music.mdbc.ownership.DagNode;
import org.onap.music.mdbc.ownership.OwnershipAndCheckpoint;
import org.onap.music.mdbc.query.QueryProcessor;
-import org.onap.music.mdbc.tables.MusicTxDigest;
+import org.onap.music.mdbc.tables.MusicTxDigestDaemon;
import org.onap.music.mdbc.tables.MusicRangeInformationRow;
import org.onap.music.mdbc.tables.StagingTable;
import org.onap.music.mdbc.tables.TxCommitProgress;
logger.debug("Mdbc connection created with id: "+id);
}
+ public DBInterface getDatabaseInterface(){
+ return this.dbi;
+ }
+
@Override
public <T> T unwrap(Class<T> iface) throws SQLException {
logger.error(EELFLoggerDelegate.errorLogger, "proxyconn unwrap: " + iface.getName());
jdbcConn.close();
logger.debug("Connection was closed for id:" + id);
}
+ try {
+ mi.relinquish(partition.getLockId(),partition.getMRIIndex().toString());
+ } catch (MDBCServiceException e) {
+ throw new SQLException("Failure during relinquish of partition",e);
+ }
}
@Override
this.partition.updateDatabasePartition(tempPartition);
statemanager.getOwnAndCheck().reloadAlreadyApplied(this.partition);
}
- dbi.preStatementHook(sql);
+ dbi.preStatementHook(sql);
}
mi.relinquishIfRequired(partition);
}
+ public Connection getConnection(){
+ return jdbcConn;
+ }
+
+ public DatabasePartition getPartition() {
+ return partition;
+ }
+
+ public StagingTable getTransactionDigest(){
+ return transactionDigest;
+ }
+
}
package org.onap.music.mdbc;
import org.onap.music.mdbc.configurations.NodeConfiguration;
-import org.onap.music.mdbc.tables.MusicTxDigest;
+import org.onap.music.mdbc.tables.MusicTxDigestDaemon;
import org.apache.calcite.avatica.remote.Driver.Serialization;
import org.apache.calcite.avatica.remote.LocalService;
import org.apache.calcite.avatica.server.HttpServer;
import org.onap.music.exceptions.MDBCServiceException;
import org.onap.music.mdbc.configurations.NodeConfiguration;
-import org.onap.music.mdbc.tables.MusicTxDigest;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
* In the future we may decide to partition ranges differently
* @author Enrique Saurez
*/
-public class Range implements Serializable, Cloneable{
-
- private static final long serialVersionUID = 1610744496930800088L;
+public class Range implements Cloneable{
private String table;
}
@Override
- protected Range clone() {
+ public Range clone() {
Range newRange = null;
try{
newRange = (Range) super.clone();
import java.util.List;
import java.util.Set;
-import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.tuple.Pair;
import org.onap.music.exceptions.MDBCServiceException;
import org.onap.music.logging.EELFLoggerDelegate;
import org.onap.music.mdbc.mixins.MusicInterface.OwnershipReturn;
import org.onap.music.mdbc.ownership.OwnershipAndCheckpoint;
import org.onap.music.mdbc.tables.MriReference;
-import org.onap.music.mdbc.tables.MusicTxDigest;
+import org.onap.music.mdbc.tables.MusicTxDigestDaemon;
import org.onap.music.mdbc.tables.TxCommitProgress;
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/** map of transactions that have already been applied/updated in this sites SQL db */
private Map<Range, Pair<MriReference, Integer>> alreadyApplied;
private OwnershipAndCheckpoint ownAndCheck;
+ private Thread txDaemon ;
/**
* For testing purposes only
musicmixin = info.getProperty(Configuration.KEY_MUSIC_MIXIN_NAME, Configuration.MUSIC_MIXIN_DEFAULT);
initMusic();
- initSqlDatabase();
-
+ initSqlDatabase();
+ initTxDaemonThread();
String t = info.getProperty(Configuration.KEY_OWNERSHIP_TIMEOUT);
- long timeoutMs = (t == null) ? Configuration.DEFAULT_OWNERSHIP_TIMEOUT : Integer.parseInt(t);
+ long timeout = (t == null) ? Configuration.DEFAULT_OWNERSHIP_TIMEOUT : Integer.parseInt(t);
alreadyApplied = new ConcurrentHashMap<>();
- ownAndCheck = new OwnershipAndCheckpoint(alreadyApplied, timeoutMs);
-
- rangesToWarmup = initWarmupRanges();
- logger.info("Warmup ranges for this site is " + rangesToWarmup);
+ ownAndCheck = new OwnershipAndCheckpoint(alreadyApplied, timeout);
+ }
- MusicTxDigest txDaemon = new MusicTxDigest(this);
- txDaemon.startBackgroundDaemon(Integer.parseInt(
- info.getProperty(Configuration.TX_DAEMON_SLEEPTIME_S, Configuration.TX_DAEMON_SLEEPTIME_S_DEFAULT)));
+ protected void initTxDaemonThread(){
+ txDaemon = new Thread(
+ new MusicTxDigestDaemon(Integer.parseInt(
+ info.getProperty(Configuration.TX_DAEMON_SLEEPTIME_S, Configuration.TX_DAEMON_SLEEPTIME_S_DEFAULT)),
+ this));
+ txDaemon.setName("TxDaemon");
+ txDaemon.setDaemon(true);
+ txDaemon.start();
}
/**
.append(";");
Statement stmt = sqlConnection.createStatement();
stmt.execute(sql.toString());
+ sqlConnection.close();
} catch (SQLException e) {
logger.error(EELFLoggerDelegate.errorLogger, e.getMessage(),AppMessages.UNKNOWNERROR, ErrorSeverity.CRITICAL,
ErrorTypes.GENERALSERVICEERROR);
returnArray = new ArrayList<>(eventualRanges);
}
else{
- returnArray= null;
+ returnArray= new ArrayList<>();
}
}
finally{
}
if(connectionRanges.containsKey(connectionId)){
//We relinquish all locks obtained by a given
- relinquish(connectionRanges.get(connectionId));
+ //relinquish(connectionRanges.get(connectionId));
connectionRanges.remove(connectionId);
}
}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc.configurations;
+
+import com.google.gson.Gson;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import org.onap.music.datastore.PreparedQueryObject;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.exceptions.MusicServiceException;
+import org.onap.music.logging.EELFLoggerDelegate;
+import org.onap.music.main.MusicCore;
+import org.onap.music.mdbc.mixins.MusicMixin;
+
+public class ClusterConfiguration {
+ private String internalNamespace;
+ private int internalReplicationFactor;
+ private String musicNamespace;
+ private int musicReplicationFactor;
+ private String mriTableName;
+ private String mtxdTableName;
+ private String eventualMtxdTableName;
+ private String nodeInfoTableName;
+ private String rangeDependencyTableName;
+
+ private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(ClusterConfiguration.class);
+
+ public void initNamespaces() throws MDBCServiceException{
+ MusicMixin.createKeyspace(internalNamespace,internalReplicationFactor);
+ MusicMixin.createKeyspace(musicNamespace,musicReplicationFactor);
+ }
+
+ public void initTables() throws MDBCServiceException{
+ MusicMixin.createMusicRangeInformationTable(musicNamespace, mriTableName);
+ MusicMixin.createMusicTxDigest(mtxdTableName,musicNamespace, -1);
+ MusicMixin.createMusicEventualTxDigest(eventualMtxdTableName,musicNamespace, -1);
+ MusicMixin.createMusicNodeInfoTable(nodeInfoTableName,musicNamespace,-1);
+ MusicMixin.createMusicRangeDependencyTable(musicNamespace,rangeDependencyTableName);
+ }
+
+ private void initInternalTable() throws MDBCServiceException {
+ StringBuilder createKeysTableCql = new StringBuilder("CREATE TABLE IF NOT EXISTS ")
+ .append(internalNamespace)
+ .append(".unsynced_keys (key text PRIMARY KEY);");
+ PreparedQueryObject queryObject = new PreparedQueryObject();
+ queryObject.appendQueryString(createKeysTableCql.toString());
+ try {
+ MusicCore.createTable(internalNamespace,"unsynced_keys", queryObject,"critical");
+ } catch (MusicServiceException e) {
+ logger.error("Error creating unsynced keys table" );
+ throw new MDBCServiceException("Error creating unsynced keys table", e);
+ }
+ }
+
+ public static ClusterConfiguration readJsonFromFile(String filepath) throws FileNotFoundException {
+ BufferedReader br;
+ try {
+ br = new BufferedReader(
+ new FileReader(filepath));
+ } catch (FileNotFoundException e) {
+ logger.error(EELFLoggerDelegate.errorLogger,"File was not found when reading json"+e);
+ throw e;
+ }
+ Gson gson = new Gson();
+ ClusterConfiguration config = gson.fromJson(br, ClusterConfiguration.class);
+ return config;
+ }
+
+
+}
}
protected List<Range> toRanges(String tables){
+ if(tables.isEmpty()){
+ return new ArrayList<>();
+ }
List<Range> newRange = new ArrayList<>();
String[] tablesArray=tables.split(",");
for(String table: tablesArray) {
package org.onap.music.mdbc.configurations;
import com.datastax.driver.core.ResultSet;
-import java.util.stream.Collectors;
import org.onap.music.exceptions.MDBCServiceException;
import org.onap.music.logging.EELFLoggerDelegate;
import org.onap.music.mdbc.MDBCUtils;
import org.onap.music.mdbc.Range;
-import org.onap.music.mdbc.RedoRow;
import org.onap.music.mdbc.mixins.MusicMixin;
-import org.onap.music.mdbc.tables.MusicTxDigest;
+import org.onap.music.mdbc.tables.MusicRangeInformationRow;
import com.google.gson.Gson;
import org.onap.music.datastore.PreparedQueryObject;
private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(TablesConfiguration.class);
private List<PartitionInformation> partitions;
- private String internalNamespace;
- private int internalReplicationFactor;
+ String tableToPartitionName;
private String musicNamespace;
- private int musicReplicationFactor;
- private String tableToPartitionName;
private String partitionInformationTableName;
private String redoHistoryTableName;
private String sqlDatabaseName;
*/
public List<NodeConfiguration> initializeAndCreateNodeConfigurations() throws MDBCServiceException {
logger.info("initializing the required spaces");
- createKeyspaces();
- initInternalNamespace();
List<NodeConfiguration> nodeConfigs = new ArrayList<>();
if(partitions == null){
}
for(PartitionInformation partitionInfo : partitions){
String mriTableName = partitionInfo.mriTableName;
- checkIfMriIsEmpty(mriTableName);
- //0) Create the corresponding Music Range Information table
- MusicMixin.createMusicRangeInformationTable(musicNamespace,mriTableName);
-
+ checkIfMriDoesNotExists(mriTableName,partitionInfo);
String partitionId;
if(partitionInfo.partitionId==null || partitionInfo.partitionId.isEmpty()){
//1) Create a row in the partition info table
return nodeConfigs;
}
- private void createKeyspaces() throws MDBCServiceException {
- MusicMixin.createKeyspace(internalNamespace,internalReplicationFactor);
- MusicMixin.createKeyspace(musicNamespace,musicReplicationFactor);
-
- }
-
- private void checkIfMriIsEmpty(String mriTableName) throws MDBCServiceException {
- //First check if table exists
- StringBuilder checkTableExistsString = new StringBuilder("SELECT table_name FROM system_schema.tables WHERE keyspace_name='")
- .append(musicNamespace)
- .append("';");
- PreparedQueryObject checkTableExists = new PreparedQueryObject();
- checkTableExists.appendQueryString(checkTableExistsString.toString());
- final ResultSet resultSet = MusicCore.quorumGet(checkTableExists);
- if(resultSet.isExhausted()){
- //Table doesn't exist
- return;
- }
+ private void checkIfMriDoesNotExists(String mriTableName, PartitionInformation partition) throws MDBCServiceException {
//If exists, check if empty
StringBuilder checkRowsInTableString = new StringBuilder("SELECT * FROM ")
.append(musicNamespace)
.append("';");
PreparedQueryObject checkRowsInTable = new PreparedQueryObject();
checkRowsInTable.appendQueryString(checkRowsInTableString.toString());
- final ResultSet resultSet2 = MusicCore.quorumGet(checkTableExists);
- if(!resultSet2.isExhausted()) {
- throw new MDBCServiceException("When initializing the configuration of the system, the MRI should not exits "
- + "be empty");
+ final ResultSet resultSet = MusicCore.quorumGet(checkRowsInTable);
+ while(resultSet!=null && !resultSet.isExhausted()){
+ final MusicRangeInformationRow mriRowFromCassandraRow = MusicMixin.getMRIRowFromCassandraRow(resultSet.one());
+ List<Range> ranges = mriRowFromCassandraRow.getDBPartition().getSnapshot();
+ for(Range range: partition.getTables()) {
+ if (Range.overlaps(ranges,range.getTable())){
+ throw new MDBCServiceException("MRI row already exists");
+ }
+ }
}
}
- private void initInternalNamespace() throws MDBCServiceException {
- StringBuilder createKeysTableCql = new StringBuilder("CREATE TABLE IF NOT EXISTS ")
- .append(internalNamespace)
- .append(".unsynced_keys (key text PRIMARY KEY);");
- PreparedQueryObject queryObject = new PreparedQueryObject();
- queryObject.appendQueryString(createKeysTableCql.toString());
- try {
- MusicCore.createTable(internalNamespace,"unsynced_keys", queryObject,"critical");
- } catch (MusicServiceException e) {
- logger.error("Error creating unsynced keys table" );
- throw new MDBCServiceException("Error creating unsynced keys table", e);
- }
- }
public static TablesConfiguration readJsonFromFile(String filepath) throws FileNotFoundException {
BufferedReader br;
private List<Range> tables;
private String owner;
private String mriTableName;
- private String mtxdTableName;
private String partitionId;
public List<Range> getTables() {
this.partitionId = partitionId;
}
- public String getMtxdTableName(){
- return mtxdTableName;
- }
-
- public void setMtxdTableName(String mtxdTableName) {
- this.mtxdTableName = mtxdTableName;
- }
}
}
}\r
Connection connection;\r
try { \r
- String metricURL = "http://localhost:300000/test"; \r
- if(args[0] != null) { \r
+ String metricURL = "http://localhost:30000/test"; \r
+ if (args.length>0 && args[0] != null) { \r
metricURL = args[0]; \r
} \r
connection = DriverManager.getConnection("jdbc:avatica:remote:url=" + metricURL+ ";serialization=protobuf"); \r
e.printStackTrace(); \r
return; \r
}\r
-\r
try {\r
connection.setAutoCommit(false);\r
} catch (SQLException e) {\r
return;\r
}\r
\r
-\r
final String sql = "CREATE TABLE IF NOT EXISTS Persons (\n" +\r
" PersonID int,\n" +\r
" LastName varchar(255),\n" +\r
e.printStackTrace();\r
return;\r
}\r
-\r
boolean execute = true;\r
// try {\r
// execute = stmt.execute(sql);\r
}\r
\r
try {\r
- //execute = insertStmt.execute(insertSQL);\r
+ execute = insertStmt.execute(insertSQL);\r
execute = insertStmt.execute(insertSQL1);\r
- //execute = insertStmt.execute(insertSQL2);\r
+ execute = insertStmt.execute(insertSQL2);\r
//execute = insertStmt.execute(insertSQL3);\r
//execute = insertStmt.execute(insertSQL4);\r
\r
*/
package org.onap.music.mdbc.mixins;
+import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
void enableForeignKeyChecks() throws SQLException;
void applyTxDigest(StagingTable txDigest, List<Range> ranges) throws SQLException, MDBCServiceException;
+
+ Connection getSQLConnection();
}
import com.datastax.driver.core.ResultSet;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
+
import org.json.JSONObject;
import org.onap.music.exceptions.MDBCServiceException;
import org.onap.music.exceptions.MusicLockingException;
/**
* This function is used to append an index to the redo log in a MRI row
- * @param partition information related to ownership of partitions, used to verify ownership
+ * @param MRIIndex index of the row to which the record is going to be added (obtained from the Partition)
+ * @param lockId reference to lock associated to the row in the MRI table MRIIndex.
* @param newRecord index of the new record to be appended to the redo log
* @throws MDBCServiceException
*/
- void appendToRedoLog( DatabasePartition partition, MusicTxDigestId newRecord) throws MDBCServiceException;
-
+ void appendToRedoLog(UUID MRIIndex, String lockId, MusicTxDigestId newRecord)throws MDBCServiceException;
/**
* This functions adds the tx digest to
* @param newId id used as index in the MTD table
* @throws MDBCServiceException
*/
- public LinkedHashMap<UUID, StagingTable> getEveTxDigest(String nodeName) throws MDBCServiceException;
+ LinkedHashMap<UUID, StagingTable> getEveTxDigest(String nodeName) throws MDBCServiceException;
/**
* Function used to retrieve a given transaction digest and deserialize it
* @param id of the transaction digest to be retrieved
*/
void relinquishIfRequired(DatabasePartition partition) throws MDBCServiceException;
- /**
- * This function is in charge of owning all the ranges requested and creating a new row that show the ownership of all
- * those ranges.
- * @param rangeId new id to be used in the new row
- * @param ranges ranges to be owned by the end of the function called
- * @param partition current ownership status
- * @return
- * @throws MDBCServiceException
- */
- //OwnershipReturn appendRange(String rangeId, List<Range> ranges, DatabasePartition partition) throws MDBCServiceException;
-
/**
* This functions relinquishes a range
* @param ownerId id of the current ownerh
List<MusicRangeInformationRow> getAllMriRows() throws MDBCServiceException;
- public void updateNodeInfoTableWithTxTimeIDKey(UUID txTimeID, String nodeName) throws MDBCServiceException;
- public LockResult requestLock(LockRequest request) throws MDBCServiceException;
- public void releaseLocks(Map<UUID, LockResult> newLocks) throws MDBCServiceException;
- public OwnershipReturn mergeLatestRows(Dag extendedDag, List<MusicRangeInformationRow> latestRows, List<Range> ranges,
+
+ void deleteMriRow(MusicRangeInformationRow row) throws MDBCServiceException;
+
+ void updateNodeInfoTableWithTxTimeIDKey(UUID txTimeID, String nodeName) throws MDBCServiceException;
+
+ LockResult requestLock(LockRequest request) throws MDBCServiceException;
+
+ void releaseLocks(Map<UUID, LockResult> newLocks) throws MDBCServiceException;
+
+ OwnershipReturn mergeLatestRows(Dag extendedDag, List<MusicRangeInformationRow> latestRows, List<Range> ranges,
Map<UUID, LockResult> locks, UUID ownershipId) throws MDBCServiceException;
+
}
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.BiFunction;
+import java.util.concurrent.*;
+
+import com.datastax.driver.core.*;
import org.apache.commons.lang3.tuple.Pair;
import org.json.JSONObject;
import org.onap.music.datastore.Condition;
import org.onap.music.mdbc.tables.RangeDependency;
import org.onap.music.mdbc.tables.StagingTable;
import org.onap.music.mdbc.tables.TxCommitProgress;
-import com.datastax.driver.core.BoundStatement;
-import com.datastax.driver.core.ColumnDefinitions;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.PreparedStatement;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.TupleValue;
/**
* This class provides the methods that MDBC needs to access Cassandra directly in order to provide persistence
public static final String KEY_MUSIC_RFACTOR = "music_rfactor";
/** The property name to use to provide the replication factor for Cassandra. */
public static final String KEY_MUSIC_NAMESPACE = "music_namespace";
+ /** The property name to use to provide a flag indicating if compression is required */
+ public static final String KEY_COMPRESSION = "mdbc_compression";
/** Namespace for the tables in MUSIC (Cassandra) */
public static final String DEFAULT_MUSIC_NAMESPACE = "namespace";
/** The default property value to use for the Cassandra IP address. */
public static final String DEFAULT_MUSIC_ADDRESS = "localhost";
/** The default property value to use for the Cassandra replication factor. */
- public static final int DEFAULT_MUSIC_RFACTOR = 1;
+ public static final int DEFAULT_MUSIC_RFACTOR = 3;
/** The default primary string column, if none is provided. */
public static final String MDBC_PRIMARYKEY_NAME = "mdbc_cuid";
/** Type of the primary key, if none is defined by the user */
public static final String MDBC_PRIMARYKEY_TYPE = "uuid";
+ public static final boolean DEFAULT_COMPRESSION = true;
//\TODO Add logic to change the names when required and create the tables when necessary
//typemap.put(Types.DATE, "TIMESTAMP");
}
+
protected final String music_ns;
protected final String myId;
protected final String[] allReplicaIds;
+ protected ExecutorService commitExecutorThreads;
+
private final String musicAddress;
private final int music_rfactor;
private MusicConnector mCon = null;
private Map<String, PreparedStatement> ps_cache = new HashMap<>();
private Set<String> in_progress = Collections.synchronizedSet(new HashSet<String>());
private StateManager stateManager;
-
+ private boolean useCompression;
public MusicMixin() {
// Default to using the host_ids of the various peers as the replica IDs (this is probably preferred)
this.musicAddress = info.getProperty(KEY_MUSIC_ADDRESS, DEFAULT_MUSIC_ADDRESS);
logger.info(EELFLoggerDelegate.applicationLogger,"MusicSqlManager: musicAddress="+musicAddress);
-
- String s = info.getProperty(KEY_MUSIC_RFACTOR);
- this.music_rfactor = (s == null) ? DEFAULT_MUSIC_RFACTOR : Integer.parseInt(s);
+ MusicDataStore dsHandle = null;
+ try {
+ dsHandle = MusicDataStoreHandle.getDSHandle();
+ } catch (MusicServiceException e) {
+ e.printStackTrace();
+ }
this.myId = info.getProperty(KEY_MY_ID, getMyHostId());
logger.info(EELFLoggerDelegate.applicationLogger,"MusicSqlManager: myId="+myId);
-
this.allReplicaIds = info.getProperty(KEY_REPLICAS, getAllHostIds()).split(",");
logger.info(EELFLoggerDelegate.applicationLogger,"MusicSqlManager: allReplicaIds="+info.getProperty(KEY_REPLICAS, this.myId));
this.stateManager = stateManager;
+ String c = info.getProperty(KEY_COMPRESSION);
+ this.useCompression = (c == null) ? DEFAULT_COMPRESSION: Boolean.parseBoolean(c);
+
+ String s = info.getProperty(KEY_MUSIC_RFACTOR);
+ this.music_rfactor = (s == null) ? DEFAULT_MUSIC_RFACTOR : Integer.parseInt(s);
+
initializeMetricTables();
+ commitExecutorThreads = Executors.newFixedThreadPool(4);
}
public String getMusicTxDigestTableName(){
public static void createKeyspace(String keyspace, int replicationFactor) throws MDBCServiceException {
Map<String,Object> replicationInfo = new HashMap<>();
- replicationInfo.put("'class'", "'SimpleStrategy'");
- replicationInfo.put("'replication_factor'", replicationFactor);
+ replicationInfo.put("'class'", "'NetworkTopologyStrategy'");
+ if(replicationFactor==3){
+ replicationInfo.put("'dc1'", 1);
+ replicationInfo.put("'dc2'", 1);
+ replicationInfo.put("'dc3'", 1);
+ }
+ else {
+ replicationInfo.put("'class'", "'SimpleStrategy'");
+ replicationInfo.put("'replication_factor'", replicationFactor);
+ }
PreparedQueryObject queryObject = new PreparedQueryObject();
queryObject.appendQueryString(
createMusicEventualTxDigest();
createMusicNodeInfoTable();
createMusicRangeInformationTable(this.music_ns,this.musicRangeInformationTableName);
- createMusicRangeDependencyTable();
+ createMusicRangeDependencyTable(this.music_ns,this.musicRangeDependencyTableName);
}
catch(MDBCServiceException e){
logger.error(EELFLoggerDelegate.errorLogger,"Error creating tables in MUSIC");
return previous.keySet().equals(current.keySet());
}
- protected List<Range> waitForLock(LockRequest request, DatabasePartition partition,
- Map<UUID, LockResult> rowLock) throws MDBCServiceException {
- List<Range> newRanges = new ArrayList<>();
- if(partition.getMRIIndex()!=request.getId()){
- throw new MDBCServiceException("Invalid argument for wait for lock, range id in request and partition should match");
- }
- String fullyQualifiedKey= music_ns+"."+ request.getTable()+"."+request.getId();
- String lockId = MusicCore.createLockReference(fullyQualifiedKey);
- ReturnType lockReturn = acquireLock(fullyQualifiedKey,lockId);
- if(lockReturn.getResult().compareTo(ResultType.SUCCESS) != 0 ) {
- //\TODO Improve the exponential backoff
- List<Range> pendingToLock = request.getToLockRanges();
- Map<UUID, String> currentLockRef = new HashMap<>();
- int n = 1;
- int low = 1;
- int high = 1000;
- Random r = new Random();
- Map<Range, RangeMriRow> rangeRows = findRangeRows(pendingToLock);
- NavigableMap<UUID, List<Range>> rowsToLock = getPendingRows(rangeRows);
- NavigableMap<UUID, List<Range>> prevRows = new TreeMap<>();
- while (!pendingToLock.isEmpty() && isDifferent(prevRows,rowsToLock) ) {
- pendingToLock.clear();
- try {
- Thread.sleep(((int) Math.round(Math.pow(2, n)) * 1000)
- + (r.nextInt(high - low) + low));
- } catch (InterruptedException e) {
- continue;
- }
- n++;
- if (n == 20) {
- throw new MDBCServiceException("Lock was impossible to obtain, waited for 20 exponential backoffs!");
- }
- //\TODO do this in parallel
- //\TODO there is a race condition here, from the time we get the find range rows, to the time we lock the row,
- //\TODO this race condition can only be solved if require to obtain lock to all related rows in MRI
- //\TODO before fully owning the range
- //\TODO The rows need to be lock in increasing order of timestamp
- //there could be a new row created
- // Note: This loop needs to be perfomed in sorted order of timebased UUID
- for (Map.Entry<UUID, List<Range>> pending : rowsToLock.entrySet()) {
- List<Range> rs = lockRow(request, pending, currentLockRef, fullyQualifiedKey, lockId, pendingToLock, rowLock);
- newRanges.addAll(rs);
- }
- if (n++ == 20) {
- throw new MDBCServiceException(
- "Lock was impossible to obtain, waited for 20 exponential backoffs!");
- }
- rangeRows = findRangeRows(pendingToLock);
- prevRows = rowsToLock;
- rowsToLock = getPendingRows(rangeRows);
- }
- }
- else {
- partition.setLockId(lockId);
- rowLock.put(partition.getMRIIndex(),new LockResult(partition.getMRIIndex(), lockId, true, partition.getSnapshot()));
- }
- return newRanges;
- }
-
-
-
protected String createAndAssignLock(String fullyQualifiedKey, DatabasePartition partition) throws MDBCServiceException {
UUID mriIndex = partition.getMRIIndex();
String lockId;
}
}
- protected void appendIndexToMri(String lockId, UUID commitId, UUID MriIndex) throws MDBCServiceException{
- PreparedQueryObject appendQuery = createAppendMtxdIndexToMriQuery(musicRangeInformationTableName, MriIndex, musicTxDigestTableName, commitId);
- ReturnType returnType = MusicCore.criticalPut(music_ns, musicRangeInformationTableName, MriIndex.toString(), appendQuery, lockId, null);
- if(returnType.getResult().compareTo(ResultType.SUCCESS) != 0 ){
- logger.error(EELFLoggerDelegate.errorLogger, "Error when executing append operation with return type: "+returnType.getMessage());
- throw new MDBCServiceException("Error when executing append operation with return type: "+returnType.getMessage());
+ public void createAndAddTxDigest(final StagingTable transactionDigest, UUID digestId)
+ throws MDBCServiceException {
+ ByteBuffer serializedTransactionDigest;
+ serializedTransactionDigest = transactionDigest.getSerializedStagingAndClean();
+ if(useCompression){
+ serializedTransactionDigest = StagingTable.Compress(serializedTransactionDigest);
}
+ addTxDigest(digestId, serializedTransactionDigest);
}
/**
@Override
public void commitLog(DatabasePartition partition,List<Range> eventualRanges, StagingTable transactionDigest,
String txId ,TxCommitProgress progressKeeper) throws MDBCServiceException {
- // first deal with commit for eventually consistent tables
- filterAndAddEventualTxDigest(eventualRanges, transactionDigest, txId, progressKeeper);
-
+
if(partition==null){
logger.warn("Trying tcommit log with null partition");
return;
logger.warn("Trying to commit log with empty ranges");
return;
}
-
+
+ // first deal with commit for eventually consistent tables
+ filterAndAddEventualTxDigest(eventualRanges, transactionDigest, txId, progressKeeper);
UUID mriIndex = partition.getMRIIndex();
String fullyQualifiedMriKey = music_ns+"."+ this.musicRangeInformationTableName+"."+mriIndex;
throw new MDBCServiceException("Not able to commit, as you are no longer the lock-holder for this partition");
}
- UUID commitId;
- //Generate a local commit id
- if(progressKeeper.containsTx(txId)) {
- commitId = progressKeeper.getCommitId(txId);
- }
- else{
- logger.error(EELFLoggerDelegate.errorLogger, "Tx with id "+txId+" was not created in the TxCommitProgress ");
- throw new MDBCServiceException("Tx with id "+txId+" was not created in the TxCommitProgress ");
- }
//Add creation type of transaction digest
-
- //1. Push new row to RRT and obtain its index
if(transactionDigest == null || transactionDigest.isEmpty()) {
return;
}
-
- ByteBuffer serializedTransactionDigest;
- if(!transactionDigest.isEmpty()) {
- serializedTransactionDigest = transactionDigest.getSerializedStagingAndClean();
- MusicTxDigestId digestId = new MusicTxDigestId(mriIndex, -1);
- addTxDigest(digestId, serializedTransactionDigest);
- //2. Save RRT index to RQ
- if (progressKeeper != null) {
- progressKeeper.setRecordId(txId, digestId);
+
+ final MusicTxDigestId digestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1);
+ Callable<Boolean> insertDigestCallable =()-> {
+ try {
+ createAndAddTxDigest(transactionDigest,digestId.transactionId);
+ return true;
+ } catch (MDBCServiceException e) {
+ logger.error(EELFLoggerDelegate.errorLogger, "Error creating and pushing tx digest to music",e);
+ return false;
}
- //3. Append RRT index into the corresponding TIT row array
- appendToRedoLog(partition, digestId);
- List<Range> ranges = partition.getSnapshot();
- for(Range r : ranges) {
- Map<Range, Pair<MriReference, Integer>> alreadyApplied = stateManager.getOwnAndCheck().getAlreadyApplied();
- if(!alreadyApplied.containsKey(r)){
- throw new MDBCServiceException("already applied data structure was not updated correctly and range "
+ };
+ Callable<Boolean> appendCallable=()-> {
+ try {
+ appendToRedoLog(music_ns, mriIndex, digestId.transactionId, lockId, musicTxDigestTableName,
+ musicRangeInformationTableName);
+ return true;
+ } catch (MDBCServiceException e) {
+ logger.error(EELFLoggerDelegate.errorLogger, "Error creating and pushing tx digest to music",e);
+ return false;
+ }
+ };
+
+ Future<Boolean> appendResultFuture = commitExecutorThreads.submit(appendCallable);
+ Future<Boolean> digestFuture = commitExecutorThreads.submit(insertDigestCallable);
+ try {
+ //Boolean appendResult = appendResultFuture.get();
+ Boolean digestResult = digestFuture.get();
+ if(/*!appendResult ||*/ !digestResult){
+ logger.error(EELFLoggerDelegate.errorLogger, "Error appending to log or adding tx digest");
+ throw new MDBCServiceException("Error appending to log or adding tx digest");
+ }
+ } catch (InterruptedException|ExecutionException e) {
+ logger.error(EELFLoggerDelegate.errorLogger, "Error executing futures for creating and pushing tx " +
+ "digest to music",e);
+ throw new MDBCServiceException("Failure when retrieving futures for execution of digestion creation and append", e);
+ }
+
+ if (progressKeeper != null) {
+ progressKeeper.setRecordId(txId, digestId);
+ }
+ List<Range> ranges = partition.getSnapshot();
+ for(Range r : ranges) {
+ Map<Range, Pair<MriReference, Integer>> alreadyApplied = stateManager.getOwnAndCheck().getAlreadyApplied();
+ if(!alreadyApplied.containsKey(r)){
+ throw new MDBCServiceException("already applied data structure was not updated correctly and range "
+r+" is not contained");
- }
- Pair<MriReference, Integer> rowAndIndex = alreadyApplied.get(r);
- MriReference key = rowAndIndex.getKey();
- if(!mriIndex.equals(key.index)){
- throw new MDBCServiceException("already applied data structure was not updated correctly and range "+
- r+" is not pointing to row: "+mriIndex.toString());
- }
- alreadyApplied.put(r, Pair.of(new MriReference(mriIndex), rowAndIndex.getValue()+1));
}
+ Pair<MriReference, Integer> rowAndIndex = alreadyApplied.get(r);
+ MriReference key = rowAndIndex.getKey();
+ if(!mriIndex.equals(key.index)){
+ throw new MDBCServiceException("already applied data structure was not updated correctly and range "+
+ r+" is not pointing to row: "+mriIndex.toString());
+ }
+ alreadyApplied.put(r, Pair.of(new MriReference(mriIndex), rowAndIndex.getValue()+1));
}
}
private void filterAndAddEventualTxDigest(List<Range> eventualRanges,
- StagingTable transactionDigest, String txId,
- TxCommitProgress progressKeeper) throws MDBCServiceException {
+ StagingTable transactionDigest, String txId,
+ TxCommitProgress progressKeeper) throws MDBCServiceException {
if(eventualRanges == null || eventualRanges.isEmpty()) {
return;
throw new MDBCServiceException();
}
- UUID commitId = getCommitId(txId, progressKeeper);
+ if(!transactionDigest.isEmpty()) {
+ ByteBuffer serialized = transactionDigest.getSerializedEventuallyStagingAndClean();
- ByteBuffer serialized = transactionDigest.getSerializedEventuallyStagingAndClean();
+ if (serialized!=null && useCompression) {
+ serialized = StagingTable.Compress(serialized);
+ }
- if(serialized != null ) {
- MusicTxDigestId digestId = new MusicTxDigestId(commitId,-1);
- addEventualTxDigest(digestId, serialized);
+ if (serialized != null) {
+ MusicTxDigestId digestId = new MusicTxDigestId(MDBCUtils.generateUniqueKey(), -1);
+ addEventualTxDigest(digestId, serialized);
+ }
}
-
- }
-
- private UUID getCommitId(String txId, TxCommitProgress progressKeeper)
- throws MDBCServiceException {
- UUID commitId;
- //Generate a local commit id
- if(progressKeeper.containsTx(txId)) {
- commitId = progressKeeper.getCommitId(txId);
- }
- else{
- logger.error(EELFLoggerDelegate.errorLogger, "Tx with id "+txId+" was not created in the TxCommitProgress ");
- throw new MDBCServiceException("Tx with id "+txId+" was not created in the TxCommitProgress ");
- }
- return commitId;
}
/**
return partitions;
}
- public MusicRangeInformationRow getMRIRowFromCassandraRow(Row newRow){
+ static public MusicRangeInformationRow getMRIRowFromCassandraRow(Row newRow){
UUID partitionIndex = newRow.getUUID("rangeid");
List<TupleValue> log = newRow.getList("txredolog",TupleValue.class);
List<MusicTxDigestId> digestIds = new ArrayList<>();
DatabasePartition newPartition = info.getDBPartition();
String fullyQualifiedMriKey = music_ns+"."+ musicRangeInformationTableName+"."+newPartition.getMRIIndex().toString();
- String lockId = createAndAssignLock(fullyQualifiedMriKey, newPartition);
+ String lockId;
+ int counter=0;
+ do {
+ lockId = createAndAssignLock(fullyQualifiedMriKey, newPartition);
//TODO: fix this retry logic
- if(lockId == null || lockId.isEmpty()){
+ } while ((lockId ==null||lockId.isEmpty())&&(counter++<3));
+ if (lockId == null || lockId.isEmpty()) {
throw new MDBCServiceException("Error initializing music range information, error creating a lock for a new row" +
"for key "+fullyQualifiedMriKey) ;
}
.append(rangeAndDependencies.getRange().getTable())
.append(",{");
boolean first=true;
- for(Range r: rangeAndDependencies.dependentRanges()){
+ for (Range r: rangeAndDependencies.dependentRanges()) {
if(first){ first=false; }
else {
insert.append(',');
.append(id)
.append(",{");
boolean first=true;
- for(Range r: rangesCopy){
+ for (Range r: rangesCopy) {
if(first){ first=false; }
else {
insert.append(',');
.append(id)
.append(",{");
boolean first=true;
- for(Range r: ranges){
+ for (Range r: ranges) {
if(first){ first=false; }
else {
insert.append(',');
}
@Override
- public void appendToRedoLog(DatabasePartition partition, MusicTxDigestId newRecord) throws MDBCServiceException {
- logger.info("Appending to redo log for partition " + partition.getMRIIndex() + " txId=" + newRecord.txId);
- PreparedQueryObject appendQuery = createAppendMtxdIndexToMriQuery(musicRangeInformationTableName, partition.getMRIIndex(),
- musicTxDigestTableName, newRecord.txId);
- ReturnType returnType = MusicCore.criticalPut(music_ns, musicRangeInformationTableName, partition.getMRIIndex().toString(),
- appendQuery, partition.getLockId(), null);
- if(returnType.getResult().compareTo(ResultType.SUCCESS) != 0 ){
+ public void appendToRedoLog(UUID MRIIndex, String lockId, MusicTxDigestId newRecord) throws MDBCServiceException {
+ logger.debug("Appending to redo log for partition " + MRIIndex + " txId=" + newRecord.transactionId);
+ appendToRedoLog(music_ns,MRIIndex,newRecord.transactionId,lockId,musicTxDigestTableName,
+ musicRangeInformationTableName);
+ }
+
+ public void appendToRedoLog(String musicNamespace, UUID MRIIndex, UUID transactionId, String lockId,
+ String musicTxDigestTableName, String musicRangeInformationTableName)
+ throws MDBCServiceException{
+ PreparedQueryObject appendQuery = createAppendMtxdIndexToMriQuery(musicRangeInformationTableName, MRIIndex,
+ musicTxDigestTableName, transactionId);
+ ReturnType returnType = MusicCore.criticalPut(musicNamespace, musicRangeInformationTableName, MRIIndex.toString(),
+ appendQuery, lockId, null);
+ //returnType.getExecutionInfo()
+ if (returnType.getResult().compareTo(ResultType.SUCCESS) != 0) {
logger.error(EELFLoggerDelegate.errorLogger, "Error when executing append operation with return type: "+returnType.getMessage());
throw new MDBCServiceException("Error when executing append operation with return type: "+returnType.getMessage());
}
}
public void createMusicTxDigest() throws MDBCServiceException {
- createMusicTxDigest(-1);
+ createMusicTxDigest(this.musicTxDigestTableName,this.music_ns,-1);
}
public void createMusicEventualTxDigest() throws MDBCServiceException {
- createMusicEventualTxDigest(-1);
+ createMusicEventualTxDigest(musicEventualTxDigestTableName,music_ns,-1);
}
* * LeaseCounter: transaction number under this lease, bigint \TODO this may need to be a varint later
* * TransactionDigest: text that contains all the changes in the transaction
*/
- private void createMusicEventualTxDigest(int musicTxDigestTableNumber) throws MDBCServiceException {
- String tableName = this.musicEventualTxDigestTableName;
- if(musicTxDigestTableNumber >= 0) {
+ public static void createMusicEventualTxDigest(String musicEventualTxDigestTableName, String musicNamespace, int musicTxDigestTableNumber) throws MDBCServiceException {
+ String tableName = musicEventualTxDigestTableName;
+ if (musicTxDigestTableNumber >= 0) {
tableName = tableName +
"-" +
Integer.toString(musicTxDigestTableNumber);
StringBuilder fields = new StringBuilder();
fields.append("txid uuid, ");
fields.append("transactiondigest blob, ");
+ fields.append("compressed boolean, ");
fields.append("txTimeId TIMEUUID ");//notice lack of ','
- String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, fields, priKey);
+ String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", musicNamespace, tableName, fields, priKey);
try {
- executeMusicWriteQuery(this.music_ns,tableName,cql);
+ executeMusicWriteQuery(musicNamespace,tableName,cql);
} catch (MDBCServiceException e) {
- logger.error("Initialization error: Failure to create redo records table");
+ logger.error("Initialization error: Failure to create eventual tx digest table");
throw(e);
}
}
* * LeaseCounter: transaction number under this lease, bigint \TODO this may need to be a varint later
* * TransactionDigest: text that contains all the changes in the transaction
*/
- private void createMusicTxDigest(int musicTxDigestTableNumber) throws MDBCServiceException {
- String tableName = this.musicTxDigestTableName;
- if(musicTxDigestTableNumber >= 0) {
+ public static void createMusicTxDigest(String musicTxDigestTableName, String musicNamespace, int musicTxDigestTableNumber) throws MDBCServiceException {
+ String tableName = musicTxDigestTableName;
+ if (musicTxDigestTableNumber >= 0) {
tableName = tableName +
"-" +
Integer.toString(musicTxDigestTableNumber);
String priKey = "txid";
StringBuilder fields = new StringBuilder();
fields.append("txid uuid, ");
+ fields.append("compressed boolean, ");
fields.append("transactiondigest blob ");//notice lack of ','
- String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName, fields, priKey);
+ String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", musicNamespace,
+ tableName, fields, priKey);
try {
- executeMusicWriteQuery(this.music_ns,tableName,cql);
+ executeMusicWriteQuery(musicNamespace,tableName,cql);
} catch (MDBCServiceException e) {
logger.error("Initialization error: Failure to create redo records table");
throw(e);
}
}
- private void createMusicRangeDependencyTable() throws MDBCServiceException {
- String tableName = this.musicRangeDependencyTableName;
+ public static void createMusicRangeDependencyTable(String musicNamespace,String musicRangeDependencyTableName)
+ throws MDBCServiceException {
+ String tableName = musicRangeDependencyTableName;
String priKey = "range";
StringBuilder fields = new StringBuilder();
fields.append("range text, ");
fields.append("dependencies set<text> ");//notice lack of ','
- String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", this.music_ns, tableName,
+ String cql = String.format("CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));", musicNamespace, tableName,
fields, priKey);
try {
- executeMusicWriteQuery(this.music_ns,tableName,cql);
+ executeMusicWriteQuery(musicNamespace,tableName,cql);
} catch (MDBCServiceException e) {
logger.error("Initialization error: Failure to create redo records table");
throw(e);
@Override
public void addTxDigest(MusicTxDigestId newId, ByteBuffer transactionDigest) throws MDBCServiceException {
//\TODO: Save Prepared query to history
+ addTxDigest(newId.transactionId,transactionDigest);
+ }
+
+ private void addTxDigest(UUID digestId, ByteBuffer transactionDigest) throws MDBCServiceException{
PreparedQueryObject query = new PreparedQueryObject();
- String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest) VALUES (?,?);",this.music_ns,
+ String cql = String.format("INSERT INTO %s.%s (txid,transactiondigest,compressed ) VALUES (?,?,?);",this.music_ns,
this.musicTxDigestTableName);
query.appendQueryString(cql);
- query.addValue(newId.txId);
+ query.addValue(digestId);
query.addValue(transactionDigest);
+ query.addValue(useCompression);
//\TODO check if I am not shooting on my own foot
try {
MusicCore.nonKeyRelatedPut(query,"critical");
} catch (MusicServiceException e) {
- logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for commit "+newId.txId.toString()+ "with error "+e.getErrorMessage());
- throw new MDBCServiceException("Transaction Digest serialization for commit "+newId.txId.toString(), e);
+ logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for digest id "+digestId.toString()+ "with error "+e.getErrorMessage());
+ throw new MDBCServiceException("Transaction Digest serialization for digest id "+digestId.toString(), e);
}
}
this.music_ns +
'.' +
this.musicEventualTxDigestTableName +
- " (txid,transactiondigest,txTimeId) " +
+ " (txid,transactiondigest,compressed,txTimeId) " +
"VALUES (" +
- newId.txId + ",'" +
- transactionDigest +
- "'," +
+ newId.transactionId+ ",'" +
+ transactionDigest + "'," +
+ useCompression + ","+
// "toTimestamp(now())" +
"now()" +
");";
try {
MusicCore.nonKeyRelatedPut(query,"critical");
} catch (MusicServiceException e) {
- logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for commit "+newId.txId.toString()+ "with error "+e.getErrorMessage());
- throw new MDBCServiceException("Transaction Digest serialization for commit "+newId.txId.toString(), e);
+ logger.error(EELFLoggerDelegate.errorLogger, "Transaction Digest serialization was invalid for commit "+newId.transactionId.toString()+ "with error "+e.getErrorMessage());
+ throw new MDBCServiceException("Transaction Digest serialization for commit "+newId.transactionId.toString(), e);
}
}
String cql = String.format("SELECT * FROM %s.%s WHERE txid = ?;", music_ns, musicTxDigestTableName);
PreparedQueryObject pQueryObject = new PreparedQueryObject();
pQueryObject.appendQueryString(cql);
- pQueryObject.addValue(id.txId);
+ pQueryObject.addValue(id.transactionId);
Row newRow;
try {
newRow = executeMusicUnlockedQuorumGet(pQueryObject);
} catch (MDBCServiceException e) {
- logger.error("Get operation error: Failure to get row from txdigesttable with id:"+id.txId);
+ logger.error("Get operation error: Failure to get row from txdigesttable with id:"+id.transactionId);
throw new MDBCServiceException("Initialization error:Failure to add new row to transaction information", e);
}
ByteBuffer digest = newRow.getBytes("transactiondigest");
+ Boolean compressed = newRow.getBool("compressed");
StagingTable changes;
try {
+ if(compressed){
+ digest = StagingTable.Decompress(digest);
+ }
changes = new StagingTable(digest);
} catch (MDBCServiceException e) {
logger.error("Deserializng digest failed with an exception:"+e.getErrorMessage());
while (!rs.isExhausted()) {
Row row = rs.one();
ByteBuffer digest = row.getBytes("transactiondigest");
+ Boolean compressed = row.getBool("compressed");
//String txTimeId = row.getString("txtimeid"); //???
UUID txTimeId = row.getUUID("txtimeid");
try {
+ if(compressed){
+ digest=StagingTable.Decompress(digest);
+ }
changes = new StagingTable(digest);
} catch (MDBCServiceException e) {
logger.error("Deserializng digest failed: "+e.getErrorMessage());
return ecDigestInformation;
}
-
-
-
ResultSet getAllMriCassandraRows() throws MDBCServiceException {
StringBuilder cqlOperation = new StringBuilder();
cqlOperation.append("SELECT * FROM ")
return rows;
}
- private RangeMriRow findRangeRow(Range range) throws MDBCServiceException {
- RangeMriRow row = null;
- final ResultSet musicResults = getAllMriCassandraRows();
- while (!musicResults.isExhausted()) {
- Row musicRow = musicResults.one();
- final MusicRangeInformationRow mriRow = getMRIRowFromCassandraRow(musicRow);
- final List<Range> musicRanges = getRanges(musicRow);
- //\TODO optimize this for loop to avoid redudant access
- for(Range retrievedRange : musicRanges) {
- if (retrievedRange.overlaps(range)) {
- if(row==null){
- row = new RangeMriRow(range);
- row.setCurrentRow(mriRow);
- }
- else if(row.getCurrentRow().getTimestamp() < mriRow.getTimestamp()){
- row.addOldRow(row.getCurrentRow());
- row.setCurrentRow(mriRow);
- }
- }
- }
- }
- if(row==null){
- logger.error("Row in MRI doesn't exist for Range "+range.toString());
- throw new MDBCServiceException("Row in MRI doesn't exist for Range "+range.toString());
- }
- return row;
- }
-
/**
* This function is used to find all the related uuids associated with the required ranges
* @param ranges ranges to be find
return result;
}
- private List<Range> lockRow(LockRequest request, DatabasePartition partition,Map<UUID, LockResult> rowLock)
- throws MDBCServiceException {
- if(partition.getMRIIndex().equals(request.getId()) && partition.isLocked()){
- return new ArrayList<>();
- }
- //\TODO: this function needs to be improved, to track possible changes in the owner of a set of ranges
- String fullyQualifiedMriKey = music_ns+"."+ this.musicRangeInformationTableName+"."+request.getId().toString();
- //return List<Range> knownRanges, UUID mriIndex, String lockId
- DatabasePartition newPartition = new DatabasePartition(request.getToLockRanges(),request.getId(),null);
- return waitForLock(request,newPartition,rowLock);
- }
-
private void unlockKeyInMusic(String table, String key, String lockref) {
String fullyQualifiedKey= music_ns+"."+ table+"."+lockref;
MusicCore.destroyLockRef(fullyQualifiedKey,lockref);
return new OwnershipReturn(ownershipId, ownRow.getOwnerId(), ownRow.getIndex(),ranges,extendedDag);
}
-
-
/**
* This function is used to check if we need to create a new row in MRI, beacause one of the new ranges is not contained
* @param ranges ranges that should be contained in the partition
return false;
}
- private Map<UUID,String> mergeMriRows(String newId, Map<UUID,LockResult> lock, DatabasePartition partition)
- throws MDBCServiceException {
- Map<UUID,String> oldIds = new HashMap<>();
- List<Range> newRanges = new ArrayList<>();
- for (Map.Entry<UUID,LockResult> entry : lock.entrySet()) {
- oldIds.put(entry.getKey(),entry.getValue().getOwnerId());
- //\TODO check if we need to do a locked get? Is that even required?
- final MusicRangeInformationRow mriRow = getMusicRangeInformation(entry.getKey());
- final DatabasePartition dbPartition = mriRow.getDBPartition();
- newRanges.addAll(dbPartition.getSnapshot());
- }
- DatabasePartition newPartition = new DatabasePartition(newRanges,UUID.fromString(newId),null);
- String fullyQualifiedMriKey = music_ns+"."+ this.musicRangeInformationTableName+"."+newId;
- UUID newUUID = UUID.fromString(newId);
- LockRequest newRequest = new LockRequest(musicRangeInformationTableName,newUUID,newRanges);
- waitForLock(newRequest, newPartition,lock);
- if(!lock.containsKey(newUUID)||!lock.get(newUUID).isNewLock()){
- logger.error("When merging rows, lock returned an invalid error");
- throw new MDBCServiceException("When merging MRI rows, lock returned an invalid error");
- }
- final LockResult lockResult = lock.get(newUUID);
- partition.updateDatabasePartition(newPartition);
- logger.info("Creating MRI " +partition.getMRIIndex()+ " for ranges " + partition.getSnapshot());
- createEmptyMriRow(this.music_ns,this.musicRangeInformationTableName,partition.getMRIIndex(),myId,
- lockResult.getOwnerId(),partition.getSnapshot(),true);
- return oldIds;
- }
-
- private void obtainAllLocks(NavigableMap<UUID, List<Range>> rowsToLock,DatabasePartition partition,
- List<Range> newRanges,Map<UUID, LockResult> rowLock) throws MDBCServiceException {
- //\TODO: perform this operations in parallel
- for(Map.Entry<UUID,List<Range>> row : rowsToLock.entrySet()){
- List<Range> additionalRanges;
- try {
- LockRequest newRequest = new LockRequest(musicRangeInformationTableName,row.getKey(),row.getValue());
- additionalRanges =lockRow(newRequest, partition, rowLock);
- } catch (MDBCServiceException e) {
- //TODO: Make a decision if retry or just fail?
- logger.error("Error locking row",e);
- throw e;
- }
- newRanges.addAll(additionalRanges);
- }
- }
-
-/* @Override
- public OwnershipReturn appendRange(String rangeId, List<Range> ranges, DatabasePartition partition)
- throws MDBCServiceException {
- if(!isAppendRequired(ranges,partition)){
- return new OwnershipReturn(partition.getLockId(),UUID.fromString(rangeId),null,null);
- }
- Map<Range, RangeMriRow> rows = findRangeRows(ranges);
- final NavigableMap<UUID, List<Range>> rowsToLock = getPendingRows(rows);
- HashMap<UUID, LockResult> rowLock = new HashMap<>();
- List<Range> newRanges = new ArrayList<>();
-
- obtainAllLocks(rowsToLock,partition,newRanges,rowLock);
- String lockId;
- Map<UUID,String> oldIds = null;
- if(rowLock.size()!=1){
- oldIds = mergeMriRows(rangeId, rowLock, partition);
- lockId = partition.getLockId();
- }
- else{
- List<LockResult> list = new ArrayList<>(rowLock.values());
- LockResult lockResult = list.get(0);
- lockId = lockResult.getOwnerId();
- }
-
- return new OwnershipReturn(lockId,UUID.fromString(rangeId),oldIds,newRanges);
- }*/
@Override
public void relinquish(String ownerId, String rangeId) throws MDBCServiceException{
* @return true if we should try to relinquish, else should avoid relinquishing in this iteration
*/
private boolean canTryRelinquishing(){
- return true;
+ //\TODO: Fix this!!!! REALLY IMPORTANT TO BE FIX
+ // This should actually have some mechanism to relinquish ownership
+ return false;
}
@Override
}
public void createMusicNodeInfoTable() throws MDBCServiceException {
- createMusicNodeInfoTable(-1);
+ createMusicNodeInfoTable(musicNodeInfoTableName,music_ns,-1);
}
/**
* * TxTimeID, TIMEUUID.
* * LastTxDigestID, uuid. (not needed as of now!!)
*/
- private void createMusicNodeInfoTable(int nodeInfoTableNumber) throws MDBCServiceException {
- String tableName = this.musicNodeInfoTableName;
+ public static void createMusicNodeInfoTable(String musicNodeInfoTableName, String musicNamespace, int nodeInfoTableNumber) throws MDBCServiceException {
+ String tableName = musicNodeInfoTableName;
if(nodeInfoTableNumber >= 0) {
tableName = tableName +
"-" +
String cql = String.format(
"CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));",
- this.music_ns,
+ musicNamespace,
tableName,
fields,
priKey);
try {
- executeMusicWriteQuery(this.music_ns,tableName,cql);
+ executeMusicWriteQuery(musicNamespace,tableName,cql);
} catch (MDBCServiceException e) {
logger.error("Initialization error: Failure to create node information table");
throw(e);
}
+ @Override
+ public void deleteMriRow(MusicRangeInformationRow row) throws MDBCServiceException{
+ String cql = String.format("DELETE FROM %s.%s WHERE rangeid = ?;", music_ns, musicRangeInformationTableName);
+ PreparedQueryObject pQueryObject = new PreparedQueryObject();
+ pQueryObject.appendQueryString(cql);
+ pQueryObject.addValue(row.getPartitionIndex());
+ ReturnType rt ;
+ try {
+ rt = MusicCore.atomicPut(music_ns, musicRangeDependencyTableName, row.getPartitionIndex().toString(),
+ pQueryObject, null);
+ } catch (MusicLockingException|MusicQueryException|MusicServiceException e) {
+ logger.error("Failure when deleting mri row");
+ new MDBCServiceException("Error deleting mri row",e);
+ }
+ }
}
public static final String TRANS_TBL = "MDBC_TRANSLOG";
private static final String CREATE_TBL_SQL =
"CREATE TABLE IF NOT EXISTS "+TRANS_TBL+
- " (IX INT AUTO_INCREMENT, OP CHAR(1), TABLENAME VARCHAR(255),KEYDATA VARCHAR(1024), ROWDATA VARCHAR(1024), CONNECTION_ID INT,PRIMARY KEY (IX))";
+ " (IX INT AUTO_INCREMENT, OP CHAR(1), TABLENAME VARCHAR(255),KEYDATA VARCHAR(1024), ROWDATA VARCHAR(1024), " +
+ "CONNECTION_ID INT, PRIMARY KEY (IX));";
private final MusicInterface mi;
private final int connId;
private final String dbName;
private final Connection jdbcConn;
private final Map<String, TableInfo> tables;
+ private PreparedStatement deleteStagingStatement;
private boolean server_tbl_created = false;
public MySQLMixin() {
this.dbName = null;
this.jdbcConn = null;
this.tables = null;
+ this.deleteStagingStatement = null;
}
- public MySQLMixin(MusicInterface mi, String url, Connection conn, Properties info) {
+ public MySQLMixin(MusicInterface mi, String url, Connection conn, Properties info) throws SQLException {
this.mi = mi;
this.connId = generateConnID(conn);
this.dbName = getDBName(conn);
this.jdbcConn = conn;
this.tables = new HashMap<String, TableInfo>();
}
+
+ private PreparedStatement getStagingDeletePreparedStatement() throws SQLException {
+ return jdbcConn.prepareStatement("DELETE FROM "+TRANS_TBL+" WHERE (IX BETWEEN ? AND ? ) AND " +
+ "CONNECTION_ID = ?;");
+ }
// This is used to generate a unique connId for this connection to the DB.
private int generateConnID(Connection conn) {
int rv = (int) System.currentTimeMillis(); // random-ish
Statement stmt = jdbcConn.createStatement();
stmt.execute(CREATE_TBL_SQL);
stmt.close();
+ this.deleteStagingStatement = getStagingDeletePreparedStatement();
logger.info(EELFLoggerDelegate.applicationLogger,"createSQLTriggers: Server side dirty table created.");
server_tbl_created = true;
} catch (SQLException e) {
// copy from DB.MDBC_TRANSLOG where connid == myconnid
// then delete from MDBC_TRANSLOG
String sql2 = "SELECT IX, TABLENAME, OP, ROWDATA,KEYDATA FROM "+TRANS_TBL +" WHERE CONNECTION_ID = " + this.connId;
+ Integer biggestIx = Integer.MIN_VALUE;
+ Integer smallestIx = Integer.MAX_VALUE;
try {
ResultSet rs = executeSQLRead(sql2);
Set<Integer> rows = new TreeSet<Integer>();
while (rs.next()) {
int ix = rs.getInt("IX");
+ biggestIx = Integer.max(biggestIx,ix);
+ smallestIx = Integer.min(smallestIx,ix);
String op = rs.getString("OP");
OperationType opType = toOpEnum(op);
String tbl = rs.getString("TABLENAME");
rows.add(ix);
}
rs.getStatement().close();
+ // batch delete operations
if (rows.size() > 0) {
- //TODO: DO batch deletion
- sql2 = "DELETE FROM "+TRANS_TBL+" WHERE IX = ?";
- PreparedStatement ps = jdbcConn.prepareStatement(sql2);
- logger.debug("Executing: "+sql2);
- logger.debug(" For ix = "+rows);
- for (int ix : rows) {
- ps.setInt(1, ix);
- ps.execute();
- }
- ps.close();
+ this.deleteStagingStatement.setInt(1,smallestIx);
+ this.deleteStagingStatement.setInt(2,biggestIx);
+ this.deleteStagingStatement.setInt(3,this.connId);
+ logger.debug("Staging delete: Executing with vals ["+smallestIx+","+biggestIx+","+this.connId+"]");
+ this.deleteStagingStatement.execute();
}
} catch (SQLException e) {
logger.warn("Exception in postStatementHook: "+e);
String sql = "DELETE FROM " + TRANS_TBL + " WHERE CONNECTION_ID = " + this.connId;
jdbcStmt.executeQuery(sql);
}
+
+ @Override
+ public Connection getSQLConnection(){
+ return jdbcConn;
+ }
}
checkpointLock.unlock();
break;
} else {
- final StagingTable txDigest = mi.getTxDigest(pair.getKey());
- applyTxDigest(rangesToWarmup,di, txDigest);
- for (Range r : pair.getValue()) {
- MusicRangeInformationRow row = node.getRow();
- alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index));
- }
+ applyDigestAndUpdateDataStructures(mi, di, rangesToWarmup, node, pair);
}
pair = node.nextNotAppliedTransaction(rangeSet);
enableForeignKeys(di);
}
}
+ private void applyDigestAndUpdateDataStructures(MusicInterface mi, DBInterface di, List<Range> ranges, DagNode node,
+ Pair<MusicTxDigestId, List<Range>> pair) throws MDBCServiceException {
+ final StagingTable txDigest;
+ try {
+ txDigest = mi.getTxDigest(pair.getKey());
+ } catch (MDBCServiceException e) {
+ logger.warn("Transaction digest was not found, this could be caused by a failure of the previous owner"
+ +"And would normally only happen as the last ID of the corresponding redo log. Please check that this is the"
+ +" case for txID "+pair.getKey().transactionId.toString());
+ return;
+ }
+ applyTxDigest(ranges,di, txDigest);
+ for (Range r : pair.getValue()) {
+ MusicRangeInformationRow row = node.getRow();
+ alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index));
+ }
+ }
+
private void applyRequiredChanges(MusicInterface mi, DBInterface db, Dag extendedDag, List<Range> ranges, UUID ownOpId)
throws MDBCServiceException {
Set<Range> rangeSet = new HashSet<Range>(ranges);
if(node!=null) {
Pair<MusicTxDigestId, List<Range>> pair = node.nextNotAppliedTransaction(rangeSet);
while (pair != null) {
- final StagingTable txDigest = mi.getTxDigest(pair.getKey());
- applyTxDigest(ranges, db, txDigest);
- for (Range r : pair.getValue()) {
- MusicRangeInformationRow row = node.getRow();
- alreadyApplied.put(r, Pair.of(new MriReference(row.getPartitionIndex()), pair.getKey().index));
- }
+ applyDigestAndUpdateDataStructures(mi, db, ranges, node, pair);
pair = node.nextNotAppliedTransaction(rangeSet);
if (timeout(ownOpId)) {
enableForeignKeys(db);
return rowsPerLatestRange;
}
+
public Map<Range, Pair<MriReference, Integer>> getAlreadyApplied() {
return this.alreadyApplied;
}
+++ /dev/null
-/*
- * ============LICENSE_START====================================================
- * org.onap.music.mdbc
- * =============================================================================
- * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved.
- * =============================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END======================================================
- */
-package org.onap.music.mdbc.tables;
-
-import java.sql.SQLException;
-import java.sql.Timestamp;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import org.onap.music.exceptions.MDBCServiceException;
-import org.onap.music.logging.EELFLoggerDelegate;
-import org.onap.music.mdbc.DatabasePartition;
-import org.onap.music.mdbc.MdbcConnection;
-import org.onap.music.mdbc.Range;
-import org.onap.music.mdbc.StateManager;
-import org.onap.music.mdbc.mixins.DBInterface;
-import org.onap.music.mdbc.mixins.MusicInterface;
-
-public class MusicTxDigest {
- private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicTxDigest.class);
-
- //private MdbcServerLogic mdbcServer;
- //private NodeConfiguration config;
- private StateManager stateManager;
-
- public MusicTxDigest(StateManager stateManager) {
- this.stateManager = stateManager;
- }
-
- /**
- * Runs the body of the background daemon
- * @param daemonSleepTimeS time, in seconds, between updates
- * @throws InterruptedException
- */
- public void backgroundDaemon(int daemonSleepTimeS) throws InterruptedException {
- MusicInterface mi = stateManager.getMusicInterface();
- DBInterface dbi = ((MdbcConnection) stateManager.getConnection("daemon")).getDBInterface();
-
- while (true) {
- Thread.sleep(TimeUnit.SECONDS.toMillis(daemonSleepTimeS));
- //update
- logger.info(String.format("[%s] Background MusicTxDigest daemon updating local db",
- new Timestamp(System.currentTimeMillis())));
-
- //1) get all other partitions from musicrangeinformation
- List<UUID> partitions = null;
- try {
- partitions = mi.getPartitionIndexes();
- } catch (MDBCServiceException e) {
- logger.error("Error obtainting partition indexes, trying again next iteration");
- continue;
- }
- //2) for each partition I don't own
- final Set<Range> warmupRanges = stateManager.getRangesToWarmup();
- final List<DatabasePartition> currentPartitions = stateManager.getPartitions();
- if (currentPartitions.size() != 0) {
- for (DatabasePartition part : currentPartitions) {
- List<Range> partitionRanges = part.getSnapshot();
- warmupRanges.removeAll(partitionRanges);
- }
- try {
- stateManager.getOwnAndCheck().warmup(mi, dbi, new ArrayList<>(warmupRanges));
- } catch (MDBCServiceException e) {
- logger.error("Unable to update for partition : " + warmupRanges + ". " + e.getMessage());
- continue;
- }
- }
-
- //Step 3: ReplayDigest() for E.C conditions
- try {
- replayDigest(mi,dbi, stateManager.getEventualRanges());
- } catch (MDBCServiceException e) {
- logger.error("Unable to perform Eventual Consistency operations" + e.getMessage());
- continue;
- }
-
- }
- }
-
- /**
- * Replay the digest for eventual consistency.
- * @param mi music interface
- * @param dbi interface to the database that will replay the operations
- * @param ranges only these ranges will be applied from the digests
- * @throws MDBCServiceException
- */
- public void replayDigest(MusicInterface mi, DBInterface dbi, List<Range> ranges) throws MDBCServiceException {
- StagingTable transaction;
- String nodeName = stateManager.getMdbcServerName();
-
- LinkedHashMap<UUID,StagingTable> ecDigestInformation = mi.getEveTxDigest(nodeName);
- Set<UUID> keys = ecDigestInformation.keySet();
- for(UUID txTimeID:keys){
- transaction = ecDigestInformation.get(txTimeID);
- try {
- dbi.replayTransaction(transaction, ranges); // I think this Might change if the data is coming from a new table.. ( what is the new table structure??)
- } catch (SQLException e) {
- logger.error("EC:Rolling back the entire digest replay.");
- return;
- }
- logger.info("EC: Successfully replayed transaction ");
-
- try {
- mi.updateNodeInfoTableWithTxTimeIDKey(txTimeID, nodeName);
- } catch (MDBCServiceException e) {
- logger.error("EC:Rolling back the entire digest replay.");
- }
- }
-;
- }
-
-
- /**
- * Replay the digest for a given partition
- * @param mi music interface
- * @param partitionId the partition to be replayed
- * @param dbi interface to the database that will replay the operations
- * @throws MDBCServiceException
- */
- public static void replayDigestForPartition(MusicInterface mi, UUID partitionId, DBInterface dbi) throws MDBCServiceException {
- final MusicRangeInformationRow row = mi.getMusicRangeInformation(partitionId);
- List<MusicTxDigestId> partitionsRedoLogTxIds = row.getRedoLog();
- for (MusicTxDigestId txId: partitionsRedoLogTxIds) {
- StagingTable transaction = mi.getTxDigest(txId);
- try {
- //\TODO do this two operations in parallel
- dbi.replayTransaction(transaction, row.getDBPartition().getSnapshot());
- mi.replayTransaction(transaction);
- } catch (SQLException e) {
- logger.error("Rolling back the entire digest replay. " + partitionId);
- return;
- }
- logger.info("Successfully replayed transaction " + txId);
- }
- //todo, keep track of where I am in pointer
- }
-
- /**
- * Start the background daemon defined by this object
- * Spawns a new thread and runs "backgroundDaemon"
- * @param daemonSleepTimeS time, in seconds, between updates run by daemon
- */
- public void startBackgroundDaemon(int daemonSleepTimeS) {
- class MusicTxBackgroundDaemon implements Runnable {
- public void run() {
- while (true) {
- try {
- logger.info("MusicTxDigest background daemon started");
- backgroundDaemon(daemonSleepTimeS);
- } catch (InterruptedException e) {
- logger.error("MusicTxDigest background daemon stopped " + e.getMessage());
- }
- }
- }
- }
- Thread t = new Thread(new MusicTxBackgroundDaemon());
- t.start();
-
- }
-}
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc.tables;
+
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.logging.EELFLoggerDelegate;
+import org.onap.music.mdbc.DatabasePartition;
+import org.onap.music.mdbc.MdbcConnection;
+import org.onap.music.mdbc.Range;
+import org.onap.music.mdbc.StateManager;
+import org.onap.music.mdbc.mixins.DBInterface;
+import org.onap.music.mdbc.mixins.MusicInterface;
+
+public class MusicTxDigestDaemon implements Runnable {
+
+ private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicTxDigestDaemon.class);
+
+ private StateManager stateManager;
+ private int daemonSleepTimeS;
+
+ public MusicTxDigestDaemon(int daemonSleepTimeS, StateManager stateManager) {
+ this.stateManager = stateManager;
+ this.daemonSleepTimeS = daemonSleepTimeS;
+ }
+
+ /**
+ * Replay the digest for eventual consistency.
+ *
+ * @param mi music interface
+ * @param dbi interface to the database that will replay the operations
+ * @param ranges only these ranges will be applied from the digests
+ */
+ public void replayDigest(MusicInterface mi, DBInterface dbi, List<Range> ranges) throws MDBCServiceException {
+ StagingTable transaction;
+ String nodeName = stateManager.getMdbcServerName();
+
+ LinkedHashMap<UUID, StagingTable> ecDigestInformation = mi.getEveTxDigest(nodeName);
+ Set<UUID> keys = ecDigestInformation.keySet();
+ for (UUID txTimeID : keys) {
+ transaction = ecDigestInformation.get(txTimeID);
+ try {
+ dbi.replayTransaction(transaction,
+ ranges); // I think this Might change if the data is coming from a new table.. ( what is the new table structure??)
+ } catch (SQLException e) {
+ logger.error("EC:Rolling back the entire digest replay.");
+ return;
+ }
+ logger.info("EC: Successfully replayed transaction ");
+
+ try {
+ mi.updateNodeInfoTableWithTxTimeIDKey(txTimeID, nodeName);
+ } catch (MDBCServiceException e) {
+ logger.error("EC:Rolling back the entire digest replay.");
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ logger.info("MusicTxDigest background daemon started");
+ if (stateManager == null) {
+ logger.error("State manager is null in background daemon");
+ return;
+ }
+ MusicInterface mi = stateManager.getMusicInterface();
+
+ if (mi == null) {
+ logger.error("Music interface or DB interface is null in background daemon");
+ return;
+ }
+ while (true) {
+ try {
+ MdbcConnection conn = (MdbcConnection) stateManager.getConnection("daemon");
+ if (conn == null) {
+ logger.error("Connection created is null in background daemon");
+ return;
+ }
+ DBInterface dbi = (conn).getDBInterface();
+ //update
+ logger.info(String.format("[%s] Background MusicTxDigest daemon updating local db",
+ new Timestamp(System.currentTimeMillis())));
+
+ //1) get all other partitions from musicrangeinformation
+ List<UUID> partitions = null;
+ try {
+ partitions = mi.getPartitionIndexes();
+ } catch (MDBCServiceException e) {
+ logger.error("Error obtainting partition indexes, trying again next iteration");
+ continue;
+ }
+ //2) for each partition I don't own
+ final Set<Range> warmupRanges = stateManager.getRangesToWarmup();
+ final List<DatabasePartition> currentPartitions = stateManager.getPartitions();
+ List<Range> missingRanges = new ArrayList<>();
+ if (currentPartitions.size() != 0) {
+ for (DatabasePartition part : currentPartitions) {
+ List<Range> partitionRanges = part.getSnapshot();
+ warmupRanges.removeAll(partitionRanges);
+ }
+ try {
+ stateManager.getOwnAndCheck().warmup(mi, dbi, new ArrayList<>(warmupRanges));
+ } catch (MDBCServiceException e) {
+ logger.error("Unable to update for partition : " + warmupRanges + ". " + e.getMessage());
+ continue;
+ }
+ }
+
+ //Step 3: ReplayDigest() for E.C conditions
+ try {
+ replayDigest(mi, dbi, stateManager.getEventualRanges());
+ } catch (MDBCServiceException e) {
+ logger.error("Unable to perform Eventual Consistency operations" + e.getMessage());
+ continue;
+ }
+ conn.close();
+ Thread.sleep(TimeUnit.SECONDS.toMillis(this.daemonSleepTimeS));
+ } catch (InterruptedException | SQLException e) {
+ logger.error("MusicTxDigest background daemon stopped " + e.getMessage(), e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+}
import java.util.UUID;
public final class MusicTxDigestId {
- public final UUID txId;
+ public final UUID transactionId;
public final int index;
- public MusicTxDigestId(UUID primaryKey, int index) {
- this.txId= primaryKey;
+ public MusicTxDigestId(UUID digestId, int index) {
+ this.transactionId= digestId;
this.index=index;
}
public boolean isEmpty() {
- return (this.txId==null);
+ return (this.transactionId==null);
}
@Override
if(o == null) return false;
if(!(o instanceof MusicTxDigestId)) return false;
MusicTxDigestId other = (MusicTxDigestId) o;
- return other.txId.equals(this.txId);
+ return other.transactionId.equals(this.transactionId);
}
@Override
public int hashCode(){
- return txId.hashCode();
+ return transactionId.hashCode();
}
}
KEY = key;
}
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ Operation clone = null;
+ try {
+ clone = (Operation) super.clone();
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ return clone;
+ }
+
public String getTable(){
return TABLE;
}
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
+import java.util.zip.DataFormatException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+import javax.validation.constraints.Null;
import org.onap.music.exceptions.MDBCServiceException;
import org.onap.music.logging.EELFLoggerDelegate;
import org.onap.music.mdbc.Range;
public class StagingTable {
- private transient static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(StagingTable.class);
+ private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(StagingTable.class);
private ArrayList<Operation> operations;
boolean builderInitialized;
Builder digestBuilder;
this(new HashSet<>());
logger.debug("Creating staging table with no parameters, most likely this is wrong, unless you are testing");
}
+
+ public StagingTable(StagingTable other) throws CloneNotSupportedException {
+ if(other==null){
+ throw new NullPointerException("Invalid constructor parameter passed, it is null");
+ }
+ //TODO this is a highly inefficient deep copy, please don't use in prod
+ operations=null;
+ if(other.operations!=null) {
+ Iterator<Operation> iterator = other.operations.iterator();
+ operations = new ArrayList<>();
+ while (iterator.hasNext()) {
+ operations.add((Operation) iterator.next().clone());
+ }
+ }
+ builderInitialized=other.builderInitialized;
+ digestBuilder=null;
+ if(other.digestBuilder!=null) {
+ CompleteDigest build = other.digestBuilder.build();
+ digestBuilder = build.toBuilder();
+ }
+ eventuallyBuilder=null;
+ if(other.eventuallyBuilder!=null) {
+ CompleteDigest build2 = other.digestBuilder.build();
+ eventuallyBuilder = build2.toBuilder();
+ }
+ eventuallyConsistentRanges=null;
+ if(other.eventuallyConsistentRanges!=null) {
+ eventuallyConsistentRanges = new HashSet<>();
+ Iterator<Range> rangeIter = other.eventuallyConsistentRanges.iterator();
+ while (rangeIter.hasNext()) {
+ eventuallyConsistentRanges.add(rangeIter.next().clone());
+ }
+ }
+ }
public StagingTable(Set<Range> eventuallyConsistentRanges) {
//operations = new ArrayList<Operation>();
}
}
+ public static ByteBuffer Compress(ByteBuffer serializedStaging) throws MDBCServiceException {
+ if(serializedStaging.hasArray()) {
+ //\TODO: Use JAVA 11 to simplify this process using ByteBuffer natively
+ Deflater compressor = new Deflater();
+ final byte[] inputArray = serializedStaging.array();
+ compressor.setInput(inputArray);
+ compressor.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream(serializedStaging.array().length);
+ byte[] buf = new byte[1024];
+ try {
+ while (!compressor.finished()) {
+ int i = compressor.deflate(buf);
+ bos.write(buf, 0, i);
+ }
+ } finally {
+ compressor.end();
+ try {
+ bos.close();
+ } catch (IOException e) {
+ throw new MDBCServiceException("Error closing ByetArrayOutputStream:",e);
+ }
+ }
+ byte[] output = bos.toByteArray();
+ logger.debug("Staging table compressed from: "+inputArray.length+" to "+output.length);
+ return ByteBuffer.wrap(output);
+ }
+ else{
+ throw new MDBCServiceException("Byte buffer was not created correctly, it should wrap an array");
+ }
+ }
+
+ public static ByteBuffer Decompress(ByteBuffer compressedStaging) throws MDBCServiceException {
+ if(compressedStaging.hasArray()) {
+ //\TODO: Use JAVA 11 to simplify this process using ByteBuffer natively
+ Inflater decompressor = new Inflater();
+ byte[] inputArray = compressedStaging.array();
+ decompressor.setInput(inputArray);
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream(inputArray.length);
+ byte[] buffer = new byte[1024];
+ while (!decompressor.finished()) {
+ int decompressSize = 0;
+ try {
+ decompressSize = decompressor.inflate(buffer);
+ } catch (DataFormatException e) {
+ throw new MDBCServiceException("error decompressing input data",e);
+ }
+ outputStream.write(buffer, 0, decompressSize);
+ }
+ try {
+ outputStream.close();
+ } catch (IOException e) {
+ throw new MDBCServiceException("Error closing output byte stream",e);
+ }
+ byte[] output = outputStream.toByteArray();
+ return ByteBuffer.wrap(output);
+ }
+ else{
+ throw new MDBCServiceException("Byte buffer was not created correctly, it should wrap an array");
+ }
+ }
+
synchronized public boolean isBuilderInitialized(){
return isBuilderInitialized();
}
}
logger.warn("Get operation list with this type of initialization is not suggested for the"
+ "staging table");
- ArrayList newOperations = new ArrayList();
+ ArrayList<Operation> newOperations = new ArrayList<>();
for(Row row : digestBuilder.getRowsList()){
final OpType type = row.getType();
OperationType newType = (type==OpType.INSERT)?OperationType.INSERT:(type==OpType.DELETE)?
throw new MDBCServiceException("This type of staging table is unmutable, please use the constructor"
+ "with no parameters");
}
- ByteString serialized = digestBuilder.build().toByteString();
+ byte[] bytes = digestBuilder.build().toByteArray();
+ ByteBuffer serialized = ByteBuffer.wrap(bytes);
digestBuilder.clear();
- return serialized.asReadOnlyByteBuffer();
+ return serialized;
}
synchronized public ByteBuffer getSerializedEventuallyStagingAndClean() throws MDBCServiceException {
if(eventuallyBuilder == null || eventuallyBuilder.getRowsCount()==0){
return null;
}
- ByteString serialized = eventuallyBuilder.build().toByteString();
+ byte[] bytes = eventuallyBuilder.build().toByteArray();
+ ByteBuffer serialized = ByteBuffer.wrap(bytes);
eventuallyBuilder.clear();
- return serialized.asReadOnlyByteBuffer();
+ return serialized;
}
synchronized public boolean isEmpty() {
--- /dev/null
+/*
+ * ============LICENSE_START====================================================
+ * org.onap.music.mdbc
+ * =============================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END======================================================
+ */
+package org.onap.music.mdbc.tools;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import java.io.FileNotFoundException;
+import org.onap.music.exceptions.MDBCServiceException;
+import org.onap.music.logging.EELFLoggerDelegate;
+import org.onap.music.main.MusicUtil;
+import org.onap.music.mdbc.configurations.ClusterConfiguration;
+
+public class ClusterSetup {
+ public static final EELFLoggerDelegate LOG = EELFLoggerDelegate.getLogger(ClusterSetup.class);
+
+ @Parameter(names = { "-c", "--configuration" }, required = true,
+ description = "This is the input file that is going to have the configuration to setup the cluster")
+ private String configurationFile;
+ @Parameter(names = { "-h", "-help", "--help" }, help = true,
+ description = "Print the help message")
+ private boolean help = false;
+
+ private ClusterConfiguration inputConfig;
+
+ public ClusterSetup(){}
+
+
+ public void readInput(){
+ LOG.info("Reading inputs");
+ try {
+ inputConfig = ClusterConfiguration.readJsonFromFile(configurationFile);
+ } catch (FileNotFoundException e) {
+ LOG.error("Input file is invalid or not found");
+ System.exit(1);
+ }
+ }
+
+ public void createAll() throws MDBCServiceException {
+ inputConfig.initNamespaces();
+ inputConfig.initTables();
+ }
+
+ public static void main(String[] args) {
+ LOG.info("Starting cassandra cluster initializer");
+ LOG.info("Using music file configuration:"+ MusicUtil.getMusicPropertiesFilePath());
+ ClusterSetup configs = new ClusterSetup();
+ @SuppressWarnings("deprecation")
+ JCommander jc = new JCommander(configs, args);
+ if (configs.help) {
+ jc.usage();
+ System.exit(1);
+ return;
+ }
+ configs.readInput();
+ try {
+ configs.createAll();
+ } catch (MDBCServiceException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ System.exit(0);
+ }
+}
cassandra.password =\
cassandra
music_namespace =\
- mdbc_namespace
\ No newline at end of file
+ mdbc_namespace
import java.util.HashMap;
import org.junit.Test;
-import org.onap.music.mdbc.tables.MusicTxDigest;
+import org.onap.music.mdbc.tables.MusicTxDigestDaemon;
import org.onap.music.mdbc.tables.StagingTable;
package org.onap.music.mdbc;
import com.datastax.driver.core.*;
-import com.datastax.driver.core.exceptions.QueryExecutionException;
-import com.datastax.driver.core.exceptions.SyntaxError;
import org.onap.music.exceptions.MDBCServiceException;
import org.onap.music.exceptions.MusicLockingException;
-import org.onap.music.lockingservice.cassandra.CassaLockStore;
import org.onap.music.lockingservice.cassandra.MusicLockState;
import org.onap.music.logging.EELFLoggerDelegate;
import org.onap.music.main.MusicCore;
import org.onap.music.mdbc.mixins.MusicInterface;
import org.onap.music.mdbc.tables.MusicRangeInformationRow;
-import static junit.framework.TestCase.assertNotNull;
-import static junit.framework.TestCase.fail;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
public class TestUtils {
private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(TestUtils.class);
- public static DatabasePartition createBasicRow(Range range, MusicInterface mixin, String mdbcServerName){
+ public static DatabasePartition createBasicRow(Range range, MusicInterface mixin, String mdbcServerName)
+ throws MDBCServiceException {
final UUID uuid = MDBCUtils.generateTimebasedUniqueKey();
List<Range> ranges = new ArrayList<>();
ranges.add(range);
MusicRangeInformationRow newRow = new MusicRangeInformationRow(uuid,dbPartition, new ArrayList<>(), "",
mdbcServerName, true);
DatabasePartition partition=null;
- try {
- partition = mixin.createMusicRangeInformation(newRow);
- } catch (MDBCServiceException e) {
- fail("failure when creating new row");
- }
+ partition = mixin.createMusicRangeInformation(newRow);
return partition;
}
- public static void unlockRow(String keyspace, String mriTableName, DatabasePartition partition){
+ public static void unlockRow(String keyspace, String mriTableName, DatabasePartition partition)
+ throws MusicLockingException {
String fullyQualifiedMriKey = keyspace+"."+ mriTableName+"."+partition.getMRIIndex().toString();
- try {
- MusicLockState musicLockState = MusicCore.voluntaryReleaseLock(fullyQualifiedMriKey, partition.getLockId());
- } catch (MusicLockingException e) {
- fail("failure when releasing lock");
- }
+ MusicLockState musicLockState = MusicCore.voluntaryReleaseLock(fullyQualifiedMriKey, partition.getLockId());
}
public static void createKeyspace(String keyspace, Session session) {
" WITH REPLICATION " +
"= {'class':'SimpleStrategy', 'replication_factor':1}; ";
ResultSet res=null;
- try {
- res = session.execute(queryOp);
- }
- catch(QueryExecutionException e){
- fail("Failure executing creation of keyspace with error: " + e.getMessage());
- } catch(SyntaxError e){
- fail("Failure executing creation of keyspace with syntax error: " + e.getMessage());
- }
- assertTrue("Keyspace "+keyspace+" is already being used, please change it to avoid loosing data",res.wasApplied());
+ res = session.execute(queryOp);
}
public static void deleteKeyspace(String keyspace, Session session){
keyspace +
";";
ResultSet res = session.execute(queryBuilder);
- assertTrue("Keyspace "+keyspace+" doesn't exist and it should",res.wasApplied());
}
public static HashSet<String> getMriColNames(){
);
}
- public static HashMap<String, DataType> getMriColTypes(Cluster cluster){
+ public static HashMap<String, DataType> getMriColTypes(Cluster cluster) throws Exception {
HashMap<String, DataType> expectedTypes = new HashMap<>();
expectedTypes.put("rangeid",DataType.uuid());
expectedTypes.put("keys",DataType.set(DataType.text()));
ProtocolVersion currentVer = cluster.getConfiguration().getProtocolOptions().getProtocolVersion();
- assertNotNull("Protocol version for cluster is invalid", currentVer);
+ if(currentVer != null) {
+ throw new Exception("Protocol version for cluster is invalid");
+ }
CodecRegistry registry = cluster.getConfiguration().getCodecRegistry();
- assertNotNull("Codec registry for cluster is invalid", registry);
+ if(registry!= null) {
+ throw new Exception("Codec registry for cluster is invalid");
+ }
expectedTypes.put("txredolog",DataType.list(TupleType.of(currentVer,registry,DataType.text(),DataType.uuid())));
expectedTypes.put("ownerid",DataType.text());
expectedTypes.put("metricprocessid",DataType.text());
}
public static void checkDataTypeForTable(List<ColumnMetadata> columnsMeta, HashSet<String> expectedColumns,
- HashMap<String,DataType> expectedTypes){
+ HashMap<String,DataType> expectedTypes) throws Exception {
for(ColumnMetadata cMeta : columnsMeta){
String columnName = cMeta.getName();
DataType type = cMeta.getType();
- assertTrue("Invalid column name: "+columnName,expectedColumns.contains(columnName));
- assertTrue("Fix the contents of expectedtypes for column: "+columnName,
- expectedTypes.containsKey(columnName));
- assertEquals("Invalid type for column: "+columnName,
- expectedTypes.get(columnName),type);
+ if(!expectedColumns.contains(columnName)){
+ throw new Exception("Invalid column name: ");
+ }
+ if(!expectedTypes.containsKey(columnName)){
+ throw new Exception("Fix the contents of expectedtypes for column: "+columnName);
+ }
+ if(expectedTypes.get(columnName)!=type) {
+ throw new Exception("Invalid type for column: "+columnName);
+ }
}
}
// Range range = new Range("TABLE1");
// List<Range> ranges = new ArrayList<>();
// ranges.add(range);
-// final DatabasePartition partition = TestUtils.createBasicRow(range, mixin, mdbcServerName);
-// TestUtils.unlockRow(keyspace,mriTableName,partition);
+// DatabasePartition partition=null;
+// try {
+// partition = TestUtils.createBasicRow(range, mixin, mdbcServerName);
+// }
+// catch(Exception e){
+// fail(e.getMessage());
+// }
+// try {
+// TestUtils.unlockRow(keyspace,mriTableName,partition);
+// } catch (MusicLockingException e) {
+// fail(e.getMessage());
+// }
//
// DatabasePartition currentPartition = new DatabasePartition(MDBCUtils.generateTimebasedUniqueKey());
// try {
// assertFalse(node3Row.getIsLatest());
// }
-
@Test
public void relinquish() {
}
Properties properties = new Properties();
properties.setProperty(MusicMixin.KEY_MY_ID,mdbcServerName);
properties.setProperty(MusicMixin.KEY_MUSIC_NAMESPACE,keyspace);
+ properties.setProperty(MusicMixin.KEY_MUSIC_RFACTOR,"1");
//StateManager stateManager = new StateManager("dbUrl", properties, "serverName", "dbName");
ownAndCheck = new OwnershipAndCheckpoint();
musicMixin =new MusicMixin(stateManager, mdbcServerName,properties);
TxCommitProgress progressKeeper = new TxCommitProgress();
progressKeeper.createNewTransactionTracker(id ,this.conn);
musicMixin.commitLog(partition, null, stagingTable, id, progressKeeper);
- TestUtils.unlockRow(keyspace,mriTableName,partition);
+ try {
+ TestUtils.unlockRow(keyspace, mriTableName, partition);
+ }
+ catch(Exception e){
+ fail(e.getMessage());
+ }
}
private OwnershipReturn cleanAndOwnPartition(List<Range> ranges, UUID ownOpId) throws SQLException {
private void cleanAlreadyApplied(OwnershipAndCheckpoint ownAndCheck) {
ownAndCheck.getAlreadyApplied().clear();
}
-}
\ No newline at end of file
+}
import java.util.List;
import org.apache.calcite.sql.parser.SqlParseException;
import org.junit.Test;
-import org.onap.music.mdbc.tables.MusicTxDigest;
+import org.onap.music.mdbc.tables.MusicTxDigestDaemon;
import org.onap.music.mdbc.tables.StagingTable;
<module>mdbc-server</module>
<module>mdbc-packages</module>
<module>mdbc-benchmark</module>
+ <module>mdbc-internal-benchmark</module>
</modules>
<dependencies>
<artifactId>json</artifactId>
<version>20160810</version>
</dependency>
- <dependency>
- <groupId>org.mariadb.jdbc</groupId>
- <artifactId>mariadb-java-client</artifactId>
- <version>2.3.0</version>
- </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
Code Review / music / mdbc.git / commitdiff