From: Vikram Potturi Date: Mon, 19 Nov 2018 22:01:19 +0000 (-0500) Subject: Query processing with Calcite X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=14186ff595b31035d401b71111dc75da1c80a807;p=music%2Fmdbc.git Query processing with Calcite Change-Id: Id8ebbc48159810ca782081bb4f63071071c926dd Issue-ID: MUSIC-164 Signed-off-by: Vikram Potturi --- diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.gitreview b/.gitreview old mode 100644 new mode 100755 diff --git a/INFO.yaml b/INFO.yaml old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/docs/histo_graph.svg b/docs/histo_graph.svg old mode 100644 new mode 100755 diff --git a/docs/redo_arch.svg b/docs/redo_arch.svg old mode 100644 new mode 100755 diff --git a/docs/specs.md b/docs/specs.md old mode 100644 new mode 100755 diff --git a/docs/uml.svg b/docs/uml.svg old mode 100644 new mode 100755 diff --git a/mdbc-server/pom.xml b/mdbc-server/pom.xml index d2e5fd5..f01f90a 100755 --- a/mdbc-server/pom.xml +++ b/mdbc-server/pom.xml @@ -34,6 +34,12 @@ + + org.apache.calcite + calcite + 1.11.0 + pom + com.datastax.cassandra cassandra-driver-core diff --git a/mdbc-server/src/main/java/org/onap/music/exceptions/MDBCServiceException.java b/mdbc-server/src/main/java/org/onap/music/exceptions/MDBCServiceException.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/exceptions/QueryException.java b/mdbc-server/src/main/java/org/onap/music/exceptions/QueryException.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/logging/EELFLoggerDelegate.java b/mdbc-server/src/main/java/org/onap/music/logging/EELFLoggerDelegate.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/logging/format/AppMessages.java b/mdbc-server/src/main/java/org/onap/music/logging/format/AppMessages.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/logging/format/ErrorSeverity.java b/mdbc-server/src/main/java/org/onap/music/logging/format/ErrorSeverity.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/logging/format/ErrorTypes.java b/mdbc-server/src/main/java/org/onap/music/logging/format/ErrorTypes.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/ArchiveProcess.java b/mdbc-server/src/main/java/org/onap/music/mdbc/ArchiveProcess.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/Configuration.java b/mdbc-server/src/main/java/org/onap/music/mdbc/Configuration.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/DatabaseOperations.java b/mdbc-server/src/main/java/org/onap/music/mdbc/DatabaseOperations.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/DatabasePartition.java b/mdbc-server/src/main/java/org/onap/music/mdbc/DatabasePartition.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/LockId.java b/mdbc-server/src/main/java/org/onap/music/mdbc/LockId.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MDBCUtils.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MDBCUtils.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcCallableStatement.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcCallableStatement.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcConnection.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcPreparedStatement.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcPreparedStatement.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServer.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServer.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServerLogic.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcServerLogic.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcStatement.java b/mdbc-server/src/main/java/org/onap/music/mdbc/MdbcStatement.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java b/mdbc-server/src/main/java/org/onap/music/mdbc/Range.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/RedoRow.java b/mdbc-server/src/main/java/org/onap/music/mdbc/RedoRow.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/StateManager.java b/mdbc-server/src/main/java/org/onap/music/mdbc/StateManager.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/NodeConfiguration.java b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/NodeConfiguration.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/TablesConfiguration.java b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/TablesConfiguration.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/config-0.json b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/config-0.json old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/ranges.json b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/ranges.json old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/tableConfiguration.json b/mdbc-server/src/main/java/org/onap/music/mdbc/configurations/tableConfiguration.json old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/examples/EtdbTestClient.java b/mdbc-server/src/main/java/org/onap/music/mdbc/examples/EtdbTestClient.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java b/mdbc-server/src/main/java/org/onap/music/mdbc/mixins/MusicMixin.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/query/Operation.java b/mdbc-server/src/main/java/org/onap/music/mdbc/query/Operation.java new file mode 100644 index 0000000..6c2d52b --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/query/Operation.java @@ -0,0 +1,37 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc.query; + +public enum Operation { + + INSERT("W"), SELECT("R"), UPDATE("W"), DELETE("W"), TABLE("T"); + + String operation; + + String getOperation() { + return this.operation; + } + + Operation(String operation) { + this.operation = operation; + } + + } \ No newline at end of file diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/query/QueryProcessor.java b/mdbc-server/src/main/java/org/onap/music/mdbc/query/QueryProcessor.java new file mode 100644 index 0000000..bc9a8fc --- /dev/null +++ b/mdbc-server/src/main/java/org/onap/music/mdbc/query/QueryProcessor.java @@ -0,0 +1,245 @@ +/* + * ============LICENSE_START==================================================== + * org.onap.music.mdbc + * ============================================================================= + * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. + * ============================================================================= + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END====================================================== + */ + +package org.onap.music.mdbc.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.avatica.util.Quoting; +import org.apache.calcite.config.Lex; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlSetOption; +import org.apache.calcite.sql.SqlUpdate; +import org.apache.calcite.sql.fun.SqlInOperator; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserImplFactory; +import org.apache.calcite.sql.parser.impl.SqlParserImpl; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.calcite.sql.validate.SqlConformance; +import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.util.Util; +import org.onap.music.logging.EELFLoggerDelegate; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.delete.Delete; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.update.Update; +import net.sf.jsqlparser.util.TablesNamesFinder; + +public class QueryProcessor { + + private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(QueryProcessor.class); + + public List tables = null; + + public QueryProcessor() { + + } + + protected static SqlParserImplFactory parserImplFactory() { + return SqlParserImpl.FACTORY; + } + + protected static SqlParser getSqlParser(String sql) { + Quoting quoting = Quoting.DOUBLE_QUOTE; + Casing unquotedCasing = Casing.TO_UPPER; + Casing quotedCasing = Casing.UNCHANGED; + SqlConformance conformance = SqlConformanceEnum.DEFAULT; + + return SqlParser.create(sql, SqlParser.configBuilder().setParserFactory(parserImplFactory()).setQuoting(quoting) + .setUnquotedCasing(unquotedCasing).setQuotedCasing(quotedCasing).setConformance(conformance).build()); + } + + public static Map> parseSqlQuery(String query) throws SqlParseException { + Map> tableOpsMap = new HashMap<>(); + /*SqlParser parser = SqlParser.create(query); + SqlNode sqlNode = parser.parseQuery();*/ + SqlNode sqlNode = getSqlParser(query).parseStmt(); + + SqlBasicVisitor visitor = new SqlBasicVisitor() { + + public Void visit(SqlCall call) { + if (call.getOperator() instanceof SqlInOperator) { + throw new Util.FoundOne(call); + } + return super.visit(call); + } + + }; + + // sqlNode.accept(new SqlAnalyzer()); + sqlNode.accept(visitor); + + if (sqlNode instanceof SqlInsert) { + SqlInsert sqlInsert = (SqlInsert) sqlNode; + SqlNodeList targetColumnList = sqlInsert.getTargetColumnList(); + System.out.println("targetColumnList: "+targetColumnList); + String tableName = sqlInsert.getTargetTable().toString(); + List Ops = tableOpsMap.get(tableName); + if (Ops == null) + Ops = new ArrayList<>(); + Ops.add(Operation.INSERT.getOperation()); + tableOpsMap.put(tableName, Ops); + } else if (sqlNode instanceof SqlUpdate) { + SqlUpdate sqlUpdate = (SqlUpdate) sqlNode; + String tableName = sqlUpdate.getTargetTable().toString(); + List Ops = tableOpsMap.get(tableName); + if (Ops == null) + Ops = new ArrayList<>(); + Ops.add(Operation.UPDATE.getOperation()); + tableOpsMap.put(tableName, Ops); + } else if (sqlNode instanceof SqlSelect) { + SqlSelect sqlSelect = (SqlSelect) sqlNode; + SqlNodeList selectList = sqlSelect.getSelectList(); + String tables = sqlSelect.getFrom().toString(); + String[] tablesArr = tables.split(","); + + SqlNode where = sqlSelect.getWhere(); + + for (String table : tablesArr) { + String[] split = table.split("`"); + String tableName = split[1]; + List Ops = tableOpsMap.get(tableName); + if (Ops == null) Ops = new ArrayList<>(); + if (where == null) { + Ops.add(Operation.TABLE.getOperation()); + tableOpsMap.put(split[1], Ops); + } else { + Ops.add(Operation.SELECT.getOperation()); + tableOpsMap.put(split[1], Ops); + } + } + } + + return tableOpsMap; + } + + public static Map> extractTableFromQuery(String sqlQuery) { + List tables = null; + Map> tableOpsMap = new HashMap<>(); + try { + net.sf.jsqlparser.statement.Statement stmt = CCJSqlParserUtil.parse(sqlQuery); + if (stmt instanceof Insert) { + Insert s = (Insert) stmt; + String tbl = s.getTable().getName(); + List Ops = tableOpsMap.get(tbl); + if (Ops == null) + Ops = new ArrayList<>(); + Ops.add(Operation.INSERT.getOperation()); + tableOpsMap.put(tbl, Ops); + logger.debug(EELFLoggerDelegate.applicationLogger, "Inserting into table: " + tbl); + } else { + String tbl; + String where = ""; + if (stmt instanceof Update) { + Update u = (Update) stmt; + tbl = u.getTables().get(0).getName(); + List Ops = tableOpsMap.get(tbl); + if (Ops == null) + Ops = new ArrayList<>(); + if (u.getWhere() != null) { + where = u.getWhere().toString(); + logger.debug(EELFLoggerDelegate.applicationLogger, "Updating table: " + tbl); + Ops.add(Operation.UPDATE.getOperation()); + } else { + Ops.add(Operation.TABLE.getOperation()); + } + tableOpsMap.put(tbl, Ops); + } else if (stmt instanceof Delete) { + Delete d = (Delete) stmt; + tbl = d.getTable().getName(); + List Ops = tableOpsMap.get(tbl); + if (Ops == null) + Ops = new ArrayList<>(); + if (d.getWhere() != null) { + where = d.getWhere().toString(); + Ops.add(Operation.DELETE.getOperation()); + } else { + Ops.add(Operation.TABLE.getOperation()); + } + tableOpsMap.put(tbl, Ops); + logger.debug(EELFLoggerDelegate.applicationLogger, "Deleting from table: " + tbl); + } else if (stmt instanceof Select) { + TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); + tables = tablesNamesFinder.getTableList(stmt); + for (String table : tables) { + List Ops = tableOpsMap.get(table); + if (Ops == null) + Ops = new ArrayList<>(); + Ops.add(Operation.SELECT.getOperation()); + tableOpsMap.put(table, Ops); + } + } else { + logger.error(EELFLoggerDelegate.errorLogger, "Not recognized sql type"); + tbl = ""; + } + } + } catch (JSQLParserException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return tableOpsMap; + } + + public static void main(String[] args) throws SqlParseException { + String sqlQuery = "SELECT name, age FROM table1 t1, table2 t2 WHERE t1.id = t2.id"; + // Map> tableOpsMap = extractTableFromQuery(sqlQuery); + System.out.println(parseSqlQuery(sqlQuery)); + + sqlQuery = "SELECT name, age FROM table1, table2 t2 WHERE id = t2.id"; + // Map> tableOpsMap = extractTableFromQuery(sqlQuery); + System.out.println(parseSqlQuery(sqlQuery)); + + sqlQuery = "SELECT name, age FROM table1 t1"; + // Map> tableOpsMap = extractTableFromQuery(sqlQuery); + System.out.println(parseSqlQuery(sqlQuery)); + + sqlQuery = "INSERT INTO Employees (id, name) values ('1','Vikram')"; + // tableOpsMap = extractTableFromQuery(sqlQuery); + // System.out.println(tableOpsMap); + System.out.println(parseSqlQuery(sqlQuery)); + + sqlQuery = "UPDATE Employees SET id = 1 WHERE id = 2"; + System.out.println(parseSqlQuery(sqlQuery)); + + sqlQuery = "UPDATE Employees SET id = 1"; + System.out.println(parseSqlQuery(sqlQuery)); + + sqlQuery = "UPDATE table1 SET id = 1"; + System.out.println(parseSqlQuery(sqlQuery)); + + } + +} diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MriReference.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MriReference.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicRangeInformationRow.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicRangeInformationRow.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestId.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/MusicTxDigestId.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/Operation.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/OperationType.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/OperationType.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/PartitionInformation.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/PartitionInformation.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/StagingTable.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tables/TxCommitProgress.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tables/TxCommitProgress.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tests/ConnectionTest.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tests/ConnectionTest.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tools/CreateNodeConfigurations.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tools/CreateNodeConfigurations.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/java/org/onap/music/mdbc/tools/CreatePartition.java b/mdbc-server/src/main/java/org/onap/music/mdbc/tools/CreatePartition.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/resources/logback.xml b/mdbc-server/src/main/resources/logback.xml old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/resources/mdbc_driver.properties b/mdbc-server/src/main/resources/mdbc_driver.properties old mode 100644 new mode 100755 diff --git a/mdbc-server/src/main/resources/music.properties b/mdbc-server/src/main/resources/music.properties old mode 100644 new mode 100755 diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/DatabaseOperationsTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/DatabaseOperationsTest.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/MDBCUtilsTest.java b/mdbc-server/src/test/java/org/onap/music/mdbc/MDBCUtilsTest.java old mode 100644 new mode 100755 diff --git a/mdbc-server/src/test/java/org/onap/music/mdbc/TestUtils.java b/mdbc-server/src/test/java/org/onap/music/mdbc/TestUtils.java old mode 100644 new mode 100755