From a471c7f9eaa59d55dfff5b9d1a858f304a6b3a84 Mon Sep 17 00:00:00 2001 From: sureshthalamati Date: Fri, 1 Apr 2016 18:33:31 +0200 Subject: [SPARK-14133][SQL] Throws exception for unsupported create/drop/alter index , and lock/unlock operations. ## What changes were proposed in this pull request? This PR throws Unsupported Operation exception for create index, drop index, alter index , lock table , lock database, unlock table, and unlock database operations that are not supported in Spark SQL. Currently these operations are executed executed by Hive. Error: spark-sql> drop index my_index on my_table; Error in query: Unsupported operation: drop index(line 1, pos 0) ## How was this patch tested? Added test cases to HiveQuerySuite yhuai hvanhovell andrewor14 Author: sureshthalamati Closes #12069 from sureshthalamati/unsupported_ddl_spark-14133. --- .../org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 12 ++++++++++-- .../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 10 ++++++---- .../apache/spark/sql/hive/execution/HiveQuerySuite.scala | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) (limited to 'sql') diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 5513bbdc7f..d1747b9915 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -146,7 +146,7 @@ hiveNativeCommands | ROLLBACK WORK? | SHOW PARTITIONS tableIdentifier partitionSpec? | DFS .*? - | (CREATE | ALTER | DROP | SHOW | DESC | DESCRIBE | LOCK | UNLOCK | MSCK | LOAD) .*? + | (CREATE | ALTER | DROP | SHOW | DESC | DESCRIBE | MSCK | LOAD) .*? ; unsupportedHiveNativeCommands @@ -166,6 +166,13 @@ unsupportedHiveNativeCommands | kw1=SHOW kw2=TRANSACTIONS | kw1=SHOW kw2=INDEXES | kw1=SHOW kw2=LOCKS + | kw1=CREATE kw2=INDEX + | kw1=DROP kw2=INDEX + | kw1=ALTER kw2=INDEX + | kw1=LOCK kw2=TABLE + | kw1=LOCK kw2=DATABASE + | kw1=UNLOCK kw2=TABLE + | kw1=UNLOCK kw2=DATABASE ; createTableHeader @@ -640,7 +647,7 @@ nonReserved | INPUTDRIVER | OUTPUTDRIVER | DBPROPERTIES | DFS | TRUNCATE | METADATA | REPLICATION | COMPUTE | STATISTICS | ANALYZE | PARTITIONED | EXTERNAL | DEFINED | RECORDWRITER | REVOKE | GRANT | LOCK | UNLOCK | MSCK | EXPORT | IMPORT | LOAD | VALUES | COMMENT | ROLE - | ROLES | COMPACTIONS | PRINCIPALS | TRANSACTIONS | INDEXES | LOCKS | OPTION + | ROLES | COMPACTIONS | PRINCIPALS | TRANSACTIONS | INDEX | INDEXES | LOCKS | OPTION ; SELECT: 'SELECT'; @@ -861,6 +868,7 @@ ROLES: 'ROLES'; COMPACTIONS: 'COMPACTIONS'; PRINCIPALS: 'PRINCIPALS'; TRANSACTIONS: 'TRANSACTIONS'; +INDEX: 'INDEX'; INDEXES: 'INDEXES'; LOCKS: 'LOCKS'; OPTION: 'OPTION'; diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index bedbf9ae17..695b5ef733 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -352,7 +352,12 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "show_create_table_index", "show_create_table_partitioned", "show_create_table_serde", - "show_create_table_view" + "show_create_table_view", + + // Index commands are not supported + "drop_index", + "drop_index_removes_partition_dirs", + "alter_index" ) /** @@ -369,7 +374,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "alter3", "alter4", "alter5", - "alter_index", "alter_merge_2", "alter_partition_format_loc", "alter_partition_with_whitelist", @@ -496,8 +500,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "distinct_stats", "drop_database_removes_partition_dirs", "drop_function", - "drop_index", - "drop_index_removes_partition_dirs", "drop_multi_partitions", "drop_partitions_filter", "drop_partitions_filter2", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 79774f5913..58259060bf 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1280,6 +1280,21 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { assertUnsupportedFeature { sql("SHOW LOCKS my_table") } } + test("lock/unlock table and database commands are not supported") { + assertUnsupportedFeature { sql("LOCK TABLE my_table SHARED") } + assertUnsupportedFeature { sql("UNLOCK TABLE my_table") } + assertUnsupportedFeature { sql("LOCK DATABASE my_db SHARED") } + assertUnsupportedFeature { sql("UNLOCK DATABASE my_db") } + } + + test("create/drop/alter index commands are not supported") { + assertUnsupportedFeature { + sql("CREATE INDEX my_index ON TABLE my_table(a) as 'COMPACT' WITH DEFERRED REBUILD")} + assertUnsupportedFeature { sql("DROP INDEX my_index ON my_table") } + assertUnsupportedFeature { sql("ALTER INDEX my_index ON my_table REBUILD")} + assertUnsupportedFeature { + sql("ALTER INDEX my_index ON my_table set IDXPROPERTIES (\"prop1\"=\"val1_new\")")} + } } // for SPARK-2180 test -- cgit v1.2.3