diff options
author | Wenchen Fan <wenchen@databricks.com> | 2016-08-04 16:48:30 +0800 |
---|---|---|
committer | Cheng Lian <lian@databricks.com> | 2016-08-04 16:48:30 +0800 |
commit | 43f4fd6f9bfff749af17e3c65b53a33f5ecb0922 (patch) | |
tree | a9625bdc9b5c5c5851a2e50c2cdbd8b7f8a71a67 /sql/catalyst | |
parent | 27e815c31de26636df089b0b8d9bd678b92d3588 (diff) | |
download | spark-43f4fd6f9bfff749af17e3c65b53a33f5ecb0922.tar.gz spark-43f4fd6f9bfff749af17e3c65b53a33f5ecb0922.tar.bz2 spark-43f4fd6f9bfff749af17e3c65b53a33f5ecb0922.zip |
[SPARK-16867][SQL] createTable and alterTable in ExternalCatalog should not take db
## What changes were proposed in this pull request?
These 2 methods take `CatalogTable` as parameter, which already have the database information.
## How was this patch tested?
existing test
Author: Wenchen Fan <wenchen@databricks.com>
Closes #14476 from cloud-fan/minor5.
Diffstat (limited to 'sql/catalyst')
4 files changed, 22 insertions, 18 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala index 35fc6ddacb..27e1810814 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala @@ -69,20 +69,21 @@ abstract class ExternalCatalog { // Tables // -------------------------------------------------------------------------- - def createTable(db: String, tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit + def createTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit def dropTable(db: String, table: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit def renameTable(db: String, oldName: String, newName: String): Unit /** - * Alter a table whose name that matches the one specified in `tableDefinition`, - * assuming the table exists. + * Alter a table whose database and name match the ones specified in `tableDefinition`, assuming + * the table exists. Note that, even though we can specify database in `tableDefinition`, it's + * used to identify the table, not to alter the table's database, which is not allowed. * * Note: If the underlying implementation does not support altering a certain field, * this becomes a no-op. */ - def alterTable(db: String, tableDefinition: CatalogTable): Unit + def alterTable(tableDefinition: CatalogTable): Unit def getTable(db: String, table: String): CatalogTable diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala index 67a90c8895..9ebf7de1a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala @@ -192,9 +192,10 @@ class InMemoryCatalog(hadoopConfig: Configuration = new Configuration) extends E // -------------------------------------------------------------------------- override def createTable( - db: String, tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit = synchronized { + assert(tableDefinition.identifier.database.isDefined) + val db = tableDefinition.identifier.database.get requireDbExists(db) val table = tableDefinition.identifier.table if (tableExists(db, table)) { @@ -266,7 +267,9 @@ class InMemoryCatalog(hadoopConfig: Configuration = new Configuration) extends E catalog(db).tables.remove(oldName) } - override def alterTable(db: String, tableDefinition: CatalogTable): Unit = synchronized { + override def alterTable(tableDefinition: CatalogTable): Unit = synchronized { + assert(tableDefinition.identifier.database.isDefined) + val db = tableDefinition.identifier.database.get requireTableExists(db, tableDefinition.identifier.table) catalog(db).tables(tableDefinition.identifier.table).table = tableDefinition } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 980efda6cf..fabab32592 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -223,7 +223,7 @@ class SessionCatalog( val table = formatTableName(tableDefinition.identifier.table) val newTableDefinition = tableDefinition.copy(identifier = TableIdentifier(table, Some(db))) requireDbExists(db) - externalCatalog.createTable(db, newTableDefinition, ignoreIfExists) + externalCatalog.createTable(newTableDefinition, ignoreIfExists) } /** @@ -242,7 +242,7 @@ class SessionCatalog( val newTableDefinition = tableDefinition.copy(identifier = tableIdentifier) requireDbExists(db) requireTableExists(tableIdentifier) - externalCatalog.alterTable(db, newTableDefinition) + externalCatalog.alterTable(newTableDefinition) } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala index 963a225cdf..201d39a364 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala @@ -157,7 +157,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac val catalog = newBasicCatalog() val table = newTable("external_table1", "db2").copy(tableType = CatalogTableType.EXTERNAL) - catalog.createTable("db2", table, ignoreIfExists = false) + catalog.createTable(table, ignoreIfExists = false) val actual = catalog.getTable("db2", "external_table1") assert(actual.tableType === CatalogTableType.EXTERNAL) } @@ -212,7 +212,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac test("alter table") { val catalog = newBasicCatalog() val tbl1 = catalog.getTable("db2", "tbl1") - catalog.alterTable("db2", tbl1.copy(properties = Map("toh" -> "frem"))) + catalog.alterTable(tbl1.copy(properties = Map("toh" -> "frem"))) val newTbl1 = catalog.getTable("db2", "tbl1") assert(!tbl1.properties.contains("toh")) assert(newTbl1.properties.size == tbl1.properties.size + 1) @@ -222,10 +222,10 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac test("alter table when database/table does not exist") { val catalog = newBasicCatalog() intercept[AnalysisException] { - catalog.alterTable("unknown_db", newTable("tbl1", "unknown_db")) + catalog.alterTable(newTable("tbl1", "unknown_db")) } intercept[AnalysisException] { - catalog.alterTable("db2", newTable("unknown_table", "db2")) + catalog.alterTable(newTable("unknown_table", "db2")) } } @@ -266,7 +266,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac test("basic create and list partitions") { val catalog = newEmptyCatalog() catalog.createDatabase(newDb("mydb"), ignoreIfExists = false) - catalog.createTable("mydb", newTable("tbl", "mydb"), ignoreIfExists = false) + catalog.createTable(newTable("tbl", "mydb"), ignoreIfExists = false) catalog.createPartitions("mydb", "tbl", Seq(part1, part2), ignoreIfExists = false) assert(catalogPartitionsEqual(catalog, "mydb", "tbl", Seq(part1, part2))) } @@ -555,7 +555,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac schema = new StructType().add("a", "int").add("b", "string") ) - catalog.createTable("db1", table, ignoreIfExists = false) + catalog.createTable(table, ignoreIfExists = false) assert(exists(db.locationUri, "my_table")) catalog.renameTable("db1", "my_table", "your_table") @@ -573,7 +573,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac None, None, None, false, Map.empty), schema = new StructType().add("a", "int").add("b", "string") ) - catalog.createTable("db1", externalTable, ignoreIfExists = false) + catalog.createTable(externalTable, ignoreIfExists = false) assert(!exists(db.locationUri, "external_table")) } @@ -591,7 +591,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac .add("b", "string"), partitionColumnNames = Seq("a", "b") ) - catalog.createTable("db1", table, ignoreIfExists = false) + catalog.createTable(table, ignoreIfExists = false) catalog.createPartitions("db1", "tbl", Seq(part1, part2), ignoreIfExists = false) assert(exists(databaseDir, "tbl", "a=1", "b=2")) @@ -665,8 +665,8 @@ abstract class CatalogTestUtils { catalog.createDatabase(newDb("default"), ignoreIfExists = true) catalog.createDatabase(newDb("db1"), ignoreIfExists = false) catalog.createDatabase(newDb("db2"), ignoreIfExists = false) - catalog.createTable("db2", newTable("tbl1", "db2"), ignoreIfExists = false) - catalog.createTable("db2", newTable("tbl2", "db2"), ignoreIfExists = false) + catalog.createTable(newTable("tbl1", "db2"), ignoreIfExists = false) + catalog.createTable(newTable("tbl2", "db2"), ignoreIfExists = false) catalog.createPartitions("db2", "tbl2", Seq(part1, part2), ignoreIfExists = false) catalog.createFunction("db2", newFunc("func1", Some("db2"))) catalog |