diff options
author | Wenchen Fan <wenchen@databricks.com> | 2016-08-15 21:43:41 -0700 |
---|---|---|
committer | Yin Huai <yhuai@databricks.com> | 2016-08-15 21:43:41 -0700 |
commit | 7de30d6e9e5d3020d2ba8c2ce08893d9cd822b56 (patch) | |
tree | d88ef8c9d57b32cd240190343d85fe1602e37eb6 /sql/core | |
parent | 268b71d0d792f875fcfaec5314862236754a00d6 (diff) | |
download | spark-7de30d6e9e5d3020d2ba8c2ce08893d9cd822b56.tar.gz spark-7de30d6e9e5d3020d2ba8c2ce08893d9cd822b56.tar.bz2 spark-7de30d6e9e5d3020d2ba8c2ce08893d9cd822b56.zip |
[SPARK-16916][SQL] serde/storage properties should not have limitations
## What changes were proposed in this pull request?
`CatalogStorageFormat.properties` can be used in 2 ways:
1. for hive tables, it stores the serde properties.
2. for data source tables, it stores the data source options, e.g. `path`, `skipHiveMetadata`, etc.
however, both of them have nothing to do with data source properties, e.g. `spark.sql.sources.provider`, so they should not have limitations about data source properties.
## How was this patch tested?
existing tests
Author: Wenchen Fan <wenchen@databricks.com>
Closes #14506 from cloud-fan/table-prop.
Diffstat (limited to 'sql/core')
3 files changed, 0 insertions, 11 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 8fa7615b97..2eff9337bc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -306,9 +306,6 @@ case class AlterTableSerDePropertiesCommand( "ALTER TABLE attempted to set neither serde class name nor serde properties") override def run(sparkSession: SparkSession): Seq[Row] = { - DDLUtils.verifyTableProperties( - serdeProperties.toSeq.flatMap(_.keys.toSeq), - "ALTER TABLE SERDEPROPERTIES") val catalog = sparkSession.sessionState.catalog val table = catalog.getTableMetadata(tableName) // For datasource tables, disallow setting serde or specifying partition diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 3b1052619b..720399ecc5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -120,7 +120,6 @@ case class CreateTableCommand(table: CatalogTable, ifNotExists: Boolean) extends override def run(sparkSession: SparkSession): Seq[Row] = { DDLUtils.verifyTableProperties(table.properties.keys.toSeq, "CREATE TABLE") - DDLUtils.verifyTableProperties(table.storage.properties.keys.toSeq, "CREATE TABLE") sparkSession.sessionState.catalog.createTable(table, ifNotExists) Seq.empty[Row] } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index ce1f7c5082..0f7fda7666 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1273,11 +1273,6 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { intercept[AnalysisException] { sql("ALTER TABLE does_not_exist SET SERDEPROPERTIES ('x' = 'y')") } - // serde properties must not be a datasource property - val e = intercept[AnalysisException] { - sql(s"ALTER TABLE tab1 SET SERDEPROPERTIES ('${DATASOURCE_PREFIX}foo'='wah')") - } - assert(e.getMessage.contains(DATASOURCE_PREFIX + "foo")) } private def testSetSerdePartition(isDatasourceTable: Boolean): Unit = { @@ -1580,8 +1575,6 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { test("create table with datasource properties (not allowed)") { assertUnsupported("CREATE TABLE my_tab TBLPROPERTIES ('spark.sql.sources.me'='anything')") - assertUnsupported("CREATE TABLE my_tab ROW FORMAT SERDE 'serde' " + - "WITH SERDEPROPERTIES ('spark.sql.sources.me'='anything')") } test("Create Hive Table As Select") { |