diff options
16 files changed, 65 insertions, 65 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index b12606e17d..b7f35b3af4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -49,12 +49,12 @@ case class CatalogStorageFormat( outputFormat: Option[String], serde: Option[String], compressed: Boolean, - serdeProperties: Map[String, String]) { + properties: Map[String, String]) { override def toString: String = { val serdePropsToString = - if (serdeProperties.nonEmpty) { - s"Properties: " + serdeProperties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") + if (properties.nonEmpty) { + s"Properties: " + properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") } else { "" } @@ -73,7 +73,7 @@ case class CatalogStorageFormat( object CatalogStorageFormat { /** Empty storage format for default values and copies. */ val empty = CatalogStorageFormat(locationUri = None, inputFormat = None, - outputFormat = None, serde = None, compressed = false, serdeProperties = Map.empty) + outputFormat = None, serde = None, compressed = false, properties = Map.empty) } /** @@ -165,7 +165,7 @@ case class CatalogTable( outputFormat: Option[String] = storage.outputFormat, compressed: Boolean = false, serde: Option[String] = storage.serde, - serdeProperties: Map[String, String] = storage.serdeProperties): CatalogTable = { + serdeProperties: Map[String, String] = storage.properties): CatalogTable = { copy(storage = CatalogStorageFormat( locationUri, inputFormat, outputFormat, serde, compressed, serdeProperties)) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala index a9268535c4..5bb50cba53 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala @@ -399,11 +399,11 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac // alter other storage information catalog.alterPartitions("db2", "tbl2", Seq( oldPart1.copy(storage = storageFormat.copy(serde = Some(newSerde))), - oldPart2.copy(storage = storageFormat.copy(serdeProperties = newSerdeProps)))) + oldPart2.copy(storage = storageFormat.copy(properties = newSerdeProps)))) val newPart1b = catalog.getPartition("db2", "tbl2", part1.spec) val newPart2b = catalog.getPartition("db2", "tbl2", part2.spec) assert(newPart1b.storage.serde == Some(newSerde)) - assert(newPart2b.storage.serdeProperties == newSerdeProps) + assert(newPart2b.storage.properties == newSerdeProps) // alter but change spec, should fail because new partition specs do not exist yet val badPart1 = part1.copy(spec = Map("a" -> "v1", "b" -> "v2")) val badPart2 = part2.copy(spec = Map("a" -> "v3", "b" -> "v4")) @@ -634,7 +634,7 @@ abstract class CatalogTestUtils { outputFormat = Some(tableOutputFormat), serde = None, compressed = false, - serdeProperties = Map.empty) + properties = Map.empty) lazy val part1 = CatalogTablePartition(Map("a" -> "1", "b" -> "2"), storageFormat) lazy val part2 = CatalogTablePartition(Map("a" -> "3", "b" -> "4"), storageFormat) lazy val part3 = CatalogTablePartition(Map("a" -> "5", "b" -> "6"), storageFormat) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 1316d90fa4..9b09801896 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -968,7 +968,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { // whether to convert a table created by CTAS to a datasource table. serde = None, compressed = false, - serdeProperties = Map()) + properties = Map()) } validateRowFormatFileFormat(ctx.rowFormat, ctx.createFileFormat, ctx) val fileStorage = Option(ctx.createFileFormat).map(visitCreateFileFormat) @@ -986,7 +986,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { outputFormat = fileStorage.outputFormat.orElse(defaultStorage.outputFormat), serde = rowStorage.serde.orElse(fileStorage.serde).orElse(defaultStorage.serde), compressed = false, - serdeProperties = rowStorage.serdeProperties ++ fileStorage.serdeProperties) + properties = rowStorage.properties ++ fileStorage.properties) // If location is defined, we'll assume this is an external table. // Otherwise, we may accidentally delete existing data. val tableType = if (external || location.isDefined) { @@ -1145,7 +1145,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { import ctx._ CatalogStorageFormat.empty.copy( serde = Option(string(name)), - serdeProperties = Option(tablePropertyList).map(visitPropertyKeyValues).getOrElse(Map.empty)) + properties = Option(tablePropertyList).map(visitPropertyKeyValues).getOrElse(Map.empty)) } /** @@ -1173,7 +1173,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { ctx) "line.delim" -> value } - CatalogStorageFormat.empty.copy(serdeProperties = entries.toMap) + CatalogStorageFormat.empty.copy(properties = entries.toMap) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala index c38eca5156..5e3cd9f895 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala @@ -392,7 +392,7 @@ object CreateDataSourceTableUtils extends Logging { outputFormat = None, serde = None, compressed = false, - serdeProperties = options + properties = options ), properties = tableProperties.toMap) } @@ -412,7 +412,7 @@ object CreateDataSourceTableUtils extends Logging { outputFormat = serde.outputFormat, serde = serde.serde, compressed = false, - serdeProperties = options + properties = options ), schema = relation.schema.map { f => CatalogColumn(f.name, f.dataType.catalogString) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index a3a057a562..2a62b864a1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -320,14 +320,14 @@ case class AlterTableSerDePropertiesCommand( if (partSpec.isEmpty) { val newTable = table.withNewStorage( serde = serdeClassName.orElse(table.storage.serde), - serdeProperties = table.storage.serdeProperties ++ serdeProperties.getOrElse(Map())) + serdeProperties = table.storage.properties ++ serdeProperties.getOrElse(Map())) catalog.alterTable(newTable) } else { val spec = partSpec.get val part = catalog.getPartition(tableName, spec) val newPart = part.copy(storage = part.storage.copy( serde = serdeClassName.orElse(part.storage.serde), - serdeProperties = part.storage.serdeProperties ++ serdeProperties.getOrElse(Map()))) + properties = part.storage.properties ++ serdeProperties.getOrElse(Map()))) catalog.alterPartitions(tableName, Seq(newPart)) } Seq.empty[Row] @@ -466,7 +466,7 @@ case class AlterTableSetLocationCommand( if (DDLUtils.isDatasourceTable(table)) { table.withNewStorage( locationUri = Some(location), - serdeProperties = table.storage.serdeProperties ++ Map("path" -> location)) + serdeProperties = table.storage.properties ++ Map("path" -> location)) } else { table.withNewStorage(locationUri = Some(location)) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 6e52a4609a..a62853b05f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -120,7 +120,7 @@ case class CreateTableCommand(table: CatalogTable, ifNotExists: Boolean) extends override def run(sparkSession: SparkSession): Seq[Row] = { DDLUtils.verifyTableProperties(table.properties.keys.toSeq, "CREATE TABLE") - DDLUtils.verifyTableProperties(table.storage.serdeProperties.keys.toSeq, "CREATE TABLE") + DDLUtils.verifyTableProperties(table.storage.properties.keys.toSeq, "CREATE TABLE") sparkSession.sessionState.catalog.createTable(table, ifNotExists) Seq.empty[Row] } @@ -167,7 +167,7 @@ case class AlterTableRenameCommand( if (DDLUtils.isDatasourceTable(table) && table.tableType == CatalogTableType.MANAGED) { val newPath = catalog.defaultTablePath(newName) val newTable = table.withNewStorage( - serdeProperties = table.storage.serdeProperties ++ Map("path" -> newPath)) + serdeProperties = table.storage.properties ++ Map("path" -> newPath)) catalog.alterTable(newTable) } // Invalidate the table last, otherwise uncaching the table would load the logical plan @@ -349,7 +349,7 @@ case class TruncateTableCommand( } val locations = if (isDatasourceTable) { - Seq(table.storage.serdeProperties.get("path")) + Seq(table.storage.properties.get("path")) } else if (table.partitionColumnNames.isEmpty) { Seq(table.storage.locationUri) } else { @@ -492,7 +492,7 @@ case class DescribeTableCommand(table: TableIdentifier, isExtended: Boolean, isF describeBucketingInfo(metadata, buffer) append(buffer, "Storage Desc Parameters:", "", "") - metadata.storage.serdeProperties.foreach { case (key, value) => + metadata.storage.properties.foreach { case (key, value) => append(buffer, s" $key", value, "") } } @@ -820,7 +820,7 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman storage.serde.foreach { serde => builder ++= s"ROW FORMAT SERDE '$serde'\n" - val serdeProps = metadata.storage.serdeProperties.map { + val serdeProps = metadata.storage.properties.map { case (key, value) => s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'" } @@ -890,7 +890,7 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman builder ++= s"USING ${props(CreateDataSourceTableUtils.DATASOURCE_PROVIDER)}\n" - val dataSourceOptions = metadata.storage.serdeProperties.filterNot { + val dataSourceOptions = metadata.storage.properties.filterNot { case (key, value) => // If it's a managed table, omit PATH option. Spark SQL always creates external table // when the table creation DDL contains the PATH option. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala index 0841636d33..8ffdc507db 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala @@ -214,7 +214,7 @@ private[sql] class FindDataSourceTable(sparkSession: SparkSession) extends Rule[ val bucketSpec = DDLUtils.getBucketSpecFromTableProperties(table) - val options = table.storage.serdeProperties + val options = table.storage.properties val dataSource = DataSource( sparkSession, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 467a2287fc..34c980e321 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -85,7 +85,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { outputFormat = None, serde = None, compressed = false, - serdeProperties = Map()) + properties = Map()) CatalogTable( identifier = name, tableType = CatalogTableType.EXTERNAL, @@ -892,9 +892,9 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { convertToDatasourceTable(catalog, tableIdent) } assert(catalog.getTableMetadata(tableIdent).storage.locationUri.isDefined) - assert(catalog.getTableMetadata(tableIdent).storage.serdeProperties.isEmpty) + assert(catalog.getTableMetadata(tableIdent).storage.properties.isEmpty) assert(catalog.getPartition(tableIdent, partSpec).storage.locationUri.isEmpty) - assert(catalog.getPartition(tableIdent, partSpec).storage.serdeProperties.isEmpty) + assert(catalog.getPartition(tableIdent, partSpec).storage.properties.isEmpty) // Verify that the location is set to the expected string def verifyLocation(expected: String, spec: Option[TablePartitionSpec] = None): Unit = { val storageFormat = spec @@ -902,10 +902,10 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { .getOrElse { catalog.getTableMetadata(tableIdent).storage } if (isDatasourceTable) { if (spec.isDefined) { - assert(storageFormat.serdeProperties.isEmpty) + assert(storageFormat.properties.isEmpty) assert(storageFormat.locationUri.isEmpty) } else { - assert(storageFormat.serdeProperties.get("path") === Some(expected)) + assert(storageFormat.properties.get("path") === Some(expected)) assert(storageFormat.locationUri === Some(expected)) } } else { @@ -948,7 +948,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { convertToDatasourceTable(catalog, tableIdent) } assert(catalog.getTableMetadata(tableIdent).storage.serde.isEmpty) - assert(catalog.getTableMetadata(tableIdent).storage.serdeProperties.isEmpty) + assert(catalog.getTableMetadata(tableIdent).storage.properties.isEmpty) // set table serde and/or properties (should fail on datasource tables) if (isDatasourceTable) { val e1 = intercept[AnalysisException] { @@ -963,21 +963,21 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { } else { sql("ALTER TABLE dbx.tab1 SET SERDE 'org.apache.jadoop'") assert(catalog.getTableMetadata(tableIdent).storage.serde == Some("org.apache.jadoop")) - assert(catalog.getTableMetadata(tableIdent).storage.serdeProperties.isEmpty) + assert(catalog.getTableMetadata(tableIdent).storage.properties.isEmpty) sql("ALTER TABLE dbx.tab1 SET SERDE 'org.apache.madoop' " + "WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')") assert(catalog.getTableMetadata(tableIdent).storage.serde == Some("org.apache.madoop")) - assert(catalog.getTableMetadata(tableIdent).storage.serdeProperties == + assert(catalog.getTableMetadata(tableIdent).storage.properties == Map("k" -> "v", "kay" -> "vee")) } // set serde properties only sql("ALTER TABLE dbx.tab1 SET SERDEPROPERTIES ('k' = 'vvv', 'kay' = 'vee')") - assert(catalog.getTableMetadata(tableIdent).storage.serdeProperties == + assert(catalog.getTableMetadata(tableIdent).storage.properties == Map("k" -> "vvv", "kay" -> "vee")) // set things without explicitly specifying database catalog.setCurrentDatabase("dbx") sql("ALTER TABLE tab1 SET SERDEPROPERTIES ('kay' = 'veee')") - assert(catalog.getTableMetadata(tableIdent).storage.serdeProperties == + assert(catalog.getTableMetadata(tableIdent).storage.properties == Map("k" -> "vvv", "kay" -> "veee")) // table to alter does not exist intercept[AnalysisException] { @@ -1004,7 +1004,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { convertToDatasourceTable(catalog, tableIdent) } assert(catalog.getPartition(tableIdent, spec).storage.serde.isEmpty) - assert(catalog.getPartition(tableIdent, spec).storage.serdeProperties.isEmpty) + assert(catalog.getPartition(tableIdent, spec).storage.properties.isEmpty) // set table serde and/or properties (should fail on datasource tables) if (isDatasourceTable) { val e1 = intercept[AnalysisException] { @@ -1019,25 +1019,25 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { } else { sql("ALTER TABLE dbx.tab1 PARTITION (a=1, b=2) SET SERDE 'org.apache.jadoop'") assert(catalog.getPartition(tableIdent, spec).storage.serde == Some("org.apache.jadoop")) - assert(catalog.getPartition(tableIdent, spec).storage.serdeProperties.isEmpty) + assert(catalog.getPartition(tableIdent, spec).storage.properties.isEmpty) sql("ALTER TABLE dbx.tab1 PARTITION (a=1, b=2) SET SERDE 'org.apache.madoop' " + "WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')") assert(catalog.getPartition(tableIdent, spec).storage.serde == Some("org.apache.madoop")) - assert(catalog.getPartition(tableIdent, spec).storage.serdeProperties == + assert(catalog.getPartition(tableIdent, spec).storage.properties == Map("k" -> "v", "kay" -> "vee")) } // set serde properties only maybeWrapException(isDatasourceTable) { sql("ALTER TABLE dbx.tab1 PARTITION (a=1, b=2) " + "SET SERDEPROPERTIES ('k' = 'vvv', 'kay' = 'vee')") - assert(catalog.getPartition(tableIdent, spec).storage.serdeProperties == + assert(catalog.getPartition(tableIdent, spec).storage.properties == Map("k" -> "vvv", "kay" -> "vee")) } // set things without explicitly specifying database catalog.setCurrentDatabase("dbx") maybeWrapException(isDatasourceTable) { sql("ALTER TABLE tab1 PARTITION (a=1, b=2) SET SERDEPROPERTIES ('kay' = 'veee')") - assert(catalog.getPartition(tableIdent, spec).storage.serdeProperties == + assert(catalog.getPartition(tableIdent, spec).storage.properties == Map("k" -> "vvv", "kay" -> "veee")) } // table to alter does not exist diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala index 2be51ed0e8..d308a31061 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala @@ -119,7 +119,7 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log BucketSpec(n.toInt, getColumnNames("bucket"), getColumnNames("sort")) } - val options = table.storage.serdeProperties + val options = table.storage.properties val dataSource = DataSource( sparkSession, diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala index 3ab1bdabb9..f3c849b9f2 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala @@ -103,7 +103,7 @@ private[hive] case class MetastoreRelation( sd.setSerdeInfo(serdeInfo) val serdeParameters = new java.util.HashMap[String, String]() - catalogTable.storage.serdeProperties.foreach { case (k, v) => serdeParameters.put(k, v) } + catalogTable.storage.properties.foreach { case (k, v) => serdeParameters.put(k, v) } serdeInfo.setParameters(serdeParameters) new HiveTable(tTable) @@ -173,8 +173,8 @@ private[hive] case class MetastoreRelation( p.storage.serde.foreach(serdeInfo.setSerializationLib) val serdeParameters = new java.util.HashMap[String, String]() - catalogTable.storage.serdeProperties.foreach { case (k, v) => serdeParameters.put(k, v) } - p.storage.serdeProperties.foreach { case (k, v) => serdeParameters.put(k, v) } + catalogTable.storage.properties.foreach { case (k, v) => serdeParameters.put(k, v) } + p.storage.properties.foreach { case (k, v) => serdeParameters.put(k, v) } serdeInfo.setParameters(serdeParameters) new Partition(hiveQlTable, tPartition) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 2f102a88cc..9f5782f045 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -390,7 +390,7 @@ private[hive] class HiveClientImpl( outputFormat = Option(h.getOutputFormatClass).map(_.getName), serde = Option(h.getSerializationLib), compressed = h.getTTable.getSd.isCompressed, - serdeProperties = Option(h.getTTable.getSd.getSerdeInfo.getParameters) + properties = Option(h.getTTable.getSd.getSerdeInfo.getParameters) .map(_.asScala.toMap).orNull ), properties = properties, @@ -775,7 +775,7 @@ private[hive] class HiveClientImpl( table.storage.outputFormat.map(toOutputFormat).foreach(hiveTable.setOutputFormatClass) hiveTable.setSerializationLib( table.storage.serde.getOrElse("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")) - table.storage.serdeProperties.foreach { case (k, v) => hiveTable.setSerdeParam(k, v) } + table.storage.properties.foreach { case (k, v) => hiveTable.setSerdeParam(k, v) } table.properties.foreach { case (k, v) => hiveTable.setProperty(k, v) } table.comment.foreach { c => hiveTable.setProperty("comment", c) } table.viewOriginalText.foreach { t => hiveTable.setViewOriginalText(t) } @@ -799,7 +799,7 @@ private[hive] class HiveClientImpl( p.storage.inputFormat.foreach(storageDesc.setInputFormat) p.storage.outputFormat.foreach(storageDesc.setOutputFormat) p.storage.serde.foreach(serdeInfo.setSerializationLib) - serdeInfo.setParameters(p.storage.serdeProperties.asJava) + serdeInfo.setParameters(p.storage.properties.asJava) storageDesc.setSerdeInfo(serdeInfo) tpart.setDbName(ht.getDbName) tpart.setTableName(ht.getTableName) @@ -818,7 +818,7 @@ private[hive] class HiveClientImpl( outputFormat = Option(apiPartition.getSd.getOutputFormat), serde = Option(apiPartition.getSd.getSerdeInfo.getSerializationLib), compressed = apiPartition.getSd.isCompressed, - serdeProperties = Option(apiPartition.getSd.getSerdeInfo.getParameters) + properties = Option(apiPartition.getSd.getSerdeInfo.getParameters) .map(_.asScala.toMap).orNull)) } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala index 867aadb5f5..9d99d960ac 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala @@ -100,7 +100,7 @@ class HiveDDLCommandSuite extends PlanTest { assert(desc.viewText.isEmpty) assert(desc.viewOriginalText.isEmpty) assert(desc.partitionColumns == Seq.empty[CatalogColumn]) - assert(desc.storage.serdeProperties == Map()) + assert(desc.storage.properties == Map()) assert(desc.storage.inputFormat == Some("parquet.hive.DeprecatedParquetInputFormat")) assert(desc.storage.outputFormat == Some("parquet.hive.DeprecatedParquetOutputFormat")) assert(desc.storage.serde == Some("parquet.hive.serde.ParquetHiveSerDe")) @@ -118,7 +118,7 @@ class HiveDDLCommandSuite extends PlanTest { assert(desc.schema == Seq.empty[CatalogColumn]) assert(desc.viewText == None) // TODO will be SQLText assert(desc.viewOriginalText.isEmpty) - assert(desc.storage.serdeProperties == Map()) + assert(desc.storage.properties == Map()) assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.TextInputFormat")) assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")) @@ -154,7 +154,7 @@ class HiveDDLCommandSuite extends PlanTest { assert(desc.schema == Seq.empty[CatalogColumn]) assert(desc.viewText == None) // TODO will be SQLText assert(desc.viewOriginalText.isEmpty) - assert(desc.storage.serdeProperties == Map(("serde_p1" -> "p1"), ("serde_p2" -> "p2"))) + assert(desc.storage.properties == Map(("serde_p1" -> "p1"), ("serde_p2" -> "p2"))) assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileInputFormat")) assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.RCFileOutputFormat")) assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe")) @@ -305,7 +305,7 @@ class HiveDDLCommandSuite extends PlanTest { assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")) assert(desc.storage.serde.isEmpty) - assert(desc.storage.serdeProperties.isEmpty) + assert(desc.storage.properties.isEmpty) assert(desc.properties.isEmpty) assert(desc.comment.isEmpty) } @@ -391,10 +391,10 @@ class HiveDDLCommandSuite extends PlanTest { val (desc2, _) = extractTableDesc(query2) val (desc3, _) = extractTableDesc(query3) assert(desc1.storage.serde == Some("org.apache.poof.serde.Baff")) - assert(desc1.storage.serdeProperties.isEmpty) + assert(desc1.storage.properties.isEmpty) assert(desc2.storage.serde == Some("org.apache.poof.serde.Baff")) - assert(desc2.storage.serdeProperties == Map("k1" -> "v1")) - assert(desc3.storage.serdeProperties == Map( + assert(desc2.storage.properties == Map("k1" -> "v1")) + assert(desc3.storage.properties == Map( "field.delim" -> "x", "escape.delim" -> "y", "serialization.format" -> "x", @@ -463,7 +463,7 @@ class HiveDDLCommandSuite extends PlanTest { assert(desc.storage.inputFormat == Some("winput")) assert(desc.storage.outputFormat == Some("wowput")) assert(desc.storage.serde == Some("org.apache.poof.serde.Baff")) - assert(desc.storage.serdeProperties == Map("k1" -> "v1")) + assert(desc.storage.properties == Map("k1" -> "v1")) assert(desc.properties == Map("k1" -> "v1", "k2" -> "v2")) assert(desc.comment == Some("no comment")) } @@ -479,7 +479,7 @@ class HiveDDLCommandSuite extends PlanTest { assert(desc.schema == Seq.empty[CatalogColumn]) assert(desc.viewText == Option("SELECT * FROM tab1")) assert(desc.viewOriginalText == Option("SELECT * FROM tab1")) - assert(desc.storage.serdeProperties == Map()) + assert(desc.storage.properties == Map()) assert(desc.storage.inputFormat.isEmpty) assert(desc.storage.outputFormat.isEmpty) assert(desc.storage.serde.isEmpty) @@ -505,7 +505,7 @@ class HiveDDLCommandSuite extends PlanTest { CatalogColumn("col3", null, nullable = true, None) :: Nil) assert(desc.viewText == Option("SELECT * FROM tab1")) assert(desc.viewOriginalText == Option("SELECT * FROM tab1")) - assert(desc.storage.serdeProperties == Map()) + assert(desc.storage.properties == Map()) assert(desc.storage.inputFormat.isEmpty) assert(desc.storage.outputFormat.isEmpty) assert(desc.storage.serde.isEmpty) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index af071f95e6..22f8c0f19c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -733,7 +733,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv outputFormat = None, serde = None, compressed = false, - serdeProperties = Map( + properties = Map( "path" -> sessionState.catalog.hiveDefaultTableFilePath(TableIdentifier(tableName))) ), properties = Map( @@ -1171,8 +1171,8 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv checkAnswer(table("t"), Seq(Row(1, 2, 3), Row(2, 3, 4))) val catalogTable = sharedState.externalCatalog.getTable("default", "t") // there should not be a lowercase key 'path' now - assert(catalogTable.storage.serdeProperties.get("path").isEmpty) - assert(catalogTable.storage.serdeProperties.get("PATH").isDefined) + assert(catalogTable.storage.properties.get("path").isEmpty) + assert(catalogTable.storage.properties.get("PATH").isDefined) } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala index 83f1b192f7..7ba880e476 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala @@ -29,7 +29,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle val expectedPath = spark.sharedState.externalCatalog.getDatabase(dbName).locationUri + "/" + tableName - assert(metastoreTable.storage.serdeProperties("path") === expectedPath) + assert(metastoreTable.storage.properties("path") === expectedPath) } private def getTableNames(dbName: Option[String] = None): Array[String] = { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala index a972f61e25..066c3ffaba 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala @@ -153,7 +153,7 @@ class VersionsSuite extends SparkFunSuite with Logging { outputFormat = Some(classOf[HiveIgnoreKeyTextOutputFormat[_, _]].getName), serde = Some(classOf[LazySimpleSerDe].getName()), compressed = false, - serdeProperties = Map.empty + properties = Map.empty )) } @@ -275,7 +275,7 @@ class VersionsSuite extends SparkFunSuite with Logging { outputFormat = None, serde = None, compressed = false, - serdeProperties = Map.empty) + properties = Map.empty) test(s"$version: sql create partitioned table") { client.runSqlHive("CREATE TABLE src_part (value INT) PARTITIONED BY (key1 INT, key2 INT)") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 9228242021..fb5c9948a5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -356,7 +356,7 @@ class HiveDDLSuite expectedSerdeProps.map { case (k, v) => s"'$k'='$v'" }.mkString(", ") val oldPart = catalog.getPartition(TableIdentifier("boxes"), Map("width" -> "4")) assume(oldPart.storage.serde != Some(expectedSerde), "bad test: serde was already set") - assume(oldPart.storage.serdeProperties.filterKeys(expectedSerdeProps.contains) != + assume(oldPart.storage.properties.filterKeys(expectedSerdeProps.contains) != expectedSerdeProps, "bad test: serde properties were already set") sql(s"""ALTER TABLE boxes PARTITION (width=4) | SET SERDE '$expectedSerde' @@ -364,7 +364,7 @@ class HiveDDLSuite |""".stripMargin) val newPart = catalog.getPartition(TableIdentifier("boxes"), Map("width" -> "4")) assert(newPart.storage.serde == Some(expectedSerde)) - assume(newPart.storage.serdeProperties.filterKeys(expectedSerdeProps.contains) == + assume(newPart.storage.properties.filterKeys(expectedSerdeProps.contains) == expectedSerdeProps) } |