From cc4ab37ee78c888867e773d732e2b3ed89683fe2 Mon Sep 17 00:00:00 2001 From: Hossein Date: Tue, 8 Mar 2016 17:45:15 -0800 Subject: [SPARK-13754] Keep old data source name for backwards compatibility ## Motivation CSV data source was contributed by Databricks. It is the inlined version of https://github.com/databricks/spark-csv. The data source name was `com.databricks.spark.csv`. As a result there are many tables created on older versions of spark with that name as the source. For backwards compatibility we should keep the old name. ## Proposed changes `com.databricks.spark.csv` was added to list of `backwardCompatibilityMap` in `ResolvedDataSource.scala` ## Tests A unit test was added to `CSVSuite` to parse a csv file using the old name. Author: Hossein Closes #11589 from falaki/SPARK-13754. --- .../apache/spark/sql/execution/datasources/DataSource.scala | 3 ++- .../apache/spark/sql/execution/datasources/csv/CSVSuite.scala | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'sql') diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala index e90e72dc8c..e048ee1441 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala @@ -75,7 +75,8 @@ case class DataSource( "org.apache.spark.sql.json" -> classOf[json.DefaultSource].getCanonicalName, "org.apache.spark.sql.json.DefaultSource" -> classOf[json.DefaultSource].getCanonicalName, "org.apache.spark.sql.parquet" -> classOf[parquet.DefaultSource].getCanonicalName, - "org.apache.spark.sql.parquet.DefaultSource" -> classOf[parquet.DefaultSource].getCanonicalName + "org.apache.spark.sql.parquet.DefaultSource" -> classOf[parquet.DefaultSource].getCanonicalName, + "com.databricks.spark.csv" -> classOf[csv.DefaultSource].getCanonicalName ) /** Given a provider name, look up the data source class definition. */ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala index 53027bb698..076fe5e041 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala @@ -466,4 +466,14 @@ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils { df.schema.fields.map(field => field.dataType).deep == Array(IntegerType, IntegerType, IntegerType, IntegerType).deep) } + + test("old csv data source name works") { + val cars = sqlContext + .read + .format("com.databricks.spark.csv") + .option("header", "false") + .load(testFile(carsFile)) + + verifyCars(cars, withHeader = false, checkTypes = false) + } } -- cgit v1.2.3