From 852782b83c89c358cf429e3913b71a1a6c44f27a Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 30 Dec 2016 11:16:03 +0000 Subject: [SPARK-18922][TESTS] Fix more path-related test failures on Windows ## What changes were proposed in this pull request? This PR proposes to fix the test failures due to different format of paths on Windows. Failed tests are as below: ``` ColumnExpressionSuite: - input_file_name, input_file_block_start, input_file_block_length - FileScanRDD *** FAILED *** (187 milliseconds) "file:///C:/projects/spark/target/tmp/spark-0b21b963-6cfa-411c-8d6f-e6a5e1e73bce/part-00001-c083a03a-e55e-4b05-9073-451de352d006.snappy.parquet" did not contain "C:\projects\spark\target\tmp\spark-0b21b963-6cfa-411c-8d6f-e6a5e1e73bce" (ColumnExpressionSuite.scala:545) - input_file_name, input_file_block_start, input_file_block_length - HadoopRDD *** FAILED *** (172 milliseconds) "file:/C:/projects/spark/target/tmp/spark-5d0afa94-7c2f-463b-9db9-2e8403e2bc5f/part-00000-f6530138-9ad3-466d-ab46-0eeb6f85ed0b.txt" did not contain "C:\projects\spark\target\tmp\spark-5d0afa94-7c2f-463b-9db9-2e8403e2bc5f" (ColumnExpressionSuite.scala:569) - input_file_name, input_file_block_start, input_file_block_length - NewHadoopRDD *** FAILED *** (156 milliseconds) "file:/C:/projects/spark/target/tmp/spark-a894c7df-c74d-4d19-82a2-a04744cb3766/part-00000-29674e3f-3fcf-4327-9b04-4dab1d46338d.txt" did not contain "C:\projects\spark\target\tmp\spark-a894c7df-c74d-4d19-82a2-a04744cb3766" (ColumnExpressionSuite.scala:598) ``` ``` DataStreamReaderWriterSuite: - source metadataPath *** FAILED *** (62 milliseconds) org.mockito.exceptions.verification.junit.ArgumentsAreDifferent: Argument(s) are different! Wanted: streamSourceProvider.createSource( org.apache.spark.sql.SQLContext3b04133b, "C:\projects\spark\target\tmp\streaming.metadata-b05db6ae-c8dc-4ce4-b0d9-1eb8c84876c0/sources/0", None, "org.apache.spark.sql.streaming.test", Map() ); -> at org.apache.spark.sql.streaming.test.DataStreamReaderWriterSuite$$anonfun$12.apply$mcV$sp(DataStreamReaderWriterSuite.scala:374) Actual invocation has different arguments: streamSourceProvider.createSource( org.apache.spark.sql.SQLContext3b04133b, "/C:/projects/spark/target/tmp/streaming.metadata-b05db6ae-c8dc-4ce4-b0d9-1eb8c84876c0/sources/0", None, "org.apache.spark.sql.streaming.test", Map() ); ``` ``` GlobalTempViewSuite: - CREATE GLOBAL TEMP VIEW USING *** FAILED *** (110 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-960398ba-a0a1-45f6-a59a-d98533f9f519; ``` ``` CreateTableAsSelectSuite: - CREATE TABLE USING AS SELECT *** FAILED *** (0 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - create a table, drop it and create another one with the same name *** FAILED *** (16 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - create table using as select - with partitioned by *** FAILED *** (0 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - create table using as select - with non-zero buckets *** FAILED *** (0 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string ``` ``` HiveMetadataCacheSuite: - partitioned table is cached when partition pruning is true *** FAILED *** (532 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - partitioned table is cached when partition pruning is false *** FAILED *** (297 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ``` MultiDatabaseSuite: - createExternalTable() to non-default database - with USE *** FAILED *** (954 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-0839d9a7-5e29-467a-9e3e-3e4cd618ee09; - createExternalTable() to non-default database - without USE *** FAILED *** (500 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-c7e24d73-1d8f-45e8-ab7d-53a83087aec3; - invalid database name and table names *** FAILED *** (31 milliseconds) "Path does not exist: file:/C:projectsspark arget mpspark-15a2a494-3483-4876-80e5-ec396e704b77;" did not contain "`t:a` is not a valid name for tables/databases. Valid names only contain alphabet characters, numbers and _." (MultiDatabaseSuite.scala:296) ``` ``` OrcQuerySuite: - SPARK-8501: Avoids discovery schema from empty ORC files *** FAILED *** (15 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - Verify the ORC conversion parameter: CONVERT_METASTORE_ORC *** FAILED *** (78 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - converted ORC table supports resolving mixed case field *** FAILED *** (297 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ``` HadoopFsRelationTest - JsonHadoopFsRelationSuite, OrcHadoopFsRelationSuite, ParquetHadoopFsRelationSuite, SimpleTextHadoopFsRelationSuite: - Locality support for FileScanRDD *** FAILED *** (15 milliseconds) java.lang.IllegalArgumentException: Wrong FS: file://C:\projects\spark\target\tmp\spark-383d1f13-8783-47fd-964d-9c75e5eec50f, expected: file:/// ``` ``` HiveQuerySuite: - CREATE TEMPORARY FUNCTION *** FAILED *** (0 milliseconds) java.net.MalformedURLException: For input string: "%5Cprojects%5Cspark%5Csql%5Chive%5Ctarget%5Cscala-2.11%5Ctest-classes%5CTestUDTF.jar" - ADD FILE command *** FAILED *** (500 milliseconds) java.net.URISyntaxException: Illegal character in opaque part at index 2: C:\projects\spark\sql\hive\target\scala-2.11\test-classes\data\files\v1.txt - ADD JAR command 2 *** FAILED *** (110 milliseconds) org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilessample.json; ``` ``` PruneFileSourcePartitionsSuite: - PruneFileSourcePartitions should not change the output of LogicalRelation *** FAILED *** (15 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ``` HiveCommandSuite: - LOAD DATA LOCAL *** FAILED *** (109 milliseconds) org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilesemployee.dat; - LOAD DATA *** FAILED *** (93 milliseconds) java.net.URISyntaxException: Illegal character in opaque part at index 15: C:projectsspark arget mpemployee.dat7496657117354281006.tmp - Truncate Table *** FAILED *** (78 milliseconds) org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilesemployee.dat; ``` ``` HiveExternalCatalogBackwardCompatibilitySuite: - make sure we can read table created by old version of Spark *** FAILED *** (0 milliseconds) "[/C:/projects/spark/target/tmp/]spark-0554d859-74e1-..." did not equal "[C:\projects\spark\target\tmp\]spark-0554d859-74e1-..." (HiveExternalCatalogBackwardCompatibilitySuite.scala:213) org.scalatest.exceptions.TestFailedException - make sure we can alter table location created by old version of Spark *** FAILED *** (110 milliseconds) java.net.URISyntaxException: Illegal character in opaque part at index 15: C:projectsspark arget mpspark-0e9b2c5f-49a1-4e38-a32a-c0ab1813a79f ``` ``` ExternalCatalogSuite: - create/drop/rename partitions should create/delete/rename the directory *** FAILED *** (610 milliseconds) java.net.URISyntaxException: Illegal character in opaque part at index 2: C:\projects\spark\target\tmp\spark-4c24f010-18df-437b-9fed-990c6f9adece ``` ``` SQLQuerySuite: - describe functions - temporary user defined functions *** FAILED *** (16 milliseconds) java.net.URISyntaxException: Illegal character in opaque part at index 22: C:projectssparksqlhive argetscala-2.11 est-classesTestUDTF.jar - specifying database name for a temporary table is not allowed *** FAILED *** (125 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-a34c9814-a483-43f2-be29-37f616b6df91; ``` ``` PartitionProviderCompatibilitySuite: - convert partition provider to hive with repair table *** FAILED *** (281 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-ee5fc96d-8c7d-4ebf-8571-a1d62736473e; - when partition management is enabled, new tables have partition provider hive *** FAILED *** (187 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-803ad4d6-3e8c-498d-9ca5-5cda5d9b2a48; - when partition management is disabled, new tables have no partition provider *** FAILED *** (172 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-c9fda9e2-4020-465f-8678-52cd72d0a58f; - when partition management is disabled, we preserve the old behavior even for new tables *** FAILED *** (203 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-f4a518a6-c49d-43d3-b407-0ddd76948e13; - insert overwrite partition of legacy datasource table *** FAILED *** (188 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-f4a518a6-c49d-43d3-b407-0ddd76948e79; - insert overwrite partition of new datasource table overwrites just partition *** FAILED *** (219 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-6ba3a88d-6f6c-42c5-a9f4-6d924a0616ff; - SPARK-18544 append with saveAsTable - partition management true *** FAILED *** (173 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-cd234a6d-9cb4-4d1d-9e51-854ae9543bbd; - SPARK-18635 special chars in partition values - partition management true *** FAILED *** (2 seconds, 967 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-18635 special chars in partition values - partition management false *** FAILED *** (62 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-18659 insert overwrite table with lowercase - partition management true *** FAILED *** (63 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-18544 append with saveAsTable - partition management false *** FAILED *** (266 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-18659 insert overwrite table files - partition management false *** FAILED *** (63 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-18659 insert overwrite table with lowercase - partition management false *** FAILED *** (78 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - sanity check table setup *** FAILED *** (31 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - insert into partial dynamic partitions *** FAILED *** (47 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - insert into fully dynamic partitions *** FAILED *** (62 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - insert into static partition *** FAILED *** (78 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - overwrite partial dynamic partitions *** FAILED *** (63 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - overwrite fully dynamic partitions *** FAILED *** (47 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - overwrite static partition *** FAILED *** (63 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ``` MetastoreDataSourcesSuite: - check change without refresh *** FAILED *** (203 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-00713fe4-ca04-448c-bfc7-6c5e9a2ad2a1; - drop, change, recreate *** FAILED *** (78 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-2030a21b-7d67-4385-a65b-bb5e2bed4861; - SPARK-15269 external data source table creation *** FAILED *** (78 milliseconds) org.apache.spark.sql.AnalysisException: Path does not exist: file:/C:projectsspark arget mpspark-4d50fd4a-14bc-41d6-9232-9554dd233f86; - CTAS *** FAILED *** (109 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - CTAS with IF NOT EXISTS *** FAILED *** (109 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - CTAS: persisted partitioned bucketed data source table *** FAILED *** (0 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - SPARK-15025: create datasource table with path with select *** FAILED *** (16 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - CTAS: persisted partitioned data source table *** FAILED *** (47 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string ``` ``` HiveMetastoreCatalogSuite: - Persist non-partitioned parquet relation into metastore as managed table using CTAS *** FAILED *** (16 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string - Persist non-partitioned orc relation into metastore as managed table using CTAS *** FAILED *** (16 milliseconds) java.lang.IllegalArgumentException: Can not create a Path from an empty string ``` ``` HiveUDFSuite: - SPARK-11522 select input_file_name from non-parquet table *** FAILED *** (16 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ``` QueryPartitionSuite: - SPARK-13709: reading partitioned Avro table with nested schema *** FAILED *** (250 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ``` ParquetHiveCompatibilitySuite: - simple primitives *** FAILED *** (16 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-10177 timestamp *** FAILED *** (0 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - array *** FAILED *** (16 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - map *** FAILED *** (16 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - struct *** FAILED *** (0 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); - SPARK-16344: array of struct with a single field named 'array_element' *** FAILED *** (15 milliseconds) org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:java.lang.IllegalArgumentException: Can not create a Path from an empty string); ``` ## How was this patch tested? Manually tested via AppVeyor. ``` ColumnExpressionSuite: - input_file_name, input_file_block_start, input_file_block_length - FileScanRDD (234 milliseconds) - input_file_name, input_file_block_start, input_file_block_length - HadoopRDD (235 milliseconds) - input_file_name, input_file_block_start, input_file_block_length - NewHadoopRDD (203 milliseconds) ``` ``` DataStreamReaderWriterSuite: - source metadataPath (63 milliseconds) ``` ``` GlobalTempViewSuite: - CREATE GLOBAL TEMP VIEW USING (436 milliseconds) ``` ``` CreateTableAsSelectSuite: - CREATE TABLE USING AS SELECT (171 milliseconds) - create a table, drop it and create another one with the same name (422 milliseconds) - create table using as select - with partitioned by (141 milliseconds) - create table using as select - with non-zero buckets (125 milliseconds) ``` ``` HiveMetadataCacheSuite: - partitioned table is cached when partition pruning is true (3 seconds, 211 milliseconds) - partitioned table is cached when partition pruning is false (1 second, 781 milliseconds) ``` ``` MultiDatabaseSuite: - createExternalTable() to non-default database - with USE (797 milliseconds) - createExternalTable() to non-default database - without USE (640 milliseconds) - invalid database name and table names (62 milliseconds) ``` ``` OrcQuerySuite: - SPARK-8501: Avoids discovery schema from empty ORC files (703 milliseconds) - Verify the ORC conversion parameter: CONVERT_METASTORE_ORC (750 milliseconds) - converted ORC table supports resolving mixed case field (625 milliseconds) ``` ``` HadoopFsRelationTest - JsonHadoopFsRelationSuite, OrcHadoopFsRelationSuite, ParquetHadoopFsRelationSuite, SimpleTextHadoopFsRelationSuite: - Locality support for FileScanRDD (296 milliseconds) ``` ``` HiveQuerySuite: - CREATE TEMPORARY FUNCTION (125 milliseconds) - ADD FILE command (250 milliseconds) - ADD JAR command 2 (609 milliseconds) ``` ``` PruneFileSourcePartitionsSuite: - PruneFileSourcePartitions should not change the output of LogicalRelation (359 milliseconds) ``` ``` HiveCommandSuite: - LOAD DATA LOCAL (1 second, 829 milliseconds) - LOAD DATA (1 second, 735 milliseconds) - Truncate Table (1 second, 641 milliseconds) ``` ``` HiveExternalCatalogBackwardCompatibilitySuite: - make sure we can read table created by old version of Spark (32 milliseconds) - make sure we can alter table location created by old version of Spark (125 milliseconds) - make sure we can rename table created by old version of Spark (281 milliseconds) ``` ``` ExternalCatalogSuite: - create/drop/rename partitions should create/delete/rename the directory (625 milliseconds) ``` ``` SQLQuerySuite: - describe functions - temporary user defined functions (31 milliseconds) - specifying database name for a temporary table is not allowed (390 milliseconds) ``` ``` PartitionProviderCompatibilitySuite: - convert partition provider to hive with repair table (813 milliseconds) - when partition management is enabled, new tables have partition provider hive (562 milliseconds) - when partition management is disabled, new tables have no partition provider (344 milliseconds) - when partition management is disabled, we preserve the old behavior even for new tables (422 milliseconds) - insert overwrite partition of legacy datasource table (750 milliseconds) - SPARK-18544 append with saveAsTable - partition management true (985 milliseconds) - SPARK-18635 special chars in partition values - partition management true (3 seconds, 328 milliseconds) - SPARK-18635 special chars in partition values - partition management false (2 seconds, 891 milliseconds) - SPARK-18659 insert overwrite table with lowercase - partition management true (750 milliseconds) - SPARK-18544 append with saveAsTable - partition management false (656 milliseconds) - SPARK-18659 insert overwrite table files - partition management false (922 milliseconds) - SPARK-18659 insert overwrite table with lowercase - partition management false (469 milliseconds) - sanity check table setup (937 milliseconds) - insert into partial dynamic partitions (2 seconds, 985 milliseconds) - insert into fully dynamic partitions (1 second, 937 milliseconds) - insert into static partition (1 second, 578 milliseconds) - overwrite partial dynamic partitions (7 seconds, 561 milliseconds) - overwrite fully dynamic partitions (1 second, 766 milliseconds) - overwrite static partition (1 second, 797 milliseconds) ``` ``` MetastoreDataSourcesSuite: - check change without refresh (610 milliseconds) - drop, change, recreate (437 milliseconds) - SPARK-15269 external data source table creation (297 milliseconds) - CTAS with IF NOT EXISTS (437 milliseconds) - CTAS: persisted partitioned bucketed data source table (422 milliseconds) - SPARK-15025: create datasource table with path with select (265 milliseconds) - CTAS (438 milliseconds) - CTAS with IF NOT EXISTS (469 milliseconds) - CTAS: persisted partitioned bucketed data source table (406 milliseconds) ``` ``` HiveMetastoreCatalogSuite: - Persist non-partitioned parquet relation into metastore as managed table using CTAS (406 milliseconds) - Persist non-partitioned orc relation into metastore as managed table using CTAS (313 milliseconds) ``` ``` HiveUDFSuite: - SPARK-11522 select input_file_name from non-parquet table (3 seconds, 144 milliseconds) ``` ``` QueryPartitionSuite: - SPARK-13709: reading partitioned Avro table with nested schema (1 second, 67 milliseconds) ``` ``` ParquetHiveCompatibilitySuite: - simple primitives (745 milliseconds) - SPARK-10177 timestamp (375 milliseconds) - array (407 milliseconds) - map (409 milliseconds) - struct (437 milliseconds) - SPARK-16344: array of struct with a single field named 'array_element' (391 milliseconds) ``` Author: hyukjinkwon Closes #16397 from HyukjinKwon/SPARK-18922-paths. --- ...ExternalCatalogBackwardCompatibilitySuite.scala | 45 +++++++++++++--------- .../spark/sql/hive/HiveMetadataCacheSuite.scala | 2 +- .../spark/sql/hive/HiveMetastoreCatalogSuite.scala | 4 +- .../spark/sql/hive/MetastoreDataSourcesSuite.scala | 24 +++++------- .../apache/spark/sql/hive/MultiDatabaseSuite.scala | 14 +++---- .../sql/hive/ParquetHiveCompatibilitySuite.scala | 2 +- .../hive/PartitionProviderCompatibilitySuite.scala | 12 +++--- .../spark/sql/hive/QueryPartitionSuite.scala | 8 ++-- .../sql/hive/execution/HiveCommandSuite.scala | 25 ++++++------ .../spark/sql/hive/execution/HiveQuerySuite.scala | 10 ++--- .../spark/sql/hive/execution/HiveUDFSuite.scala | 6 +-- .../execution/PruneFileSourcePartitionsSuite.scala | 2 +- .../spark/sql/hive/execution/SQLQuerySuite.scala | 8 ++-- .../apache/spark/sql/hive/orc/OrcQuerySuite.scala | 6 +-- .../spark/sql/sources/HadoopFsRelationTest.scala | 2 +- 15 files changed, 85 insertions(+), 85 deletions(-) (limited to 'sql/hive/src/test') diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala index c5753cec80..00fdfbcebb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hive import java.net.URI +import org.apache.hadoop.fs.Path import org.scalatest.BeforeAndAfterEach import org.apache.spark.sql.QueryTest @@ -39,6 +40,7 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client val tempDir = Utils.createTempDir().getCanonicalFile + val tempDirUri = tempDir.toURI.toString.stripSuffix("/") override def beforeEach(): Unit = { sql("CREATE DATABASE test_db") @@ -56,11 +58,12 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest spark.sharedState.externalCatalog.getTable("test_db", tableName) } - private def defaultTablePath(tableName: String): String = { - spark.sessionState.catalog.defaultTablePath(TableIdentifier(tableName, Some("test_db"))) + private def defaultTableURI(tableName: String): URI = { + val defaultPath = + spark.sessionState.catalog.defaultTablePath(TableIdentifier(tableName, Some("test_db"))) + new Path(defaultPath).toUri } - // Raw table metadata that are dumped from tables created by Spark 2.0. Note that, all spark // versions prior to 2.1 would generate almost same raw table metadata for a specific table. val simpleSchema = new StructType().add("i", "int") @@ -78,7 +81,7 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest identifier = TableIdentifier("tbl2", Some("test_db")), tableType = CatalogTableType.EXTERNAL, storage = CatalogStorageFormat.empty.copy( - locationUri = Some(tempDir.getAbsolutePath), + locationUri = Some(tempDirUri), inputFormat = Some("org.apache.hadoop.mapred.TextInputFormat"), outputFormat = Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")), schema = simpleSchema) @@ -128,7 +131,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest lazy val dataSourceTable = CatalogTable( identifier = TableIdentifier("tbl4", Some("test_db")), tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty.copy(properties = Map("path" -> defaultTablePath("tbl4"))), + storage = CatalogStorageFormat.empty.copy( + properties = Map("path" -> defaultTableURI("tbl4").toString)), schema = new StructType(), properties = Map( "spark.sql.sources.provider" -> "json", @@ -138,7 +142,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest lazy val hiveCompatibleDataSourceTable = CatalogTable( identifier = TableIdentifier("tbl5", Some("test_db")), tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty.copy(properties = Map("path" -> defaultTablePath("tbl5"))), + storage = CatalogStorageFormat.empty.copy( + properties = Map("path" -> defaultTableURI("tbl5").toString)), schema = simpleSchema, properties = Map( "spark.sql.sources.provider" -> "parquet", @@ -148,7 +153,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest lazy val partitionedDataSourceTable = CatalogTable( identifier = TableIdentifier("tbl6", Some("test_db")), tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat.empty.copy(properties = Map("path" -> defaultTablePath("tbl6"))), + storage = CatalogStorageFormat.empty.copy( + properties = Map("path" -> defaultTableURI("tbl6").toString)), schema = new StructType(), properties = Map( "spark.sql.sources.provider" -> "json", @@ -161,8 +167,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest identifier = TableIdentifier("tbl7", Some("test_db")), tableType = CatalogTableType.EXTERNAL, storage = CatalogStorageFormat.empty.copy( - locationUri = Some(defaultTablePath("tbl7") + "-__PLACEHOLDER__"), - properties = Map("path" -> tempDir.getAbsolutePath)), + locationUri = Some(defaultTableURI("tbl7").toString + "-__PLACEHOLDER__"), + properties = Map("path" -> tempDirUri)), schema = new StructType(), properties = Map( "spark.sql.sources.provider" -> "json", @@ -173,8 +179,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest identifier = TableIdentifier("tbl8", Some("test_db")), tableType = CatalogTableType.EXTERNAL, storage = CatalogStorageFormat.empty.copy( - locationUri = Some(tempDir.getAbsolutePath), - properties = Map("path" -> tempDir.getAbsolutePath)), + locationUri = Some(tempDirUri), + properties = Map("path" -> tempDirUri)), schema = simpleSchema, properties = Map( "spark.sql.sources.provider" -> "parquet", @@ -185,8 +191,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest identifier = TableIdentifier("tbl9", Some("test_db")), tableType = CatalogTableType.EXTERNAL, storage = CatalogStorageFormat.empty.copy( - locationUri = Some(defaultTablePath("tbl9") + "-__PLACEHOLDER__"), - properties = Map("path" -> tempDir.getAbsolutePath)), + locationUri = Some(defaultTableURI("tbl9").toString + "-__PLACEHOLDER__"), + properties = Map("path" -> tempDirUri)), schema = new StructType(), properties = Map("spark.sql.sources.provider" -> "json")) @@ -210,7 +216,8 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest if (tbl.tableType == CatalogTableType.EXTERNAL) { // trim the URI prefix val tableLocation = new URI(readBack.storage.locationUri.get).getPath - assert(tableLocation == tempDir.getAbsolutePath) + val expectedLocation = tempDir.toURI.getPath.stripSuffix("/") + assert(tableLocation == expectedLocation) } } } @@ -218,13 +225,15 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest test("make sure we can alter table location created by old version of Spark") { withTempDir { dir => for ((tbl, _) <- rawTablesAndExpectations if tbl.tableType == CatalogTableType.EXTERNAL) { - sql(s"ALTER TABLE ${tbl.identifier} SET LOCATION '${dir.getAbsolutePath}'") + val path = dir.toURI.toString.stripSuffix("/") + sql(s"ALTER TABLE ${tbl.identifier} SET LOCATION '$path'") val readBack = getTableMetadata(tbl.identifier.table) // trim the URI prefix val actualTableLocation = new URI(readBack.storage.locationUri.get).getPath - assert(actualTableLocation == dir.getAbsolutePath) + val expected = dir.toURI.getPath.stripSuffix("/") + assert(actualTableLocation == expected) } } } @@ -240,10 +249,10 @@ class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest // trim the URI prefix val actualTableLocation = new URI(readBack.storage.locationUri.get).getPath val expectedLocation = if (tbl.tableType == CatalogTableType.EXTERNAL) { - tempDir.getAbsolutePath + tempDir.toURI.getPath.stripSuffix("/") } else { // trim the URI prefix - new URI(defaultTablePath(newName)).getPath + defaultTableURI(newName).getPath } assert(actualTableLocation == expectedLocation) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala index 6e887d95c0..0c28a1b609 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala @@ -75,7 +75,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi |create external table test (id long) |partitioned by (f1 int, f2 int) |stored as parquet - |location "${dir.getAbsolutePath}"""".stripMargin) + |location "${dir.toURI}"""".stripMargin) spark.sql("msck repair table test") val df = spark.sql("select * from test") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala index 0a280b4952..16cf4d7ec6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala @@ -156,11 +156,9 @@ class DataSourceWithHiveMetastoreCatalogSuite test(s"Persist non-partitioned $provider relation into metastore as managed table using CTAS") { withTempPath { dir => withTable("t") { - val path = dir.getCanonicalPath - sql( s"""CREATE TABLE t USING $provider - |OPTIONS (path '$path') + |OPTIONS (path '${dir.toURI}') |AS SELECT 1 AS d1, "val_1" AS d2 """.stripMargin) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 0f787be0bb..2b8d4e2bb3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -179,7 +179,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv s"""CREATE TABLE jsonTable |USING org.apache.spark.sql.json |OPTIONS ( - | path '${tempDir.getCanonicalPath}' + | path '${tempDir.toURI}' |) """.stripMargin) @@ -215,7 +215,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv s"""CREATE TABLE jsonTable |USING org.apache.spark.sql.json |OPTIONS ( - | path '${tempDir.getCanonicalPath}' + | path '${tempDir.toURI}' |) """.stripMargin) @@ -232,7 +232,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv s"""CREATE TABLE jsonTable |USING org.apache.spark.sql.json |OPTIONS ( - | path '${tempDir.getCanonicalPath}' + | path '${tempDir.toURI}' |) """.stripMargin) @@ -291,7 +291,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv s"""CREATE TABLE ctasJsonTable |USING org.apache.spark.sql.json.DefaultSource |OPTIONS ( - | path '$tempPath' + | path '${tempPath.toURI}' |) AS |SELECT * FROM jsonTable """.stripMargin) @@ -307,7 +307,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv test("CTAS with IF NOT EXISTS") { withTempPath { path => - val tempPath = path.getCanonicalPath + val tempPath = path.toURI withTable("jsonTable", "ctasJsonTable") { sql( @@ -1049,11 +1049,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv test("CTAS: persisted partitioned data source table") { withTempPath { dir => withTable("t") { - val path = dir.getCanonicalPath - sql( s"""CREATE TABLE t USING PARQUET - |OPTIONS (PATH '$path') + |OPTIONS (PATH '${dir.toURI}') |PARTITIONED BY (a) |AS SELECT 1 AS a, 2 AS b """.stripMargin @@ -1119,11 +1117,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv test("CTAS: persisted partitioned bucketed data source table") { withTempPath { dir => withTable("t") { - val path = dir.getCanonicalPath - sql( s"""CREATE TABLE t USING PARQUET - |OPTIONS (PATH '$path') + |OPTIONS (PATH '${dir.toURI}') |PARTITIONED BY (a) |CLUSTERED BY (b) SORTED BY (c) INTO 2 BUCKETS |AS SELECT 1 AS a, 2 AS b, 3 AS c @@ -1289,11 +1285,9 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv test("SPARK-15025: create datasource table with path with select") { withTempPath { dir => withTable("t") { - val path = dir.getCanonicalPath - sql( s"""CREATE TABLE t USING PARQUET - |OPTIONS (PATH '$path') + |OPTIONS (PATH '${dir.toURI}') |AS SELECT 1 AS a, 2 AS b, 3 AS c """.stripMargin ) @@ -1307,7 +1301,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv test("SPARK-15269 external data source table creation") { withTempPath { dir => - val path = dir.getCanonicalPath + val path = dir.toURI.toString spark.range(1).write.json(path) withTable("t") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala index 7322465109..47ee4dd4d9 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala @@ -80,7 +80,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle |CREATE TABLE t1 |USING parquet |OPTIONS ( - | path '$path' + | path '${dir.toURI}' |) """.stripMargin) assert(getTableNames(Option(db)).contains("t1")) @@ -105,7 +105,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle |CREATE TABLE $db.t1 |USING parquet |OPTIONS ( - | path '$path' + | path '${dir.toURI}' |) """.stripMargin) assert(getTableNames(Option(db)).contains("t1")) @@ -212,7 +212,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle s"""CREATE EXTERNAL TABLE t (id BIGINT) |PARTITIONED BY (p INT) |STORED AS PARQUET - |LOCATION '$path' + |LOCATION '${dir.toURI}' """.stripMargin) checkAnswer(spark.table("t"), spark.emptyDataFrame) @@ -244,7 +244,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle s"""CREATE EXTERNAL TABLE $db.t (id BIGINT) |PARTITIONED BY (p INT) |STORED AS PARQUET - |LOCATION '$path' + |LOCATION '${dir.toURI}' """.stripMargin) checkAnswer(spark.table(s"$db.t"), spark.emptyDataFrame) @@ -280,8 +280,6 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle } withTempDir { dir => - val path = dir.getCanonicalPath - { val message = intercept[AnalysisException] { sql( @@ -289,7 +287,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle |CREATE TABLE `d:b`.`t:a` (a int) |USING parquet |OPTIONS ( - | path '$path' + | path '${dir.toURI}' |) """.stripMargin) }.getMessage @@ -304,7 +302,7 @@ class MultiDatabaseSuite extends QueryTest with SQLTestUtils with TestHiveSingle |CREATE TABLE `d:b`.`table` (a int) |USING parquet |OPTIONS ( - | path '$path' + | path '${dir.toURI}' |) """.stripMargin) }.getMessage diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala index 14266e6847..05b6059472 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetHiveCompatibilitySuite.scala @@ -45,7 +45,7 @@ class ParquetHiveCompatibilitySuite extends ParquetCompatibilityTest with TestHi private def testParquetHiveCompatibility(row: Row, hiveTypes: String*): Unit = { withTable("parquet_compat") { withTempPath { dir => - val path = dir.getCanonicalPath + val path = dir.toURI.toString // Hive columns are always nullable, so here we append a all-null row. val rows = row :: Row(Seq.fill(row.length)(null): _*) :: Nil diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala index 3f84cbdb1b..f88fc4a2ce 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala @@ -38,7 +38,7 @@ class PartitionProviderCompatibilitySuite spark.sql(s""" |create table $tableName (fieldOne long, partCol int) |using parquet - |options (path "${dir.getAbsolutePath}") + |options (path "${dir.toURI}") |partitioned by (partCol)""".stripMargin) } @@ -239,7 +239,7 @@ class PartitionProviderCompatibilitySuite // custom locations sanity check spark.sql(s""" |alter table test partition (A=0, B='%') - |set location '${dir.getAbsolutePath}'""".stripMargin) + |set location '${dir.toURI}'""".stripMargin) assert(spark.sql("select * from test").count() == 28) // moved to empty dir // rename partition sanity check @@ -315,11 +315,11 @@ class PartitionProviderCompatibilitySuite spark.sql(s""" |create table test (id long, P1 int, P2 int) |using parquet - |options (path "${base.getAbsolutePath}") + |options (path "${base.toURI}") |partitioned by (P1, P2)""".stripMargin) - spark.sql(s"alter table test add partition (P1=0, P2=0) location '${a.getAbsolutePath}'") - spark.sql(s"alter table test add partition (P1=0, P2=1) location '${b.getAbsolutePath}'") - spark.sql(s"alter table test add partition (P1=1, P2=0) location '${c.getAbsolutePath}'") + spark.sql(s"alter table test add partition (P1=0, P2=0) location '${a.toURI}'") + spark.sql(s"alter table test add partition (P1=0, P2=1) location '${b.toURI}'") + spark.sql(s"alter table test add partition (P1=1, P2=0) location '${c.toURI}'") spark.sql(s"alter table test add partition (P1=1, P2=1)") testFn diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala index feeaade561..b20c10c6a3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala @@ -40,7 +40,7 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl val tmpDir = Files.createTempDir() // create the table for test sql(s"CREATE TABLE table_with_partition(key int,value string) " + - s"PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ") + s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ") sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " + "SELECT key,value FROM testData") sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " + @@ -71,12 +71,12 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl test("SPARK-13709: reading partitioned Avro table with nested schema") { withTempDir { dir => - val path = dir.getCanonicalPath + val path = dir.toURI.toString val tableName = "spark_13709" val tempTableName = "spark_13709_temp" - new File(path, tableName).mkdir() - new File(path, tempTableName).mkdir() + new File(dir.getAbsolutePath, tableName).mkdir() + new File(dir.getAbsolutePath, tempTableName).mkdir() val avroSchema = """{ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala index 1680f6c40a..8fda1c5875 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} import org.apache.spark.sql.hive.test.TestHiveSingleton -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils import org.apache.spark.sql.types.StructType @@ -202,11 +201,12 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto // LOAD DATA INTO non-partitioned table can't specify partition intercept[AnalysisException] { - sql(s"""$loadQuery INPATH "$testData" INTO TABLE non_part_table PARTITION(ds="1")""") + sql( + s"""$loadQuery INPATH "${testData.toURI}" INTO TABLE non_part_table PARTITION(ds="1")""") } withInputFile { path => - sql(s"""$loadQuery INPATH "$path" INTO TABLE non_part_table""") + sql(s"""$loadQuery INPATH "${path.toURI}" INTO TABLE non_part_table""") // Non-local mode is expected to move the file, while local mode is expected to copy it. // Check once here that the behavior is the expected. @@ -222,7 +222,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto // // TODO: need a similar test for non-local mode. if (local) { - val incorrectUri = "file:/" + testData.getAbsolutePath() + val incorrectUri = "file://path/to/data/files/employee.dat" intercept[AnalysisException] { sql(s"""LOAD DATA LOCAL INPATH "$incorrectUri" INTO TABLE non_part_table""") } @@ -231,7 +231,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto // Use URI as inpath: // file:/path/to/data/files/employee.dat withInputFile { path => - sql(s"""$loadQuery INPATH "${path.toURI()}" INTO TABLE non_part_table""") + sql(s"""$loadQuery INPATH "${path.toURI}" INTO TABLE non_part_table""") } checkAnswer( @@ -240,7 +240,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto // Overwrite existing data. withInputFile { path => - sql(s"""$loadQuery INPATH "${path.toURI()}" OVERWRITE INTO TABLE non_part_table""") + sql(s"""$loadQuery INPATH "${path.toURI}" OVERWRITE INTO TABLE non_part_table""") } checkAnswer( @@ -257,7 +257,8 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto """.stripMargin) // LOAD DATA INTO partitioned table must specify partition - withInputFile { path => + withInputFile { f => + val path = f.toURI intercept[AnalysisException] { sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table""") } @@ -273,16 +274,16 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto } } - withInputFile { path => - sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(c="1", d="2")""") + withInputFile { f => + sql(s"""$loadQuery INPATH "${f.toURI}" INTO TABLE part_table PARTITION(c="1", d="2")""") } checkAnswer( sql("SELECT employeeID, employeeName FROM part_table WHERE c = '1' AND d = '2'"), sql("SELECT * FROM non_part_table").collect()) // Different order of partition columns. - withInputFile { path => - sql(s"""$loadQuery INPATH "$path" INTO TABLE part_table PARTITION(d="1", c="2")""") + withInputFile { f => + sql(s"""$loadQuery INPATH "${f.toURI}" INTO TABLE part_table PARTITION(d="1", c="2")""") } checkAnswer( sql("SELECT employeeID, employeeName FROM part_table WHERE c = '2' AND d = '1'"), @@ -300,7 +301,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto |LINES TERMINATED BY '\n' """.stripMargin) - val testData = hiveContext.getHiveFile("data/files/employee.dat").getCanonicalPath + val testData = hiveContext.getHiveFile("data/files/employee.dat").toURI sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE non_part_table""") checkAnswer( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 6fbbed1d47..e5b23dafcf 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -858,8 +858,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd test("ADD JAR command 2") { // this is a test case from mapjoin_addjar.q - val testJar = TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").getCanonicalPath - val testData = TestHive.getHiveFile("data/files/sample.json").getCanonicalPath + val testJar = TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").toURI + val testData = TestHive.getHiveFile("data/files/sample.json").toURI sql(s"ADD JAR $testJar") sql( """CREATE TABLE t1(a string, b string) @@ -877,8 +877,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd } test("CREATE TEMPORARY FUNCTION") { - val funcJar = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath - val jarURL = s"file://$funcJar" + val funcJar = TestHive.getHiveFile("TestUDTF.jar") + val jarURL = funcJar.toURI.toURL sql(s"ADD JAR $jarURL") sql( """CREATE TEMPORARY FUNCTION udtf_count2 AS @@ -889,7 +889,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd } test("ADD FILE command") { - val testFile = TestHive.getHiveFile("data/files/v1.txt").getCanonicalFile + val testFile = TestHive.getHiveFile("data/files/v1.txt").toURI sql(s"ADD FILE $testFile") val checkAddFileRDD = sparkContext.parallelize(1 to 2, 1).mapPartitions { _ => diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index 4098bb597b..58909ab9ea 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -428,7 +428,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { \"separatorChar\" = \",\", \"quoteChar\" = \"\\\"\", \"escapeChar\" = \"\\\\\") - LOCATION '$tempDir' + LOCATION '${tempDir.toURI}' """) val answer1 = @@ -444,7 +444,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { sql( s"""CREATE EXTERNAL TABLE external_t5 (c1 int, c2 int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' - LOCATION '$tempDir' + LOCATION '${tempDir.toURI}' """) val answer2 = @@ -460,7 +460,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { // External parquet pointing to LOCATION - val parquetLocation = tempDir + "/external_parquet" + val parquetLocation = s"${tempDir.toURI}/external_parquet" sql("SELECT 1, 2").write.parquet(parquetLocation) sql( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala index cdbc26cd5c..cd8f94b1cc 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala @@ -42,7 +42,7 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te |CREATE EXTERNAL TABLE test(i int) |PARTITIONED BY (p int) |STORED AS parquet - |LOCATION '${dir.getAbsolutePath}'""".stripMargin) + |LOCATION '${dir.toURI}'""".stripMargin) val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test") val catalogFileIndex = new CatalogFileIndex(spark, tableMeta, 0) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index e607af67f9..f65b5f4daa 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -346,7 +346,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { s""" |CREATE TEMPORARY FUNCTION udtf_count_temp |AS 'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - |USING JAR '${hiveContext.getHiveFile("TestUDTF.jar").getCanonicalPath()}' + |USING JAR '${hiveContext.getHiveFile("TestUDTF.jar").toURI}' """.stripMargin) checkKeywordsExist(sql("describe function udtf_count_temp"), @@ -1290,7 +1290,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { test("specifying database name for a temporary table is not allowed") { withTempPath { dir => - val path = dir.getCanonicalPath + val path = dir.toURI.toString val df = sparkContext.parallelize(1 to 10).map(i => (i, i.toString)).toDF("num", "str") df .write @@ -1298,7 +1298,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { .save(path) // We don't support creating a temporary table while specifying a database - val message = intercept[AnalysisException] { + intercept[AnalysisException] { spark.sql( s""" |CREATE TEMPORARY TABLE db.t @@ -1307,7 +1307,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { | path '$path' |) """.stripMargin) - }.getMessage + } // If you use backticks to quote the name then it's OK. spark.sql( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala index b8761e9de2..9fa1fb931d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcQuerySuite.scala @@ -350,7 +350,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest { spark.sql( s"""CREATE TABLE empty_orc(key INT, value STRING) |STORED AS ORC - |LOCATION '$path' + |LOCATION '${dir.toURI}' """.stripMargin) val emptyDF = Seq.empty[(Int, String)].toDF("key", "value").coalesce(1) @@ -451,7 +451,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest { s""" |CREATE TABLE dummy_orc(key INT, value STRING) |STORED AS ORC - |LOCATION '$path' + |LOCATION '${dir.toURI}' """.stripMargin) spark.sql( @@ -500,7 +500,7 @@ class OrcQuerySuite extends QueryTest with BeforeAndAfterAll with OrcTest { |create external table dummy_orc (id long, valueField long) |partitioned by (partitionValue int) |stored as orc - |location "${dir.getAbsolutePath}"""".stripMargin) + |location "${dir.toURI}"""".stripMargin) spark.sql(s"msck repair table dummy_orc") checkAnswer(spark.sql("select * from dummy_orc"), df) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala index 06566a9550..2446bed58a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala @@ -847,7 +847,7 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes "fs.file.impl.disable.cache" -> "true" ) withTempPath { dir => - val path = "file://" + dir.getCanonicalPath + val path = dir.toURI.toString val df1 = spark.range(4) df1.coalesce(1).write.mode("overwrite").options(options).format(dataSourceName).save(path) df1.coalesce(1).write.mode("append").options(options).format(dataSourceName).save(path) -- cgit v1.2.3