diff options
author | hyukjinkwon <gurwls223@gmail.com> | 2017-01-21 14:08:01 +0000 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2017-01-21 14:08:01 +0000 |
commit | 6113fe78a5195d3325690703b20000bed6e9efa5 (patch) | |
tree | 66356c364c450a7bfd12a149b0bda0fc12ef5bd0 /sql | |
parent | bcdabaac93fc5527345754a9e10e6db5161007ef (diff) | |
download | spark-6113fe78a5195d3325690703b20000bed6e9efa5.tar.gz spark-6113fe78a5195d3325690703b20000bed6e9efa5.tar.bz2 spark-6113fe78a5195d3325690703b20000bed6e9efa5.zip |
[SPARK-19117][SPARK-18922][TESTS] Fix the rest of flaky, newly introduced and missed test failures on Windows
## What changes were proposed in this pull request?
**Failed tests**
```
org.apache.spark.sql.hive.execution.HiveQuerySuite:
- transform with SerDe3 *** FAILED ***
- transform with SerDe4 *** FAILED ***
```
```
org.apache.spark.sql.hive.execution.HiveDDLSuite:
- create hive serde table with new syntax *** FAILED ***
- add/drop partition with location - managed table *** FAILED ***
```
```
org.apache.spark.sql.hive.ParquetMetastoreSuite:
- Explicitly added partitions should be readable after load *** FAILED ***
- Non-partitioned table readable after load *** FAILED ***
```
**Aborted tests**
```
Exception encountered when attempting to run a suite with class name: org.apache.spark.sql.hive.execution.HiveSerDeSuite *** ABORTED *** (157 milliseconds)
org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: C:projectssparksqlhive argetscala-2.11 est-classesdatafilessales.txt;
```
**Flaky tests(failed 9ish out of 10)**
```
org.apache.spark.scheduler.SparkListenerSuite:
- local metrics *** FAILED ***
```
## How was this patch tested?
Manually tested via AppVeyor.
**Failed tests**
```
org.apache.spark.sql.hive.execution.HiveQuerySuite:
- transform with SerDe3 !!! CANCELED !!! (0 milliseconds)
- transform with SerDe4 !!! CANCELED !!! (0 milliseconds)
```
```
org.apache.spark.sql.hive.execution.HiveDDLSuite:
- create hive serde table with new syntax (1 second, 672 milliseconds)
- add/drop partition with location - managed table (2 seconds, 391 milliseconds)
```
```
org.apache.spark.sql.hive.ParquetMetastoreSuite:
- Explicitly added partitions should be readable after load (609 milliseconds)
- Non-partitioned table readable after load (344 milliseconds)
```
**Aborted tests**
```
spark.sql.hive.execution.HiveSerDeSuite:
- Read with RegexSerDe (2 seconds, 142 milliseconds)
- Read and write with LazySimpleSerDe (tab separated) (2 seconds)
- Read with AvroSerDe (1 second, 47 milliseconds)
- Read Partitioned with AvroSerDe (1 second, 422 milliseconds)
```
**Flaky tests (failed 9ish out of 10)**
```
org.apache.spark.scheduler.SparkListenerSuite:
- local metrics (4 seconds, 562 milliseconds)
```
Author: hyukjinkwon <gurwls223@gmail.com>
Closes #16586 from HyukjinKwon/set-path-appveyor.
Diffstat (limited to 'sql')
4 files changed, 12 insertions, 8 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 7f58603d32..235fbd3fc6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -222,8 +222,8 @@ class HiveDDLSuite sql( s""" |ALTER TABLE $tab ADD - |PARTITION (ds='2008-04-08', hr=11) LOCATION '$part1Path' - |PARTITION (ds='2008-04-08', hr=12) LOCATION '$part2Path' + |PARTITION (ds='2008-04-08', hr=11) LOCATION '${part1Path.toURI}' + |PARTITION (ds='2008-04-08', hr=12) LOCATION '${part2Path.toURI}' """.stripMargin) assert(dirSet.forall(dir => dir.listFiles == null || dir.listFiles.isEmpty)) @@ -1262,7 +1262,7 @@ class HiveDDLSuite s""" |CREATE TABLE t(id int) USING hive |OPTIONS(fileFormat 'orc', compression 'Zlib') - |LOCATION '${path.getCanonicalPath}' + |LOCATION '${path.toURI}' """.stripMargin) val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t")) assert(DDLUtils.isHiveTable(table)) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 75ba92cada..dd278f683a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -388,6 +388,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd } } + // Some tests suing script transformation are skipped as it requires `/bin/bash` which + // can be missing or differently located. createQueryTest("transform", "SELECT TRANSFORM (key) USING 'cat' AS (tKey) FROM src", skip = !TestUtils.testCommandAvailable("/bin/bash")) @@ -461,7 +463,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd |('serialization.last.column.takes.rest'='true') USING 'cat' AS (tKey, tValue) |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' |WITH SERDEPROPERTIES ('serialization.last.column.takes.rest'='true') FROM src; - """.stripMargin.replaceAll(System.lineSeparator(), " ")) + """.stripMargin.replaceAll(System.lineSeparator(), " "), + skip = !TestUtils.testCommandAvailable("/bin/bash")) createQueryTest("transform with SerDe4", """ @@ -470,7 +473,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd |('serialization.last.column.takes.rest'='true') USING 'cat' ROW FORMAT SERDE |'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES |('serialization.last.column.takes.rest'='true') FROM src; - """.stripMargin.replaceAll(System.lineSeparator(), " ")) + """.stripMargin.replaceAll(System.lineSeparator(), " "), + skip = !TestUtils.testCommandAvailable("/bin/bash")) createQueryTest("LIKE", "SELECT * FROM src WHERE value LIKE '%1%'") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala index b8af0b39c8..ec620c2403 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala @@ -34,7 +34,7 @@ class HiveSerDeSuite extends HiveComparisonTest with BeforeAndAfterAll { |ROW FORMAT SERDE '${classOf[RegexSerDe].getCanonicalName}' |WITH SERDEPROPERTIES ("input.regex" = "([^ ]*)\t([^ ]*)") """.stripMargin) - sql(s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/sales.txt")}' INTO TABLE sales") + sql(s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/sales.txt").toURI}' INTO TABLE sales") } // table sales is not a cache table, and will be clear after reset diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala index aa4a150a4b..e9239ea56f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala @@ -612,7 +612,7 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest { test("Explicitly added partitions should be readable after load") { withTable("test_added_partitions") { withTempDir { src => - val newPartitionDir = src.getCanonicalPath + val newPartitionDir = src.toURI.toString spark.range(2).selectExpr("cast(id as string)").toDF("a").write .mode("overwrite") .parquet(newPartitionDir) @@ -645,7 +645,7 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest { test("Non-partitioned table readable after load") { withTable("tab") { withTempDir { src => - val newPartitionDir = src.getCanonicalPath + val newPartitionDir = src.toURI.toString spark.range(2).selectExpr("cast(id as string)").toDF("a").write .mode("overwrite") .parquet(newPartitionDir) |