diff options
author | Josh Rosen <joshrosen@databricks.com> | 2016-06-13 17:06:22 -0700 |
---|---|---|
committer | Cheng Lian <lian@databricks.com> | 2016-06-13 17:06:22 -0700 |
commit | a6babca1bf76e70488ce6005ec3b8b53afc7edfd (patch) | |
tree | e648e59ee22b2678acc5a70ba3d2e1e0af6177ea | |
parent | ced8d669b359d6465c3bf476af0f68cc4db04a25 (diff) | |
download | spark-a6babca1bf76e70488ce6005ec3b8b53afc7edfd.tar.gz spark-a6babca1bf76e70488ce6005ec3b8b53afc7edfd.tar.bz2 spark-a6babca1bf76e70488ce6005ec3b8b53afc7edfd.zip |
[SPARK-15929] Fix portability of DataFrameSuite path globbing tests
The DataFrameSuite regression tests for SPARK-13774 fail in my environment because they attempt to glob over all of `/mnt` and some of the subdirectories restrictive permissions which cause the test to fail.
This patch rewrites those tests to remove all environment-specific assumptions; the tests now create their own unique temporary paths for use in the tests.
Author: Josh Rosen <joshrosen@databricks.com>
Closes #13649 from JoshRosen/SPARK-15929.
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 45 |
1 files changed, 36 insertions, 9 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 6bb0ce95c4..c8a0f7134d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import java.io.File import java.nio.charset.StandardCharsets +import java.util.UUID import scala.language.postfixOps import scala.util.Random @@ -35,6 +36,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSQLContext} import org.apache.spark.sql.test.SQLTestData.TestData2 import org.apache.spark.sql.types._ +import org.apache.spark.util.Utils class DataFrameSuite extends QueryTest with SharedSQLContext { import testImplicits._ @@ -1495,18 +1497,43 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { } test("SPARK-13774: Check error message for non existent path without globbed paths") { - val e = intercept[AnalysisException] (spark.read.format("csv"). - load("/xyz/file2", "/xyz/file21", "/abc/files555", "a")).getMessage() - assert(e.startsWith("Path does not exist")) + val uuid = UUID.randomUUID().toString + val baseDir = Utils.createTempDir() + try { + val e = intercept[AnalysisException] { + spark.read.format("csv").load( + new File(baseDir, "file").getAbsolutePath, + new File(baseDir, "file2").getAbsolutePath, + new File(uuid, "file3").getAbsolutePath, + uuid).rdd + } + assert(e.getMessage.startsWith("Path does not exist")) + } finally { + + } + } test("SPARK-13774: Check error message for not existent globbed paths") { - val e = intercept[AnalysisException] (spark.read.format("text"). - load( "/xyz/*")).getMessage() - assert(e.startsWith("Path does not exist")) + // Non-existent initial path component: + val nonExistentBasePath = "/" + UUID.randomUUID().toString + assert(!new File(nonExistentBasePath).exists()) + val e = intercept[AnalysisException] { + spark.read.format("text").load(s"$nonExistentBasePath/*") + } + assert(e.getMessage.startsWith("Path does not exist")) - val e1 = intercept[AnalysisException] (spark.read.json("/mnt/*/*-xyz.json").rdd). - getMessage() - assert(e1.startsWith("Path does not exist")) + // Existent initial path component, but no matching files: + val baseDir = Utils.createTempDir() + val childDir = Utils.createTempDir(baseDir.getAbsolutePath) + assert(childDir.exists()) + try { + val e1 = intercept[AnalysisException] { + spark.read.json(s"${baseDir.getAbsolutePath}/*/*-xyz.json").rdd + } + assert(e1.getMessage.startsWith("Path does not exist")) + } finally { + Utils.deleteRecursively(baseDir) + } } } |