aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src/test
diff options
context:
space:
mode:
Diffstat (limited to 'sql/core/src/test')
-rw-r--r--sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileCatalogSuite.scala45
1 files changed, 44 insertions, 1 deletions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileCatalogSuite.scala
index 5c8d3226e9..fa3abd0098 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileCatalogSuite.scala
@@ -18,10 +18,12 @@
package org.apache.spark.sql.execution.datasources
import java.io.File
+import java.net.URI
+import scala.collection.mutable
import scala.language.reflectiveCalls
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileStatus, Path, RawLocalFileSystem}
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.test.SharedSQLContext
@@ -78,4 +80,45 @@ class FileCatalogSuite extends SharedSQLContext {
assert(catalog1.listLeafFiles(catalog1.paths).isEmpty)
}
}
+
+ test("SPARK-17613 - PartitioningAwareFileCatalog: base path w/o '/' at end") {
+ class MockCatalog(
+ override val paths: Seq[Path]) extends PartitioningAwareFileCatalog(spark, Map.empty, None) {
+
+ override def refresh(): Unit = {}
+
+ override def leafFiles: mutable.LinkedHashMap[Path, FileStatus] = mutable.LinkedHashMap(
+ new Path("mockFs://some-bucket/file1.json") -> new FileStatus()
+ )
+
+ override def leafDirToChildrenFiles: Map[Path, Array[FileStatus]] = Map(
+ new Path("mockFs://some-bucket/") -> Array(new FileStatus())
+ )
+
+ override def partitionSpec(): PartitionSpec = {
+ PartitionSpec.emptySpec
+ }
+ }
+
+ withSQLConf(
+ "fs.mockFs.impl" -> classOf[FakeParentPathFileSystem].getName,
+ "fs.mockFs.impl.disable.cache" -> "true") {
+ val pathWithSlash = new Path("mockFs://some-bucket/")
+ assert(pathWithSlash.getParent === null)
+ val pathWithoutSlash = new Path("mockFs://some-bucket")
+ assert(pathWithoutSlash.getParent === null)
+ val catalog1 = new MockCatalog(Seq(pathWithSlash))
+ val catalog2 = new MockCatalog(Seq(pathWithoutSlash))
+ assert(catalog1.allFiles().nonEmpty)
+ assert(catalog2.allFiles().nonEmpty)
+ }
+ }
+}
+
+class FakeParentPathFileSystem extends RawLocalFileSystem {
+ override def getScheme: String = "mockFs"
+
+ override def getUri: URI = {
+ URI.create("mockFs://some-bucket")
+ }
}