diff options
| author | Yin Huai <yhuai@databricks.com> | 2015-04-04 23:26:10 +0800 |
|---|---|---|
| committer | Cheng Lian <lian@databricks.com> | 2015-04-04 23:27:16 +0800 |
| commit | eb57d4f88f3021997287498faa387d6e91e055d7 (patch) | |
| tree | 3ba028804613309caab19cd2b30d1c4759a783a2 | |
| parent | 3366af67bf35c1c753ce4f461e66574fd36d7827 (diff) | |
| download | spark-eb57d4f88f3021997287498faa387d6e91e055d7.tar.gz spark-eb57d4f88f3021997287498faa387d6e91e055d7.tar.bz2 spark-eb57d4f88f3021997287498faa387d6e91e055d7.zip | |
[SQL] Use path.makeQualified in newParquet.
Author: Yin Huai <yhuai@databricks.com>
Closes #5353 from yhuai/wrongFS and squashes the following commits:
849603b [Yin Huai] Not use deprecated method.
6d6ae34 [Yin Huai] Use path.makeQualified.
(cherry picked from commit da25c86d64ff9ce80f88186ba083f6c21dd9a568)
Signed-off-by: Cheng Lian <lian@databricks.com>
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala index ccc28bea79..d035897f19 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala @@ -268,7 +268,8 @@ private[sql] case class ParquetRelation2( // containing Parquet files (e.g. partitioned Parquet table). val baseStatuses = paths.distinct.map { p => val fs = FileSystem.get(URI.create(p), sparkContext.hadoopConfiguration) - val qualified = fs.makeQualified(new Path(p)) + val path = new Path(p) + val qualified = path.makeQualified(fs.getUri, fs.getWorkingDirectory) if (!fs.exists(qualified) && maybeSchema.isDefined) { fs.mkdirs(qualified) |