diff options
| author | Liwei Lin <lwlin7@gmail.com> | 2016-04-20 11:28:51 +0100 |
|---|---|---|
| committer | Sean Owen <sowen@cloudera.com> | 2016-04-20 11:28:51 +0100 |
| commit | 17db4bfeaa0074298db622db38a5b0459518c4a9 (patch) | |
| tree | d3f8b90be7a3ca83f3e01ad47ff50865a784060b /sql | |
| parent | a3451119d951949f24f3a4c5e33a5daea615dfed (diff) | |
| download | spark-17db4bfeaa0074298db622db38a5b0459518c4a9.tar.gz spark-17db4bfeaa0074298db622db38a5b0459518c4a9.tar.bz2 spark-17db4bfeaa0074298db622db38a5b0459518c4a9.zip | |
[SPARK-14687][CORE][SQL][MLLIB] Call path.getFileSystem(conf) instead of call FileSystem.get(conf)
## What changes were proposed in this pull request?
- replaced `FileSystem.get(conf)` calls with `path.getFileSystem(conf)`
## How was this patch tested?
N/A
Author: Liwei Lin <lwlin7@gmail.com>
Closes #12450 from lw-lin/fix-fs-get.
Diffstat (limited to 'sql')
2 files changed, 2 insertions, 2 deletions
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java index 5c257bc260..b224a86845 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java @@ -178,7 +178,7 @@ public abstract class SpecificParquetRecordReaderBase<T> extends RecordReader<Vo config.set("spark.sql.parquet.writeLegacyFormat", "false"); this.file = new Path(path); - long length = FileSystem.get(config).getFileStatus(this.file).getLen(); + long length = this.file.getFileSystem(config).getFileStatus(this.file).getLen(); ParquetMetadata footer = readFooter(config, file, range(0, length)); List<BlockMetaData> blocks = footer.getBlocks(); diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala index 1b70055f34..6448cb6e90 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSource.scala @@ -39,7 +39,7 @@ class FileStreamSource( providerName: String, dataFrameBuilder: Array[String] => DataFrame) extends Source with Logging { - private val fs = FileSystem.get(sqlContext.sparkContext.hadoopConfiguration) + private val fs = new Path(path).getFileSystem(sqlContext.sparkContext.hadoopConfiguration) private val metadataLog = new HDFSMetadataLog[Seq[String]](sqlContext, metadataPath) private var maxBatchId = metadataLog.getLatest().map(_._1).getOrElse(-1L) |