aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
authorSean Owen <sowen@cloudera.com>2015-06-30 10:07:26 -0700
committerAndrew Or <andrew@databricks.com>2015-06-30 10:07:26 -0700
commitada384b785c663392a0b69fad5bfe7a0a0584ee0 (patch)
tree35dd499745cb365719c86909be5b412baca6eea9 /core
parent689da28a53cf720ae607a1a935093612a7001615 (diff)
downloadspark-ada384b785c663392a0b69fad5bfe7a0a0584ee0.tar.gz
spark-ada384b785c663392a0b69fad5bfe7a0a0584ee0.tar.bz2
spark-ada384b785c663392a0b69fad5bfe7a0a0584ee0.zip
[SPARK-8437] [DOCS] Corrected: Using directory path without wildcard for filename slow for large number of files with wholeTextFiles and binaryFiles
Note that 'dir/*' can be more efficient in some Hadoop FS implementations that 'dir/' (now fixed scaladoc by using HTML entity for *) Author: Sean Owen <sowen@cloudera.com> Closes #7126 from srowen/SPARK-8437.2 and squashes the following commits: 7bb45da [Sean Owen] Note that 'dir/*' can be more efficient in some Hadoop FS implementations that 'dir/' (now fixed scaladoc by using HTML entity for *)
Diffstat (limited to 'core')
-rw-r--r--core/src/main/scala/org/apache/spark/SparkContext.scala8
1 files changed, 5 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b3c3bf3746..0e5a86f44e 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -831,7 +831,8 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* }}}
*
* @note Small files are preferred, large file is also allowable, but may cause bad performance.
- *
+ * @note On some filesystems, `.../path/&#42;` can be a more efficient way to read all files
+ * in a directory rather than `.../path/` or `.../path`
* @param minPartitions A suggestion value of the minimal splitting number for input data.
*/
def wholeTextFiles(
@@ -878,9 +879,10 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* (a-hdfs-path/part-nnnnn, its content)
* }}}
*
- * @param minPartitions A suggestion value of the minimal splitting number for input data.
- *
* @note Small files are preferred; very large files may cause bad performance.
+ * @note On some filesystems, `.../path/&#42;` can be a more efficient way to read all files
+ * in a directory rather than `.../path/` or `.../path`
+ * @param minPartitions A suggestion value of the minimal splitting number for input data.
*/
@Experimental
def binaryFiles(