From fd833e7ab1bf006c5ae1dff4767b02729e1bbfa7 Mon Sep 17 00:00:00 2001
From: Yinan Li <liyinan926@gmail.com>
Date: Fri, 17 Jan 2014 17:27:25 -0800
Subject: Allow files added through SparkContext.addFile() to be overwritten

This is useful for the cases when a file needs to be refreshed and downloaded
by the executors periodically.

Signed-off-by: Yinan Li <liyinan926@gmail.com>
---
 docs/configuration.md | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'docs')

diff --git a/docs/configuration.md b/docs/configuration.md
index da70cabba2..3b565e4347 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -431,6 +431,7 @@ Apart from these, the following properties are also available, and may be useful
   </td>
 </tr>
 <tr>
+<<<<<<< HEAD
   <td>spark.logConf</td>
   <td>false</td>
   <td>
@@ -459,6 +460,13 @@ Apart from these, the following properties are also available, and may be useful
     the whole cluster by default. <br/>
     <b>Note:</b> this setting needs to be configured in the standalone cluster master, not in individual
     applications; you can set it through <code>SPARK_JAVA_OPTS</code> in <code>spark-env.sh</code>.
+</td>
+</tr>
+<tr>
+  <td>spark.files.overwrite</td>
+  <td>false</td>
+  <td>
+    Whether to overwrite files added through SparkContext.addFile() when the target file exists and its contents do not match those of the source.
   </td>
 </tr>
 </table>
-- 
cgit v1.2.3


From 584323c6b13e0d4624eb39360d0caff6c8232aac Mon Sep 17 00:00:00 2001
From: Yinan Li <liyinan926@gmail.com>
Date: Sat, 18 Jan 2014 21:28:17 -0800
Subject: Addressed comments from Reynold

Signed-off-by: Yinan Li <liyinan926@gmail.com>
---
 core/src/main/scala/org/apache/spark/util/Utils.scala | 2 +-
 docs/configuration.md                                 | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

(limited to 'docs')

diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index e1f8e9520c..f600c7001a 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -268,7 +268,7 @@ private[spark] object Utils extends Logging {
     val tempFile =  File.createTempFile("fetchFileTemp", null, new File(tempDir))
     val targetFile = new File(targetDir, filename)
     val uri = new URI(url)
-    val fileOverwrite = System.getProperty("spark.files.overwrite", "false").toBoolean
+    val fileOverwrite = conf.getBoolean("spark.files.overwrite", false)
     uri.getScheme match {
       case "http" | "https" | "ftp" =>
         logInfo("Fetching " + url + " to " + tempFile)
diff --git a/docs/configuration.md b/docs/configuration.md
index 3b565e4347..57e47bf1b9 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -431,7 +431,6 @@ Apart from these, the following properties are also available, and may be useful
   </td>
 </tr>
 <tr>
-<<<<<<< HEAD
   <td>spark.logConf</td>
   <td>false</td>
   <td>
-- 
cgit v1.2.3