diff options
author | Denny <dennybritz@gmail.com> | 2012-10-10 16:09:53 -0700 |
---|---|---|
committer | Denny <dennybritz@gmail.com> | 2012-10-10 16:09:53 -0700 |
commit | d3f095f9043fe8c3a55e797b49e2244ee15b58ee (patch) | |
tree | 7ad80206fbbe4b69afbe8727e181cecce596db8e | |
parent | 1ac7bf893bb12a4d167141de8c61a3cd3051f87a (diff) | |
download | spark-d3f095f9043fe8c3a55e797b49e2244ee15b58ee.tar.gz spark-d3f095f9043fe8c3a55e797b49e2244ee15b58ee.tar.bz2 spark-d3f095f9043fe8c3a55e797b49e2244ee15b58ee.zip |
Fixed bug when fetching Jar dependencies.
Instead of checking currentFiles check currentJars.
-rw-r--r-- | core/src/main/scala/spark/executor/Executor.scala | 6 | ||||
-rw-r--r-- | core/src/main/scala/spark/scheduler/local/LocalScheduler.scala | 6 |
2 files changed, 6 insertions, 6 deletions
diff --git a/core/src/main/scala/spark/executor/Executor.scala b/core/src/main/scala/spark/executor/Executor.scala index 6ecf9fa8da..dfdb22024e 100644 --- a/core/src/main/scala/spark/executor/Executor.scala +++ b/core/src/main/scala/spark/executor/Executor.scala @@ -141,12 +141,12 @@ private[spark] class Executor extends Logging { private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { // Fetch missing dependencies for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) { - logInfo("Fetching " + name) + logInfo("Fetching " + name + " with timestamp " + timestamp) Utils.fetchFile(name, new File(".")) currentFiles(name) = timestamp } - for ((name, timestamp) <- newJars if currentFiles.getOrElse(name, -1L) < timestamp) { - logInfo("Fetching " + name) + for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) { + logInfo("Fetching " + name + " with timestamp " + timestamp) Utils.fetchFile(name, new File(".")) currentJars(name) = timestamp // Add it to our class loader diff --git a/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala b/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala index 2b38d8b52e..b84b4dc2ed 100644 --- a/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala +++ b/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala @@ -109,12 +109,12 @@ private[spark] class LocalScheduler(threads: Int, maxFailures: Int, sc: SparkCon private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { // Fetch missing dependencies for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) { - logInfo("Fetching " + name) + logInfo("Fetching " + name + " with timestamp " + timestamp) Utils.fetchFile(name, new File(".")) currentFiles(name) = timestamp } - for ((name, timestamp) <- newJars if currentFiles.getOrElse(name, -1L) < timestamp) { - logInfo("Fetching " + name) + for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) { + logInfo("Fetching " + name + " with timestamp " + timestamp) Utils.fetchFile(name, new File(".")) currentJars(name) = timestamp // Add it to our class loader |