diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-01-17 17:40:55 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-01-17 17:40:55 -0800 |
commit | 54c0f9f185576e9b844fa8f81ca410f188daa51c (patch) | |
tree | 3f02f8a2697d42945d750383f37446edbfaf87b1 | |
parent | b534fd363fc0db05910c5a144fcfba2d6f228f93 (diff) | |
download | spark-54c0f9f185576e9b844fa8f81ca410f188daa51c.tar.gz spark-54c0f9f185576e9b844fa8f81ca410f188daa51c.tar.bz2 spark-54c0f9f185576e9b844fa8f81ca410f188daa51c.zip |
Fix code that assumed spark.local.dir is only a single directory
-rw-r--r-- | core/src/main/scala/spark/Utils.scala | 11 | ||||
-rw-r--r-- | core/src/main/scala/spark/broadcast/HttpBroadcast.scala | 2 |
2 files changed, 11 insertions, 2 deletions
diff --git a/core/src/main/scala/spark/Utils.scala b/core/src/main/scala/spark/Utils.scala index 0e7007459d..aeed5d2f32 100644 --- a/core/src/main/scala/spark/Utils.scala +++ b/core/src/main/scala/spark/Utils.scala @@ -134,7 +134,7 @@ private object Utils extends Logging { */ def fetchFile(url: String, targetDir: File) { val filename = url.split("/").last - val tempDir = System.getProperty("spark.local.dir", System.getProperty("java.io.tmpdir")) + val tempDir = getLocalDir val tempFile = File.createTempFile("fetchFileTemp", null, new File(tempDir)) val targetFile = new File(targetDir, filename) val uri = new URI(url) @@ -205,6 +205,15 @@ private object Utils extends Logging { } /** + * Get a temporary directory using Spark's spark.local.dir property, if set. This will always + * return a single directory, even though the spark.local.dir property might be a list of + * multiple paths. + */ + def getLocalDir: String = { + System.getProperty("spark.local.dir", System.getProperty("java.io.tmpdir")).split(',')(0) + } + + /** * Shuffle the elements of a collection into a random order, returning the * result in a new collection. Unlike scala.util.Random.shuffle, this method * uses a local random number generator, avoiding inter-thread contention. diff --git a/core/src/main/scala/spark/broadcast/HttpBroadcast.scala b/core/src/main/scala/spark/broadcast/HttpBroadcast.scala index 96dc28f12a..856a4683a9 100644 --- a/core/src/main/scala/spark/broadcast/HttpBroadcast.scala +++ b/core/src/main/scala/spark/broadcast/HttpBroadcast.scala @@ -89,7 +89,7 @@ private object HttpBroadcast extends Logging { } private def createServer() { - broadcastDir = Utils.createTempDir(System.getProperty("spark.local.dir", System.getProperty("java.io.tmpdir"))) + broadcastDir = Utils.createTempDir(Utils.getLocalDir) server = new HttpServer(broadcastDir) server.start() serverUri = server.uri |