diff options
author | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2013-06-03 12:10:00 -0700 |
---|---|---|
committer | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2013-06-03 12:10:00 -0700 |
commit | a058b0acf3e5ae41e64640feeace3d4e32f47401 (patch) | |
tree | ddad69d98aa850f6000e85d59ea2bb5f0f23fb70 | |
parent | 038cfc1a9acb32f8c17d883ea64f8cbb324ed82c (diff) | |
download | spark-a058b0acf3e5ae41e64640feeace3d4e32f47401.tar.gz spark-a058b0acf3e5ae41e64640feeace3d4e32f47401.tar.bz2 spark-a058b0acf3e5ae41e64640feeace3d4e32f47401.zip |
Delete a file for a block if it already exists.
-rw-r--r-- | core/src/main/scala/spark/storage/DiskStore.scala | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/core/src/main/scala/spark/storage/DiskStore.scala b/core/src/main/scala/spark/storage/DiskStore.scala index c7281200e7..2be5d01e31 100644 --- a/core/src/main/scala/spark/storage/DiskStore.scala +++ b/core/src/main/scala/spark/storage/DiskStore.scala @@ -195,9 +195,15 @@ private class DiskStore(blockManager: BlockManager, rootDirs: String) } private def createFile(blockId: String, allowAppendExisting: Boolean = false): File = { - val file = getFile(blockId) + var file = getFile(blockId) if (!allowAppendExisting && file.exists()) { - throw new Exception("File for block " + blockId + " already exists on disk: " + file) + // NOTE(shivaram): Delete the file if it exists. This might happen if a ShuffleMap task + // was rescheduled on the same machine as the old task ? + logWarning("File for block " + blockId + " already exists on disk: " + file + ". Deleting") + file.delete() + // Reopen the file + file = getFile(blockId) + // throw new Exception("File for block " + blockId + " already exists on disk: " + file) } file } |