diff options
author | Reynold Xin <rxin@databricks.com> | 2016-02-11 13:31:13 -0800 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-02-11 13:31:13 -0800 |
commit | c86009ceb9613201b41319245526a13b1f0b5451 (patch) | |
tree | 3269951739522755071396bcca199b0149c5c2d5 | |
parent | 50fa6fd1b365d5db7e2b2c59624a365cef0d1696 (diff) | |
download | spark-c86009ceb9613201b41319245526a13b1f0b5451.tar.gz spark-c86009ceb9613201b41319245526a13b1f0b5451.tar.bz2 spark-c86009ceb9613201b41319245526a13b1f0b5451.zip |
Revert "[SPARK-13279] Remove O(n^2) operation from scheduler."
This reverts commit 50fa6fd1b365d5db7e2b2c59624a365cef0d1696.
-rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala | 15 |
1 files changed, 6 insertions, 9 deletions
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index 4b19beb43f..cf97877476 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -114,14 +114,9 @@ private[spark] class TaskSetManager( // treated as stacks, in which new tasks are added to the end of the // ArrayBuffer and removed from the end. This makes it faster to detect // tasks that repeatedly fail because whenever a task failed, it is put - // back at the head of the stack. These collections may contain duplicates - // for two reasons: - // (1): Tasks are only removed lazily; when a task is launched, it remains - // in all the pending lists except the one that it was launched from. - // (2): Tasks may be re-added to these lists multiple times as a result - // of failures. - // Duplicates are handled in dequeueTaskFromList, which ensures that a - // task hasn't already started running before launching it. + // back at the head of the stack. They are also only cleaned up lazily; + // when a task is launched, it remains in all the pending lists except + // the one that it was launched from, but gets removed from them later. private val pendingTasksForExecutor = new HashMap[String, ArrayBuffer[Int]] // Set of pending tasks for each host. Similar to pendingTasksForExecutor, @@ -186,7 +181,9 @@ private[spark] class TaskSetManager( private def addPendingTask(index: Int) { // Utility method that adds `index` to a list only if it's not already there def addTo(list: ArrayBuffer[Int]) { - list += index + if (!list.contains(index)) { + list += index + } } for (loc <- tasks(index).preferredLocations) { |