diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-02 23:39:28 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-02 23:39:28 -0800 |
commit | cd4ca936798f327357eff1aa27c238dbce18cb8d (patch) | |
tree | 5b48ef81d83d1ddfd824a94fe83c915778227183 | |
parent | d5daaab381358dbe61f51134dbb3d06f44044c6d (diff) | |
parent | 7aba123f0c0fd024105462b3a0b203cd357c67e9 (diff) | |
download | spark-cd4ca936798f327357eff1aa27c238dbce18cb8d.tar.gz spark-cd4ca936798f327357eff1aa27c238dbce18cb8d.tar.bz2 spark-cd4ca936798f327357eff1aa27c238dbce18cb8d.zip |
Merge pull request #436 from stephenh/removeextraloop
Once we find a split with no block, we don't have to look for more.
-rw-r--r-- | core/src/main/scala/spark/scheduler/DAGScheduler.scala | 23 |
1 files changed, 10 insertions, 13 deletions
diff --git a/core/src/main/scala/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/spark/scheduler/DAGScheduler.scala index edbfd1c45f..319eef6978 100644 --- a/core/src/main/scala/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/spark/scheduler/DAGScheduler.scala @@ -183,19 +183,16 @@ class DAGScheduler( def visit(rdd: RDD[_]) { if (!visited(rdd)) { visited += rdd - val locs = getCacheLocs(rdd) - for (p <- 0 until rdd.splits.size) { - if (locs(p) == Nil) { - for (dep <- rdd.dependencies) { - dep match { - case shufDep: ShuffleDependency[_,_] => - val mapStage = getShuffleMapStage(shufDep, stage.priority) - if (!mapStage.isAvailable) { - missing += mapStage - } - case narrowDep: NarrowDependency[_] => - visit(narrowDep.rdd) - } + if (getCacheLocs(rdd).contains(Nil)) { + for (dep <- rdd.dependencies) { + dep match { + case shufDep: ShuffleDependency[_,_] => + val mapStage = getShuffleMapStage(shufDep, stage.priority) + if (!mapStage.isAvailable) { + missing += mapStage + } + case narrowDep: NarrowDependency[_] => + visit(narrowDep.rdd) } } } |