diff options
author | Ryan Blue <blue@apache.org> | 2016-04-20 11:26:42 +0100 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-04-20 11:26:42 +0100 |
commit | a3451119d951949f24f3a4c5e33a5daea615dfed (patch) | |
tree | 5413972266a7932444e0b3ffa6fb258a00b6dfcf | |
parent | 7abe9a6578a99f4df50040d5cfe083c389c7b97f (diff) | |
download | spark-a3451119d951949f24f3a4c5e33a5daea615dfed.tar.gz spark-a3451119d951949f24f3a4c5e33a5daea615dfed.tar.bz2 spark-a3451119d951949f24f3a4c5e33a5daea615dfed.zip |
[SPARK-14679][UI] Fix UI DAG visualization OOM.
## What changes were proposed in this pull request?
The DAG visualization can cause an OOM when generating the DOT file.
This happens because clusters are not correctly deduped by a contains
check because they use the default equals implementation. This adds a
working equals implementation.
## How was this patch tested?
This adds a test suite that checks the new equals implementation.
Author: Ryan Blue <blue@apache.org>
Closes #12437 from rdblue/SPARK-14679-fix-ui-oom.
-rw-r--r-- | core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala | 18 | ||||
-rw-r--r-- | core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala | 37 |
2 files changed, 55 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala index bb6b663f1e..84ca750e1a 100644 --- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala +++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala @@ -17,6 +17,8 @@ package org.apache.spark.ui.scope +import java.util.Objects + import scala.collection.mutable import scala.collection.mutable.{ListBuffer, StringBuilder} @@ -72,6 +74,22 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String) def getCachedNodes: Seq[RDDOperationNode] = { _childNodes.filter(_.cached) ++ _childClusters.flatMap(_.getCachedNodes) } + + def canEqual(other: Any): Boolean = other.isInstanceOf[RDDOperationCluster] + + override def equals(other: Any): Boolean = other match { + case that: RDDOperationCluster => + (that canEqual this) && + _childClusters == that._childClusters && + id == that.id && + _name == that._name + case _ => false + } + + override def hashCode(): Int = { + val state = Seq(_childClusters, id, _name) + state.map(Objects.hashCode).foldLeft(0)((a, b) => 31 * a + b) + } } private[ui] object RDDOperationGraph extends Logging { diff --git a/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala b/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala new file mode 100644 index 0000000000..6ddcb5aba1 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphSuite.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ui.scope + +import org.apache.spark.SparkFunSuite + +class RDDOperationGraphSuite extends SparkFunSuite { + test("Test simple cluster equals") { + // create a 2-cluster chain with a child + val c1 = new RDDOperationCluster("1", "Bender") + val c2 = new RDDOperationCluster("2", "Hal") + c1.attachChildCluster(c2) + c1.attachChildNode(new RDDOperationNode(3, "Marvin", false, "collect!")) + + // create an equal cluster, but without the child node + val c1copy = new RDDOperationCluster("1", "Bender") + val c2copy = new RDDOperationCluster("2", "Hal") + c1copy.attachChildCluster(c2copy) + + assert(c1 == c1copy) + } +} |