diff options
author | Holden Karau <holden@pigscanfly.ca> | 2015-09-22 10:19:08 -0700 |
---|---|---|
committer | Joseph K. Bradley <joseph@databricks.com> | 2015-09-22 10:19:08 -0700 |
commit | f4a3c4e34ce93bcaf29c0a35573932880a8b792b (patch) | |
tree | d2e3fb5ffe733b6272617d7b661a98a959312a53 | |
parent | 870b8a2edd44c9274c43ca0db4ef5b0998e16fd8 (diff) | |
download | spark-f4a3c4e34ce93bcaf29c0a35573932880a8b792b.tar.gz spark-f4a3c4e34ce93bcaf29c0a35573932880a8b792b.tar.bz2 spark-f4a3c4e34ce93bcaf29c0a35573932880a8b792b.zip |
[SPARK-9962] [ML] Decision Tree training: prevNodeIdsForInstances.unpersist() at end of training
NodeIdCache: prevNodeIdsForInstances.unpersist() needs to be called at end of training.
Author: Holden Karau <holden@pigscanfly.ca>
Closes #8541 from holdenk/SPARK-9962-decission-tree-training-prevNodeIdsForiNstances-unpersist-at-end-of-training.
-rw-r--r-- | mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala | 8 | ||||
-rw-r--r-- | mllib/src/main/scala/org/apache/spark/mllib/tree/impl/NodeIdCache.scala | 4 |
2 files changed, 8 insertions, 4 deletions
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala index 488e8e4fb5..c5ad8df73f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala @@ -164,10 +164,10 @@ private[spark] class NodeIdCache( } } } - } - if (prevNodeIdsForInstances != null) { - // Unpersist the previous one if one exists. - prevNodeIdsForInstances.unpersist() + if (prevNodeIdsForInstances != null) { + // Unpersist the previous one if one exists. + prevNodeIdsForInstances.unpersist() + } } } diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impl/NodeIdCache.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impl/NodeIdCache.scala index 8f9eb24b57..0abed54111 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impl/NodeIdCache.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impl/NodeIdCache.scala @@ -166,6 +166,10 @@ private[spark] class NodeIdCache( fs.delete(new Path(old.getCheckpointFile.get), true) } } + if (prevNodeIdsForInstances != null) { + // Unpersist the previous one if one exists. + prevNodeIdsForInstances.unpersist() + } } } |