diff options
| author | Patrick Wendell <pwendell@gmail.com> | 2013-06-26 08:54:57 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2013-06-26 08:54:57 -0700 |
| commit | a59c15a37eba0b585ba49a1b1b2b309f71e56d02 (patch) | |
| tree | f5a65ee29e889d5984a7fead844a643f5ac3b880 | |
| parent | 274193664ae13adbe5215f5dcc2be11e672557b6 (diff) | |
| download | spark-a59c15a37eba0b585ba49a1b1b2b309f71e56d02.tar.gz spark-a59c15a37eba0b585ba49a1b1b2b309f71e56d02.tar.bz2 spark-a59c15a37eba0b585ba49a1b1b2b309f71e56d02.zip | |
Adding config option for retained stages
| -rw-r--r-- | core/src/main/scala/spark/ui/jobs/JobProgressUI.scala | 2 | ||||
| -rw-r--r-- | docs/configuration.md | 11 |
2 files changed, 10 insertions, 3 deletions
diff --git a/core/src/main/scala/spark/ui/jobs/JobProgressUI.scala b/core/src/main/scala/spark/ui/jobs/JobProgressUI.scala index 027eadde3a..46ba1d31af 100644 --- a/core/src/main/scala/spark/ui/jobs/JobProgressUI.scala +++ b/core/src/main/scala/spark/ui/jobs/JobProgressUI.scala @@ -42,7 +42,7 @@ private[spark] class JobProgressUI(val sc: SparkContext) { private[spark] class JobProgressListener extends SparkListener { // How many stages to remember - val RETAINED_STAGES = 1000 + val RETAINED_STAGES = System.getProperty("spark.ui.retained_stages", "1000").toInt val activeStages = HashSet[Stage]() val completedStages = ListBuffer[Stage]() diff --git a/docs/configuration.md b/docs/configuration.md index 17fdbf04d1..00a2dd7369 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -135,9 +135,16 @@ Apart from these, the following properties are also available, and may be useful </tr> <tr> <td>spark.ui.port</td> - <td>(random)</td> + <td>33000</td> + <td> + Port for your application's dashboard, which shows memory and workload data + </td> +</tr> +<tr> + <td>spark.ui.retained_stages</td> + <td>1000</td> <td> - Port for your application's dashboard, which shows memory usage of each RDD. + How many stages the Spark UI remembers before garbage collecting. </td> </tr> <tr> |