[SPARK-15083][WEB UI] History Server can OOM due to unlimited TaskUIData

## What changes were proposed in this pull request?

Based on #12990 by tankkyo

Since the History Server currently loads all application's data it can OOM if too many applications have a significant task count. `spark.ui.trimTasks` (default: false) can be set to true to trim tasks by `spark.ui.retainedTasks` (default: 10000)

(This is a "quick fix" to help those running into the problem until a update of how the history server loads app data can be done)

## How was this patch tested?

Manual testing and dev/run-tests

![spark-15083](https://cloud.githubusercontent.com/assets/13952758/17713694/fe82d246-63b0-11e6-9697-b87ea75ff4ef.png)

Author: Alex Bozarth <ajbozart@us.ibm.com>

Closes #14673 from ajbozarth/spark15083.

1 files changed, 65 insertions, 65 deletions

diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 11eec0b49c..96d86b7278 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -39,21 +39,21 @@
     }
   }
 }, {
-  "taskId" : 5,
-  "index" : 5,
+  "taskId" : 1,
+  "index" : 1,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.505GMT",
+  "launchTime" : "2015-05-06T13:03:06.502GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 30,
+    "executorDeserializeTime" : 31,
     "executorRunTime" : 350,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
@@ -74,26 +74,26 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3675510,
+      "writeTime" : 3934399,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 1,
-  "index" : 1,
+  "taskId" : 5,
+  "index" : 5,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.502GMT",
+  "launchTime" : "2015-05-06T13:03:06.505GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 31,
+    "executorDeserializeTime" : 30,
     "executorRunTime" : 350,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 0,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
@@ -114,22 +114,22 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3934399,
+      "writeTime" : 3675510,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 4,
-  "index" : 4,
+  "taskId" : 0,
+  "index" : 0,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.504GMT",
+  "launchTime" : "2015-05-06T13:03:06.494GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 31,
+    "executorDeserializeTime" : 32,
     "executorRunTime" : 349,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
@@ -137,7 +137,7 @@
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
-      "bytesRead" : 60488,
+      "bytesRead" : 49294,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -154,15 +154,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 83022,
+      "writeTime" : 3842811,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 7,
-  "index" : 7,
+  "taskId" : 3,
+  "index" : 3,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.506GMT",
+  "launchTime" : "2015-05-06T13:03:06.504GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
@@ -173,7 +173,7 @@
     "executorRunTime" : 349,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 0,
+    "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
@@ -194,13 +194,13 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 2579051,
+      "writeTime" : 1311694,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 3,
-  "index" : 3,
+  "taskId" : 4,
+  "index" : 4,
   "attempt" : 0,
   "launchTime" : "2015-05-06T13:03:06.504GMT",
   "executorId" : "driver",
@@ -213,7 +213,7 @@
     "executorRunTime" : 349,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 2,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
@@ -234,30 +234,30 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 1311694,
+      "writeTime" : 83022,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 0,
-  "index" : 0,
+  "taskId" : 7,
+  "index" : 7,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.494GMT",
+  "launchTime" : "2015-05-06T13:03:06.506GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 32,
+    "executorDeserializeTime" : 31,
     "executorRunTime" : 349,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
-      "bytesRead" : 49294,
+      "bytesRead" : 60488,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -274,7 +274,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3842811,
+      "writeTime" : 2579051,
       "recordsWritten" : 10
     }
   }
@@ -479,25 +479,25 @@
     }
   }
 }, {
-  "taskId" : 16,
-  "index" : 16,
+  "taskId" : 9,
+  "index" : 9,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.001GMT",
+  "launchTime" : "2015-05-06T13:03:06.915GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 10,
+    "executorDeserializeTime" : 9,
     "executorRunTime" : 84,
     "resultSize" : 2010,
-    "jvmGcTime" : 5,
+    "jvmGcTime" : 0,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
-      "bytesRead" : 70564,
+      "bytesRead" : 60489,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -514,22 +514,22 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 108320,
+      "writeTime" : 101664,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 19,
-  "index" : 19,
+  "taskId" : 16,
+  "index" : 16,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.012GMT",
+  "launchTime" : "2015-05-06T13:03:07.001GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 5,
+    "executorDeserializeTime" : 10,
     "executorRunTime" : 84,
     "resultSize" : 2010,
     "jvmGcTime" : 5,
@@ -554,30 +554,30 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95788,
+      "writeTime" : 108320,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 9,
-  "index" : 9,
+  "taskId" : 19,
+  "index" : 19,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.915GMT",
+  "launchTime" : "2015-05-06T13:03:07.012GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 9,
+    "executorDeserializeTime" : 5,
     "executorRunTime" : 84,
     "resultSize" : 2010,
-    "jvmGcTime" : 0,
+    "jvmGcTime" : 5,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
     "inputMetrics" : {
-      "bytesRead" : 60489,
+      "bytesRead" : 70564,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -594,25 +594,25 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 101664,
+      "writeTime" : 95788,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 20,
-  "index" : 20,
+  "taskId" : 14,
+  "index" : 14,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.014GMT",
+  "launchTime" : "2015-05-06T13:03:06.925GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 3,
+    "executorDeserializeTime" : 6,
     "executorRunTime" : 83,
     "resultSize" : 2010,
-    "jvmGcTime" : 5,
+    "jvmGcTime" : 0,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
@@ -634,25 +634,25 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 97716,
+      "writeTime" : 95646,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 14,
-  "index" : 14,
+  "taskId" : 20,
+  "index" : 20,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.925GMT",
+  "launchTime" : "2015-05-06T13:03:07.014GMT",
   "executorId" : "driver",
   "host" : "localhost",
   "taskLocality" : "PROCESS_LOCAL",
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 6,
+    "executorDeserializeTime" : 3,
     "executorRunTime" : 83,
     "resultSize" : 2010,
-    "jvmGcTime" : 0,
+    "jvmGcTime" : 5,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
@@ -674,7 +674,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95646,
+      "writeTime" : 97716,
       "recordsWritten" : 10
     }
   }