diff options
| author | Zhang, Liye <liye.zhang@intel.com> | 2015-04-28 12:33:48 -0700 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2015-04-28 12:36:10 -0700 |
| commit | 80098109d908b738b43d397e024756ff617d0af4 (patch) | |
| tree | 28faa4765845a3b2871797362cd5ce6986f207ea /core | |
| parent | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171 (diff) | |
| download | spark-80098109d908b738b43d397e024756ff617d0af4.tar.gz spark-80098109d908b738b43d397e024756ff617d0af4.tar.bz2 spark-80098109d908b738b43d397e024756ff617d0af4.zip | |
[SPARK-6314] [CORE] handle JsonParseException for history server
This is handled in the same way with [SPARK-6197](https://issues.apache.org/jira/browse/SPARK-6197). The result of this PR is that exception showed in history server log will be replaced by a warning, and the application that with un-complete history log file will be listed on history server webUI
Author: Zhang, Liye <liye.zhang@intel.com>
Closes #5736 from liyezhang556520/SPARK-6314 and squashes the following commits:
b8d2d88 [Zhang, Liye] handle JsonParseException for history server
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index a94ebf6e53..fb2cbbcccc 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -333,8 +333,9 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis } try { val appListener = new ApplicationEventListener + val appCompleted = isApplicationCompleted(eventLog) bus.addListener(appListener) - bus.replay(logInput, logPath.toString) + bus.replay(logInput, logPath.toString, !appCompleted) new FsApplicationHistoryInfo( logPath.getName(), appListener.appId.getOrElse(logPath.getName()), @@ -343,7 +344,7 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis appListener.endTime.getOrElse(-1L), getModificationTime(eventLog).get, appListener.sparkUser.getOrElse(NOT_STARTED), - isApplicationCompleted(eventLog)) + appCompleted) } finally { logInput.close() } |