diff options
author | Wenchen Fan <wenchen@databricks.com> | 2016-04-18 15:17:29 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-04-18 15:17:29 -0700 |
commit | 602734084c05a79c18446a3c2c051740dba143b3 (patch) | |
tree | 5cddac1f7a8add17ec705205e9cc5001d3d63d69 /core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json | |
parent | 6ff0435858eed8310c0298ef0394053dfe06df9e (diff) | |
download | spark-602734084c05a79c18446a3c2c051740dba143b3.tar.gz spark-602734084c05a79c18446a3c2c051740dba143b3.tar.bz2 spark-602734084c05a79c18446a3c2c051740dba143b3.zip |
[SPARK-14628][CORE][FOLLLOW-UP] Always tracking read/write metrics
## What changes were proposed in this pull request?
This PR is a follow up for https://github.com/apache/spark/pull/12417, now we always track input/output/shuffle metrics in spark JSON protocol and status API.
Most of the line changes are because of re-generating the gold answer for `HistoryServerSuite`, and we add a lot of 0 values for read/write metrics.
## How was this patch tested?
existing tests.
Author: Wenchen Fan <wenchen@databricks.com>
Closes #12462 from cloud-fan/follow.
Diffstat (limited to 'core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json')
-rw-r--r-- | core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json | 242 |
1 files changed, 241 insertions, 1 deletions
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json index f2cb29b31c..8e09aabbad 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json @@ -20,6 +20,18 @@ "bytesRead" : 49294, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3842811, @@ -48,6 +60,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3934399, @@ -76,6 +100,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89885, @@ -104,6 +140,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 1311694, @@ -132,6 +180,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 83022, @@ -160,6 +220,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3675510, @@ -188,6 +260,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 4016617, @@ -216,6 +300,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 2579051, @@ -244,6 +340,18 @@ "bytesRead" : 60488, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 121551, @@ -272,6 +380,18 @@ "bytesRead" : 60489, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 101664, @@ -300,6 +420,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94709, @@ -328,6 +460,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94507, @@ -356,6 +500,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102476, @@ -384,6 +540,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95004, @@ -412,6 +580,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95646, @@ -440,6 +620,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 602780, @@ -468,6 +660,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108320, @@ -496,6 +700,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 99944, @@ -524,6 +740,18 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100836, @@ -552,10 +780,22 @@ "bytesRead" : 70564, "recordsRead" : 10000 }, + "outputMetrics" : { + "bytesWritten" : 0, + "recordsWritten" : 0 + }, + "shuffleReadMetrics" : { + "remoteBlocksFetched" : 0, + "localBlocksFetched" : 0, + "fetchWaitTime" : 0, + "remoteBytesRead" : 0, + "localBytesRead" : 0, + "recordsRead" : 0 + }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95788, "recordsWritten" : 10 } } -} ]
\ No newline at end of file +} ] |