blob: a7c56c237199b2cb98366219282d02bcc521389b (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
package spark.executor
class TaskMetrics extends Serializable {
/**
* Time taken on the executor to deserialize this task
*/
var executorDeserializeTime: Int = _
/**
* Time the executor spends actually running the task (including fetching shuffle data)
*/
var executorRunTime:Int = _
/**
* The number of bytes this task transmitted back to the driver as the TaskResult
*/
var resultSize: Long = _
/**
* If this task reads from shuffle output, metrics on getting shuffle data will be collected here
*/
var shuffleReadMetrics: Option[ShuffleReadMetrics] = None
/**
* If this task writes to shuffle output, metrics on the written shuffle data will be collected here
*/
var shuffleWriteMetrics: Option[ShuffleWriteMetrics] = None
}
object TaskMetrics {
private[spark] def empty(): TaskMetrics = new TaskMetrics
}
class ShuffleReadMetrics extends Serializable {
/**
* Total number of blocks fetched in a shuffle (remote or local)
*/
var totalBlocksFetched : Int = _
/**
* Number of remote blocks fetched in a shuffle
*/
var remoteBlocksFetched: Int = _
/**
* Local blocks fetched in a shuffle
*/
var localBlocksFetched: Int = _
/**
* Total time that is spent blocked waiting for shuffle to fetch data
*/
var fetchWaitTime: Long = _
/**
* The total amount of time for all the shuffle fetches. This adds up time from overlapping
* shuffles, so can be longer than task time
*/
var remoteFetchTime: Long = _
/**
* Total number of remote bytes read from a shuffle
*/
var remoteBytesRead: Long = _
}
class ShuffleWriteMetrics extends Serializable {
/**
* Number of bytes written for a shuffle
*/
var shuffleBytesWritten: Long = _
}
|
