diff options
author | Jey Kottalam <jey@cs.berkeley.edu> | 2013-03-10 13:54:46 -0700 |
---|---|---|
committer | Jey Kottalam <jey@cs.berkeley.edu> | 2013-06-21 12:14:16 -0400 |
commit | 40afe0d2a5562738ef2ff37ed1d448ae2d0cc927 (patch) | |
tree | 026ec3ad62e855f70eb38f633a41b2a3c365a65e /core | |
parent | 1057fccf2ac980501501cc27faaf42770a7de9a0 (diff) | |
download | spark-40afe0d2a5562738ef2ff37ed1d448ae2d0cc927.tar.gz spark-40afe0d2a5562738ef2ff37ed1d448ae2d0cc927.tar.bz2 spark-40afe0d2a5562738ef2ff37ed1d448ae2d0cc927.zip |
Add Python timing instrumentation
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/spark/api/python/PythonRDD.scala | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/core/src/main/scala/spark/api/python/PythonRDD.scala b/core/src/main/scala/spark/api/python/PythonRDD.scala index 807119ca8c..e9978d713f 100644 --- a/core/src/main/scala/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/spark/api/python/PythonRDD.scala @@ -47,6 +47,7 @@ private[spark] class PythonRDD[T: ClassManifest]( currentEnvVars.put(variable, value) } + val startTime = System.currentTimeMillis val proc = pb.start() val env = SparkEnv.get @@ -108,6 +109,17 @@ private[spark] class PythonRDD[T: ClassManifest]( val obj = new Array[Byte](length) stream.readFully(obj) obj + case -3 => + // Timing data from child + val bootTime = stream.readLong() + val initTime = stream.readLong() + val finishTime = stream.readLong() + val boot = bootTime - startTime + val init = initTime - bootTime + val finish = finishTime - initTime + val total = finishTime - startTime + logInfo("Times: total = %s, boot = %s, init = %s, finish = %s".format(total, boot, init, finish)) + read case -2 => // Signals that an exception has been thrown in python val exLength = stream.readInt() |