diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-09-01 18:06:15 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-09-01 18:06:15 -0700 |
commit | 6550e5e60c501cbce40f0e968fc674e499f21949 (patch) | |
tree | 1f0af3f9385021d66475dd8636449ffbaffbe9e1 /python/pyspark/worker.py | |
parent | 3c520fea7782fd24b2e30347938af9769c72c4ea (diff) | |
download | spark-6550e5e60c501cbce40f0e968fc674e499f21949.tar.gz spark-6550e5e60c501cbce40f0e968fc674e499f21949.tar.bz2 spark-6550e5e60c501cbce40f0e968fc674e499f21949.zip |
Allow PySpark to launch worker.py directly on Windows
Diffstat (limited to 'python/pyspark/worker.py')
-rw-r--r-- | python/pyspark/worker.py | 11 |
1 files changed, 7 insertions, 4 deletions
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py index 695f6dfb84..d63c2aaef7 100644 --- a/python/pyspark/worker.py +++ b/python/pyspark/worker.py @@ -21,6 +21,7 @@ Worker that receives input from Piped RDD. import os import sys import time +import socket import traceback from base64 import standard_b64decode # CloudPickler needs to be imported so that depicklers are registered using the @@ -94,7 +95,9 @@ def main(infile, outfile): if __name__ == '__main__': - # Redirect stdout to stderr so that users must return values from functions. - old_stdout = os.fdopen(os.dup(1), 'w') - os.dup2(2, 1) - main(sys.stdin, old_stdout) + # Read a local port to connect to from stdin + java_port = int(sys.stdin.readline()) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(("127.0.0.1", java_port)) + sock_file = sock.makefile("a+", 65536) + main(sock_file, sock_file) |