diff options
| author | Simon Hafner <hafnersimon@gmail.com> | 2015-07-07 09:42:59 -0700 |
|---|---|---|
| committer | Shivaram Venkataraman <shivaram@cs.berkeley.edu> | 2015-07-07 09:42:59 -0700 |
| commit | 83a621a5a8f8a2991c4cfa687279589e5c623d46 (patch) | |
| tree | 3bbf0afabd9faaabbdf16c7ed4f7c4312fc0e2c6 | |
| parent | bf8b47d17b0ee2aa58a252cf6c2ddd7967334959 (diff) | |
| download | spark-83a621a5a8f8a2991c4cfa687279589e5c623d46.tar.gz spark-83a621a5a8f8a2991c4cfa687279589e5c623d46.tar.bz2 spark-83a621a5a8f8a2991c4cfa687279589e5c623d46.zip | |
[SPARK-8821] [EC2] Switched to binary mode for file reading
Otherwise the script will crash with
- Downloading boto...
Traceback (most recent call last):
File "ec2/spark_ec2.py", line 148, in <module>
setup_external_libs(external_libs)
File "ec2/spark_ec2.py", line 128, in setup_external_libs
if hashlib.md5(tar.read()).hexdigest() != lib["md5"]:
File "/usr/lib/python3.4/codecs.py", line 319, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte
In case of an utf8 env setting.
Author: Simon Hafner <hafnersimon@gmail.com>
Closes #7215 from reactormonk/branch-1.4 and squashes the following commits:
e86957a [Simon Hafner] [SPARK-8821] [EC2] Switched to binary mode
| -rwxr-xr-x | ec2/spark_ec2.py | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index 05fa47f188..91f0a24d12 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -127,7 +127,7 @@ def setup_external_libs(libs): ) with open(tgz_file_path, "wb") as tgz_file: tgz_file.write(download_stream.read()) - with open(tgz_file_path) as tar: + with open(tgz_file_path, "rb") as tar: if hashlib.md5(tar.read()).hexdigest() != lib["md5"]: print("ERROR: Got wrong md5sum for {lib}.".format(lib=lib["name"]), file=stderr) sys.exit(1) |