diff options
author | Nicholas Chammas <nicholas.chammas@gmail.com> | 2014-09-05 23:08:54 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2014-09-05 23:08:54 -0700 |
commit | 9422c4ee0eaf4a32d2ed7c96799feac2f5f79d40 (patch) | |
tree | 53000806a143eac041be4ad0f84a137f93e43bd3 /examples/src/main/python/avro_inputformat.py | |
parent | 19f61c165932059e7ce156da2c71429fa8dc27f0 (diff) | |
download | spark-9422c4ee0eaf4a32d2ed7c96799feac2f5f79d40.tar.gz spark-9422c4ee0eaf4a32d2ed7c96799feac2f5f79d40.tar.bz2 spark-9422c4ee0eaf4a32d2ed7c96799feac2f5f79d40.zip |
[SPARK-3361] Expand PEP 8 checks to include EC2 script and Python examples
This PR resolves [SPARK-3361](https://issues.apache.org/jira/browse/SPARK-3361) by expanding the PEP 8 checks to cover the remaining Python code base:
* The EC2 script
* All Python / PySpark examples
Author: Nicholas Chammas <nicholas.chammas@gmail.com>
Closes #2297 from nchammas/pep8-rulez and squashes the following commits:
1e5ac9a [Nicholas Chammas] PEP 8 fixes to Python examples
c3dbeff [Nicholas Chammas] PEP 8 fixes to EC2 script
65ef6e8 [Nicholas Chammas] expand PEP 8 checks
Diffstat (limited to 'examples/src/main/python/avro_inputformat.py')
-rw-r--r-- | examples/src/main/python/avro_inputformat.py | 17 |
1 files changed, 11 insertions, 6 deletions
diff --git a/examples/src/main/python/avro_inputformat.py b/examples/src/main/python/avro_inputformat.py index e902ae2975..cfda8d8327 100644 --- a/examples/src/main/python/avro_inputformat.py +++ b/examples/src/main/python/avro_inputformat.py @@ -23,7 +23,8 @@ from pyspark import SparkContext Read data file users.avro in local Spark distro: $ cd $SPARK_HOME -$ ./bin/spark-submit --driver-class-path /path/to/example/jar ./examples/src/main/python/avro_inputformat.py \ +$ ./bin/spark-submit --driver-class-path /path/to/example/jar \ +> ./examples/src/main/python/avro_inputformat.py \ > examples/src/main/resources/users.avro {u'favorite_color': None, u'name': u'Alyssa', u'favorite_numbers': [3, 9, 15, 20]} {u'favorite_color': u'red', u'name': u'Ben', u'favorite_numbers': []} @@ -40,7 +41,8 @@ $ cat examples/src/main/resources/user.avsc ] } -$ ./bin/spark-submit --driver-class-path /path/to/example/jar ./examples/src/main/python/avro_inputformat.py \ +$ ./bin/spark-submit --driver-class-path /path/to/example/jar \ +> ./examples/src/main/python/avro_inputformat.py \ > examples/src/main/resources/users.avro examples/src/main/resources/user.avsc {u'favorite_color': None, u'name': u'Alyssa'} {u'favorite_color': u'red', u'name': u'Ben'} @@ -51,8 +53,10 @@ if __name__ == "__main__": Usage: avro_inputformat <data_file> [reader_schema_file] Run with example jar: - ./bin/spark-submit --driver-class-path /path/to/example/jar /path/to/examples/avro_inputformat.py <data_file> [reader_schema_file] - Assumes you have Avro data stored in <data_file>. Reader schema can be optionally specified in [reader_schema_file]. + ./bin/spark-submit --driver-class-path /path/to/example/jar \ + /path/to/examples/avro_inputformat.py <data_file> [reader_schema_file] + Assumes you have Avro data stored in <data_file>. Reader schema can be optionally specified + in [reader_schema_file]. """ exit(-1) @@ -62,9 +66,10 @@ if __name__ == "__main__": conf = None if len(sys.argv) == 3: schema_rdd = sc.textFile(sys.argv[2], 1).collect() - conf = {"avro.schema.input.key" : reduce(lambda x, y: x+y, schema_rdd)} + conf = {"avro.schema.input.key": reduce(lambda x, y: x + y, schema_rdd)} - avro_rdd = sc.newAPIHadoopFile(path, + avro_rdd = sc.newAPIHadoopFile( + path, "org.apache.avro.mapreduce.AvroKeyInputFormat", "org.apache.avro.mapred.AvroKey", "org.apache.hadoop.io.NullWritable", |