diff options
author | hyukjinkwon <gurwls223@gmail.com> | 2017-01-02 15:23:19 +0000 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2017-01-02 15:23:19 +0000 |
commit | 46b212602428f1f11c184c836b4e09c150d0ee30 (patch) | |
tree | b30420dbdfe979f65c390edbfe2d103572c07501 /sql/hive/src/test | |
parent | f1330b1d9e7b1d5de611e59eecae1bf0b0616d81 (diff) | |
download | spark-46b212602428f1f11c184c836b4e09c150d0ee30.tar.gz spark-46b212602428f1f11c184c836b4e09c150d0ee30.tar.bz2 spark-46b212602428f1f11c184c836b4e09c150d0ee30.zip |
[SPARK-19002][BUILD][PYTHON] Check pep8 against all Python scripts
## What changes were proposed in this pull request?
This PR proposes to check pep8 against all other Python scripts and fix the errors as below:
```bash
./dev/create-release/generate-contributors.py
./dev/create-release/releaseutils.py
./dev/create-release/translate-contributors.py
./dev/lint-python
./python/docs/epytext.py
./examples/src/main/python/mllib/decision_tree_classification_example.py
./examples/src/main/python/mllib/decision_tree_regression_example.py
./examples/src/main/python/mllib/gradient_boosting_classification_example.py
./examples/src/main/python/mllib/gradient_boosting_regression_example.py
./examples/src/main/python/mllib/linear_regression_with_sgd_example.py
./examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
./examples/src/main/python/mllib/naive_bayes_example.py
./examples/src/main/python/mllib/random_forest_classification_example.py
./examples/src/main/python/mllib/random_forest_regression_example.py
./examples/src/main/python/mllib/svm_with_sgd_example.py
./examples/src/main/python/streaming/network_wordjoinsentiments.py
./sql/hive/src/test/resources/data/scripts/cat.py
./sql/hive/src/test/resources/data/scripts/cat_error.py
./sql/hive/src/test/resources/data/scripts/doubleescapedtab.py
./sql/hive/src/test/resources/data/scripts/dumpdata_script.py
./sql/hive/src/test/resources/data/scripts/escapedcarriagereturn.py
./sql/hive/src/test/resources/data/scripts/escapednewline.py
./sql/hive/src/test/resources/data/scripts/escapedtab.py
./sql/hive/src/test/resources/data/scripts/input20_script.py
./sql/hive/src/test/resources/data/scripts/newline.py
```
## How was this patch tested?
- `./python/docs/epytext.py`
```bash
cd ./python/docs $$ make html
```
- pep8 check (Python 2.7 / Python 3.3.6)
```
./dev/lint-python
```
- `./dev/merge_spark_pr.py` (Python 2.7 only / Python 3.3.6 not working)
```bash
python -m doctest -v ./dev/merge_spark_pr.py
```
- `./dev/create-release/releaseutils.py` `./dev/create-release/generate-contributors.py` `./dev/create-release/translate-contributors.py` (Python 2.7 only / Python 3.3.6 not working)
```bash
python generate-contributors.py
python translate-contributors.py
```
- Examples (Python 2.7 / Python 3.3.6)
```bash
./bin/spark-submit examples/src/main/python/mllib/decision_tree_classification_example.py
./bin/spark-submit examples/src/main/python/mllib/decision_tree_regression_example.py
./bin/spark-submit examples/src/main/python/mllib/gradient_boosting_classification_example.py
./bin/spark-submit examples/src/main/python/mllib/gradient_boosting_regression_example.p
./bin/spark-submit examples/src/main/python/mllib/random_forest_classification_example.py
./bin/spark-submit examples/src/main/python/mllib/random_forest_regression_example.py
```
- Examples (Python 2.7 only / Python 3.3.6 not working)
```
./bin/spark-submit examples/src/main/python/mllib/linear_regression_with_sgd_example.py
./bin/spark-submit examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
./bin/spark-submit examples/src/main/python/mllib/naive_bayes_example.py
./bin/spark-submit examples/src/main/python/mllib/svm_with_sgd_example.py
```
- `sql/hive/src/test/resources/data/scripts/*.py` (Python 2.7 / Python 3.3.6 within suggested changes)
Manually tested only changed ones.
- `./dev/github_jira_sync.py` (Python 2.7 only / Python 3.3.6 not working)
Manually tested this after disabling actually adding comments and links.
And also via Jenkins tests.
Author: hyukjinkwon <gurwls223@gmail.com>
Closes #16405 from HyukjinKwon/minor-pep8.
Diffstat (limited to 'sql/hive/src/test')
9 files changed, 27 insertions, 31 deletions
diff --git a/sql/hive/src/test/resources/data/scripts/cat.py b/sql/hive/src/test/resources/data/scripts/cat.py index 2395b2cdeb..aea0362f89 100644 --- a/sql/hive/src/test/resources/data/scripts/cat.py +++ b/sql/hive/src/test/resources/data/scripts/cat.py @@ -16,14 +16,14 @@ # specific language governing permissions and limitations # under the License. # -import sys, re -import datetime +from __future__ import print_function +import sys import os -table_name=None -if os.environ.has_key('hive_streaming_tablename'): - table_name=os.environ['hive_streaming_tablename'] +table_name = None +if os.environ in 'hive_streaming_tablename': + table_name = os.environ['hive_streaming_tablename'] for line in sys.stdin: - print line - print >> sys.stderr, "dummy" + print(line) + print("dummy", file=sys.stderr) diff --git a/sql/hive/src/test/resources/data/scripts/cat_error.py b/sql/hive/src/test/resources/data/scripts/cat_error.py index 9642efec8e..dc1bccece9 100644 --- a/sql/hive/src/test/resources/data/scripts/cat_error.py +++ b/sql/hive/src/test/resources/data/scripts/cat_error.py @@ -19,6 +19,6 @@ import sys for line in sys.stdin: - print line + print(line) sys.exit(1) diff --git a/sql/hive/src/test/resources/data/scripts/doubleescapedtab.py b/sql/hive/src/test/resources/data/scripts/doubleescapedtab.py index d373067bae..ff5a8b82f4 100644 --- a/sql/hive/src/test/resources/data/scripts/doubleescapedtab.py +++ b/sql/hive/src/test/resources/data/scripts/doubleescapedtab.py @@ -19,6 +19,5 @@ import sys for line in sys.stdin: - print "1\\\\\\t2" - print "1\\\\\\\\t2" - + print("1\\\\\\t2") + print("1\\\\\\\\t2") diff --git a/sql/hive/src/test/resources/data/scripts/dumpdata_script.py b/sql/hive/src/test/resources/data/scripts/dumpdata_script.py index c96c9e529b..341a1b40e0 100644 --- a/sql/hive/src/test/resources/data/scripts/dumpdata_script.py +++ b/sql/hive/src/test/resources/data/scripts/dumpdata_script.py @@ -19,9 +19,9 @@ import sys for i in xrange(50): - for j in xrange(5): - for k in xrange(20022): - print 20000 * i + k + for j in xrange(5): + for k in xrange(20022): + print(20000 * i + k) for line in sys.stdin: - pass + pass diff --git a/sql/hive/src/test/resources/data/scripts/escapedcarriagereturn.py b/sql/hive/src/test/resources/data/scripts/escapedcarriagereturn.py index 475928a243..894cbdd139 100644 --- a/sql/hive/src/test/resources/data/scripts/escapedcarriagereturn.py +++ b/sql/hive/src/test/resources/data/scripts/escapedcarriagereturn.py @@ -19,5 +19,4 @@ import sys for line in sys.stdin: - print "1\\\\r2" - + print("1\\\\r2") diff --git a/sql/hive/src/test/resources/data/scripts/escapednewline.py b/sql/hive/src/test/resources/data/scripts/escapednewline.py index 0d5751454b..ff47fe5734 100644 --- a/sql/hive/src/test/resources/data/scripts/escapednewline.py +++ b/sql/hive/src/test/resources/data/scripts/escapednewline.py @@ -19,5 +19,4 @@ import sys for line in sys.stdin: - print "1\\\\n2" - + print("1\\\\n2") diff --git a/sql/hive/src/test/resources/data/scripts/escapedtab.py b/sql/hive/src/test/resources/data/scripts/escapedtab.py index 549c91e444..d9743eec56 100644 --- a/sql/hive/src/test/resources/data/scripts/escapedtab.py +++ b/sql/hive/src/test/resources/data/scripts/escapedtab.py @@ -19,5 +19,4 @@ import sys for line in sys.stdin: - print "1\\\\t2" - + print("1\\\\t2") diff --git a/sql/hive/src/test/resources/data/scripts/input20_script.py b/sql/hive/src/test/resources/data/scripts/input20_script.py index 40e3683dc3..08669cbf0a 100644 --- a/sql/hive/src/test/resources/data/scripts/input20_script.py +++ b/sql/hive/src/test/resources/data/scripts/input20_script.py @@ -21,10 +21,10 @@ import re line = sys.stdin.readline() x = 1 while line: - tem = sys.stdin.readline() - if line == tem: - x = x + 1 - else: - print str(x).strip()+'\t'+re.sub('\t','_',line.strip()) - line = tem - x = 1
\ No newline at end of file + tem = sys.stdin.readline() + if line == tem: + x += 1 + else: + print(str(x).strip()+'\t'+re.sub('\t', '_', line.strip())) + line = tem + x = 1 diff --git a/sql/hive/src/test/resources/data/scripts/newline.py b/sql/hive/src/test/resources/data/scripts/newline.py index 6500d900dd..59c313fcc2 100644 --- a/sql/hive/src/test/resources/data/scripts/newline.py +++ b/sql/hive/src/test/resources/data/scripts/newline.py @@ -19,6 +19,6 @@ import sys for line in sys.stdin: - print "1\\n2" - print "1\\r2" - print "1\\t2" + print("1\\n2") + print("1\\r2") + print("1\\t2") |