diff options
author | Brennon York <brennon.york@capitalone.com> | 2015-10-18 22:45:14 -0700 |
---|---|---|
committer | Josh Rosen <joshrosen@databricks.com> | 2015-10-18 22:45:27 -0700 |
commit | d3180c25d8cf0899a7238e7d24b35c5ae918cc1d (patch) | |
tree | 851119cf665da91ea8b641ccafd9378aa0db3d90 /dev/sparktestsupport | |
parent | 94c8fef296e5cdac9a93ed34acc079e51839caa7 (diff) | |
download | spark-d3180c25d8cf0899a7238e7d24b35c5ae918cc1d.tar.gz spark-d3180c25d8cf0899a7238e7d24b35c5ae918cc1d.tar.bz2 spark-d3180c25d8cf0899a7238e7d24b35c5ae918cc1d.zip |
[SPARK-7018][BUILD] Refactor dev/run-tests-jenkins into Python
This commit refactors the `run-tests-jenkins` script into Python. This refactoring was done by brennonyork in #7401; this PR contains a few minor edits from joshrosen in order to bring it up to date with other recent changes.
From the original PR description (by brennonyork):
Currently a few things are left out that, could and I think should, be smaller JIRA's after this.
1. There are still a few areas where we use environment variables where we don't need to (like `CURRENT_BLOCK`). I might get around to fixing this one in lieu of everything else, but wanted to point that out.
2. The PR tests are still written in bash. I opted to not change those and just rewrite the runner into Python. This is a great follow-on JIRA IMO.
3. All of the linting scripts are still in bash as well and would likely do to just add those in as follow-on JIRA's as well.
Closes #7401.
Author: Brennon York <brennon.york@capitalone.com>
Closes #9161 from JoshRosen/run-tests-jenkins-refactoring.
Diffstat (limited to 'dev/sparktestsupport')
-rw-r--r-- | dev/sparktestsupport/__init__.py | 14 | ||||
-rw-r--r-- | dev/sparktestsupport/shellutils.py | 37 |
2 files changed, 49 insertions, 2 deletions
diff --git a/dev/sparktestsupport/__init__.py b/dev/sparktestsupport/__init__.py index 12696d98fb..8ab6d9e37c 100644 --- a/dev/sparktestsupport/__init__.py +++ b/dev/sparktestsupport/__init__.py @@ -19,3 +19,17 @@ import os SPARK_HOME = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../")) USER_HOME = os.environ.get("HOME") +ERROR_CODES = { + "BLOCK_GENERAL": 10, + "BLOCK_RAT": 11, + "BLOCK_SCALA_STYLE": 12, + "BLOCK_PYTHON_STYLE": 13, + "BLOCK_R_STYLE": 14, + "BLOCK_DOCUMENTATION": 15, + "BLOCK_BUILD": 16, + "BLOCK_MIMA": 17, + "BLOCK_SPARK_UNIT_TESTS": 18, + "BLOCK_PYSPARK_UNIT_TESTS": 19, + "BLOCK_SPARKR_UNIT_TESTS": 20, + "BLOCK_TIMEOUT": 124 +} diff --git a/dev/sparktestsupport/shellutils.py b/dev/sparktestsupport/shellutils.py index 12bd0bf3a4..d280e79707 100644 --- a/dev/sparktestsupport/shellutils.py +++ b/dev/sparktestsupport/shellutils.py @@ -22,6 +22,36 @@ import subprocess import sys +if sys.version_info >= (2, 7): + subprocess_check_output = subprocess.check_output + subprocess_check_call = subprocess.check_call +else: + # SPARK-8763 + # backported from subprocess module in Python 2.7 + def subprocess_check_output(*popenargs, **kwargs): + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd, output=output) + return output + + # backported from subprocess module in Python 2.7 + def subprocess_check_call(*popenargs, **kwargs): + retcode = call(*popenargs, **kwargs) + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd) + return 0 + + def exit_from_command_with_retcode(cmd, retcode): print("[error] running", ' '.join(cmd), "; received return code", retcode) sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) @@ -39,7 +69,7 @@ def rm_r(path): os.remove(path) -def run_cmd(cmd): +def run_cmd(cmd, return_output=False): """ Given a command as a list of arguments will attempt to execute the command and, on failure, print an error message and exit. @@ -48,7 +78,10 @@ def run_cmd(cmd): if not isinstance(cmd, list): cmd = cmd.split() try: - subprocess.check_call(cmd) + if return_output: + return subprocess_check_output(cmd) + else: + return subprocess_check_call(cmd) except subprocess.CalledProcessError as e: exit_from_command_with_retcode(e.cmd, e.returncode) |