diff options
author | Josh Rosen <joshrosen@databricks.com> | 2015-07-01 23:06:52 -0700 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2015-07-01 23:06:52 -0700 |
commit | 377ff4c9e8942882183d94698684824e9dc9f391 (patch) | |
tree | 8f163b07c53f378cf682f58cb20773098acbb738 | |
parent | 15d41cc501f5fa7ac82c4a6741e416bb557f610a (diff) | |
download | spark-377ff4c9e8942882183d94698684824e9dc9f391.tar.gz spark-377ff4c9e8942882183d94698684824e9dc9f391.tar.bz2 spark-377ff4c9e8942882183d94698684824e9dc9f391.zip |
[SPARK-8740] [PROJECT INFRA] Support GitHub OAuth tokens in dev/merge_spark_pr.py
This commit allows `dev/merge_spark_pr.py` to use personal GitHub OAuth tokens in order to make authenticated requests. This is necessary to work around per-IP rate limiting issues.
To use a token, just set the `GITHUB_OAUTH_KEY` environment variable. You can create a personal token at https://github.com/settings/tokens; we only require `public_repo` scope.
If the script fails due to a rate-limit issue, it now logs a useful message directing the user to the OAuth token instructions.
Author: Josh Rosen <joshrosen@databricks.com>
Closes #7136 from JoshRosen/pr-merge-script-oauth-authentication and squashes the following commits:
4d011bd [Josh Rosen] Fix error message
23d92ff [Josh Rosen] Support GitHub OAuth tokens in dev/merge_spark_pr.py
-rwxr-xr-x | dev/merge_spark_pr.py | 18 |
1 files changed, 16 insertions, 2 deletions
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py index cf827ce89b..4a17d48d81 100755 --- a/dev/merge_spark_pr.py +++ b/dev/merge_spark_pr.py @@ -47,6 +47,12 @@ PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache") JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "") # ASF JIRA password JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "") +# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests +# will be unauthenticated. You should only need to configure this if you find yourself regularly +# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at +# https://github.com/settings/tokens. This script only requires the "public_repo" scope. +GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY") + GITHUB_BASE = "https://github.com/apache/spark/pull" GITHUB_API_BASE = "https://api.github.com/repos/apache/spark" @@ -58,9 +64,17 @@ BRANCH_PREFIX = "PR_TOOL" def get_json(url): try: - return json.load(urllib2.urlopen(url)) + request = urllib2.Request(url) + if GITHUB_OAUTH_KEY: + request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY) + return json.load(urllib2.urlopen(request)) except urllib2.HTTPError as e: - print "Unable to fetch URL, exiting: %s" % url + if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == '0': + print "Exceeded the GitHub API rate limit; see the instructions in " + \ + "dev/merge_spark_pr.py to configure an OAuth token for making authenticated " + \ + "GitHub requests." + else: + print "Unable to fetch URL, exiting: %s" % url sys.exit(-1) |