diff options
author | Prabin Banka <prabin.banka@imaginea.com> | 2014-03-06 12:45:27 -0800 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-03-06 12:45:27 -0800 |
commit | 3d3acef0474b6dc21f1b470ea96079a491e58b75 (patch) | |
tree | 97ebf321dc9d58032599d1216fa7f675f0daa287 /python/pyspark/context.py | |
parent | 3eb009f362993dbe43028419c2d48011111a200d (diff) | |
download | spark-3d3acef0474b6dc21f1b470ea96079a491e58b75.tar.gz spark-3d3acef0474b6dc21f1b470ea96079a491e58b75.tar.bz2 spark-3d3acef0474b6dc21f1b470ea96079a491e58b75.zip |
SPARK-1187, Added missing Python APIs
The following Python APIs are added,
RDD.id()
SparkContext.setJobGroup()
SparkContext.setLocalProperty()
SparkContext.getLocalProperty()
SparkContext.sparkUser()
was raised earlier as a part of apache/incubator-spark#486
Author: Prabin Banka <prabin.banka@imaginea.com>
Closes #75 from prabinb/python-api-backup and squashes the following commits:
cc3c6cd [Prabin Banka] Added missing Python APIs
Diffstat (limited to 'python/pyspark/context.py')
-rw-r--r-- | python/pyspark/context.py | 31 |
1 files changed, 31 insertions, 0 deletions
diff --git a/python/pyspark/context.py b/python/pyspark/context.py index 93faa2e385..c9f42d3aac 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -372,6 +372,37 @@ class SparkContext(object): return newStorageLevel(storageLevel.useDisk, storageLevel.useMemory, storageLevel.deserialized, storageLevel.replication) + def setJobGroup(self, groupId, description): + """ + Assigns a group ID to all the jobs started by this thread until the group ID is set to a + different value or cleared. + + Often, a unit of execution in an application consists of multiple Spark actions or jobs. + Application programmers can use this method to group all those jobs together and give a + group description. Once set, the Spark web UI will associate such jobs with this group. + """ + self._jsc.setJobGroup(groupId, description) + + def setLocalProperty(self, key, value): + """ + Set a local property that affects jobs submitted from this thread, such as the + Spark fair scheduler pool. + """ + self._jsc.setLocalProperty(key, value) + + def getLocalProperty(self, key): + """ + Get a local property set in this thread, or null if it is missing. See + L{setLocalProperty} + """ + return self._jsc.getLocalProperty(key) + + def sparkUser(self): + """ + Get SPARK_USER for user who is running SparkContext. + """ + return self._jsc.sc().sparkUser() + def _test(): import atexit import doctest |