aboutsummaryrefslogtreecommitdiff
path: root/python
diff options
context:
space:
mode:
authorAnant <anant.asty@gmail.com>2014-06-20 18:54:00 -0700
committerPatrick Wendell <pwendell@gmail.com>2014-06-20 18:57:24 -0700
commit010c460d627c1917dc47b09e59fd41172bbf90b3 (patch)
tree73fd45ff574c2a335198ca50d316588397eedd84 /python
parenta6786424954218fc31d8cc638a5c7d6567b55047 (diff)
downloadspark-010c460d627c1917dc47b09e59fd41172bbf90b3.tar.gz
spark-010c460d627c1917dc47b09e59fd41172bbf90b3.tar.bz2
spark-010c460d627c1917dc47b09e59fd41172bbf90b3.zip
[SPARK-2061] Made splits deprecated in JavaRDDLike
The jira for the issue can be found at: https://issues.apache.org/jira/browse/SPARK-2061 Most of spark has used over to consistently using `partitions` instead of `splits`. We should do likewise and add a `partitions` method to JavaRDDLike and have `splits` just call that. We should also go through all cases where other API's (e.g. Python) call `splits` and we should change those to use the newer API. Author: Anant <anant.asty@gmail.com> Closes #1062 from anantasty/SPARK-2061 and squashes the following commits: b83ce6b [Anant] Fixed syntax issue 21f9210 [Anant] Fixed version number in deprecation string 9315b76 [Anant] made related changes to use partitions in python api 8c62dd1 [Anant] Made splits deprecated in JavaRDDLike
Diffstat (limited to 'python')
-rw-r--r--python/pyspark/context.py2
-rw-r--r--python/pyspark/rdd.py4
2 files changed, 3 insertions, 3 deletions
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 062bec2381..95c54e7a5a 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -704,7 +704,7 @@ class SparkContext(object):
[0, 1, 16, 25]
"""
if partitions == None:
- partitions = range(rdd._jrdd.splits().size())
+ partitions = range(rdd._jrdd.partitions().size())
javaPartitions = ListConverter().convert(partitions, self._gateway._gateway_client)
# Implementation note: This is implemented as a mapPartitions followed
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index 1d55c35a8b..f64f48e3a4 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -321,7 +321,7 @@ class RDD(object):
>>> rdd.getNumPartitions()
2
"""
- return self._jrdd.splits().size()
+ return self._jrdd.partitions().size()
def filter(self, f):
"""
@@ -922,7 +922,7 @@ class RDD(object):
[91, 92, 93]
"""
items = []
- totalParts = self._jrdd.splits().size()
+ totalParts = self._jrdd.partitions().size()
partsScanned = 0
while len(items) < num and partsScanned < totalParts: