aboutsummaryrefslogtreecommitdiff
path: root/python/pyspark/sql.py
diff options
context:
space:
mode:
authorDavies Liu <davies.liu@gmail.com>2014-09-06 16:12:29 -0700
committerJosh Rosen <joshrosen@apache.org>2014-09-06 16:12:29 -0700
commit110fb8b24d2454ad7c979c3934dbed87650f17b8 (patch)
tree0d3e49877f108d58557d2755b7acfbefa75edc0e /python/pyspark/sql.py
parent21a1e1bb893512b2f68598ab0c0ec8c33e8d9909 (diff)
downloadspark-110fb8b24d2454ad7c979c3934dbed87650f17b8.tar.gz
spark-110fb8b24d2454ad7c979c3934dbed87650f17b8.tar.bz2
spark-110fb8b24d2454ad7c979c3934dbed87650f17b8.zip
[SPARK-2334] fix AttributeError when call PipelineRDD.id()
The underline JavaRDD for PipelineRDD is created lazily, it's delayed until call _jrdd. The id of JavaRDD is cached as `_id`, it saves a RPC call in py4j for later calls. closes #1276 Author: Davies Liu <davies.liu@gmail.com> Closes #2296 from davies/id and squashes the following commits: e197958 [Davies Liu] fix style 9721716 [Davies Liu] fix id of PipelineRDD
Diffstat (limited to 'python/pyspark/sql.py')
-rw-r--r--python/pyspark/sql.py9
1 files changed, 5 insertions, 4 deletions
diff --git a/python/pyspark/sql.py b/python/pyspark/sql.py
index 97a51b9f8a..004d4937cb 100644
--- a/python/pyspark/sql.py
+++ b/python/pyspark/sql.py
@@ -1525,7 +1525,7 @@ class SchemaRDD(RDD):
self.sql_ctx = sql_ctx
self._sc = sql_ctx._sc
self._jschema_rdd = jschema_rdd
-
+ self._id = None
self.is_cached = False
self.is_checkpointed = False
self.ctx = self.sql_ctx._sc
@@ -1543,9 +1543,10 @@ class SchemaRDD(RDD):
self._lazy_jrdd = self._jschema_rdd.javaToPython()
return self._lazy_jrdd
- @property
- def _id(self):
- return self._jrdd.id()
+ def id(self):
+ if self._id is None:
+ self._id = self._jrdd.id()
+ return self._id
def saveAsParquetFile(self, path):
"""Save the contents as a Parquet file, preserving the schema.