diff options
| author | Andre Schumacher <schumach@icsi.berkeley.edu> | 2013-08-15 16:01:19 -0700 |
|---|---|---|
| committer | Andre Schumacher <schumach@icsi.berkeley.edu> | 2013-08-16 11:58:20 -0700 |
| commit | c7e348faec45ad1d996d16639015c4bc4fc3bc92 (patch) | |
| tree | 45e69b999c4b4af6bd7528e3dcc860bce264e14f /python/pyspark/rdd.py | |
| parent | 659553b21ddd7504889ce113a816c1db4a73f167 (diff) | |
| download | spark-c7e348faec45ad1d996d16639015c4bc4fc3bc92.tar.gz spark-c7e348faec45ad1d996d16639015c4bc4fc3bc92.tar.bz2 spark-c7e348faec45ad1d996d16639015c4bc4fc3bc92.zip | |
Implementing SPARK-878 for PySpark: adding zip and egg files to context and passing it down to workers which add these to their sys.path
Diffstat (limited to 'python/pyspark/rdd.py')
| -rw-r--r-- | python/pyspark/rdd.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py index 51c2cb9806..99f5967a8e 100644 --- a/python/pyspark/rdd.py +++ b/python/pyspark/rdd.py @@ -758,8 +758,10 @@ class PipelinedRDD(RDD): class_manifest = self._prev_jrdd.classManifest() env = MapConverter().convert(self.ctx.environment, self.ctx._gateway._gateway_client) + includes = ListConverter().convert(self.ctx._python_includes, + self.ctx._gateway._gateway_client) python_rdd = self.ctx._jvm.PythonRDD(self._prev_jrdd.rdd(), - pipe_command, env, self.preservesPartitioning, self.ctx.pythonExec, + pipe_command, env, includes, self.preservesPartitioning, self.ctx.pythonExec, broadcast_vars, self.ctx._javaAccumulator, class_manifest) self._jrdd_val = python_rdd.asJavaRDD() return self._jrdd_val |