diff options
Diffstat (limited to 'python/pyspark/context.py')
| -rw-r--r-- | python/pyspark/context.py | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/python/pyspark/context.py b/python/pyspark/context.py index d7466729b8..43bde5ae41 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -152,6 +152,11 @@ class SparkContext(object): self.master = self._conf.get("spark.master") self.appName = self._conf.get("spark.app.name") self.sparkHome = self._conf.get("spark.home", None) + + # Let YARN know it's a pyspark app, so it distributes needed libraries. + if self.master == "yarn-client": + self._conf.set("spark.yarn.isPython", "true") + for (k, v) in self._conf.getAll(): if k.startswith("spark.executorEnv."): varName = k[len("spark.executorEnv."):] |