From 49351c7f597c67950cc65e5014a89fad31b9a6f7 Mon Sep 17 00:00:00 2001 From: Lianhui Wang Date: Thu, 16 Jul 2015 19:31:14 -0700 Subject: [SPARK-8646] PySpark does not run on YARN if master not provided in command line andrewor14 davies vanzin can you take a look at this? thanks Author: Lianhui Wang Closes #7438 from lianhuiwang/SPARK-8646 and squashes the following commits: cb3f12d [Lianhui Wang] add whitespace 6d874a6 [Lianhui Wang] support pyspark for yarn-client --- python/pyspark/context.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'python/pyspark') diff --git a/python/pyspark/context.py b/python/pyspark/context.py index d7466729b8..43bde5ae41 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -152,6 +152,11 @@ class SparkContext(object): self.master = self._conf.get("spark.master") self.appName = self._conf.get("spark.app.name") self.sparkHome = self._conf.get("spark.home", None) + + # Let YARN know it's a pyspark app, so it distributes needed libraries. + if self.master == "yarn-client": + self._conf.set("spark.yarn.isPython", "true") + for (k, v) in self._conf.getAll(): if k.startswith("spark.executorEnv."): varName = k[len("spark.executorEnv."):] -- cgit v1.2.3