diff options
author | Matei Zaharia <matei@databricks.com> | 2015-07-22 15:28:09 -0700 |
---|---|---|
committer | Matei Zaharia <matei@databricks.com> | 2015-07-22 15:28:09 -0700 |
commit | fe26584a1f5b472fb2e87aa7259aec822a619a3b (patch) | |
tree | d568c3aeda422e91d2b3d1a9335605da55be73fa /python | |
parent | 1aca9c13c144fa336af6afcfa666128bf77c49d4 (diff) | |
download | spark-fe26584a1f5b472fb2e87aa7259aec822a619a3b.tar.gz spark-fe26584a1f5b472fb2e87aa7259aec822a619a3b.tar.bz2 spark-fe26584a1f5b472fb2e87aa7259aec822a619a3b.zip |
[SPARK-9244] Increase some memory defaults
There are a few memory limits that people hit often and that we could
make higher, especially now that memory sizes have grown.
- spark.akka.frameSize: This defaults at 10 but is often hit for map
output statuses in large shuffles. This memory is not fully allocated
up-front, so we can just make this larger and still not affect jobs
that never sent a status that large. We increase it to 128.
- spark.executor.memory: Defaults at 512m, which is really small. We
increase it to 1g.
Author: Matei Zaharia <matei@databricks.com>
Closes #7586 from mateiz/configs and squashes the following commits:
ce0038a [Matei Zaharia] [SPARK-9244] Increase some memory defaults
Diffstat (limited to 'python')
-rw-r--r-- | python/pyspark/tests.py | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 5be9937cb0..8bfed074c9 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -1823,7 +1823,7 @@ class SparkSubmitTests(unittest.TestCase): | return x + 1 """) proc = subprocess.Popen([self.sparkSubmit, "--py-files", zip, "--master", - "local-cluster[1,1,512]", script], + "local-cluster[1,1,1024]", script], stdout=subprocess.PIPE) out, err = proc.communicate() self.assertEqual(0, proc.returncode) @@ -1857,7 +1857,7 @@ class SparkSubmitTests(unittest.TestCase): self.create_spark_package("a:mylib:0.1") proc = subprocess.Popen([self.sparkSubmit, "--packages", "a:mylib:0.1", "--repositories", "file:" + self.programDir, "--master", - "local-cluster[1,1,512]", script], stdout=subprocess.PIPE) + "local-cluster[1,1,1024]", script], stdout=subprocess.PIPE) out, err = proc.communicate() self.assertEqual(0, proc.returncode) self.assertIn("[2, 3, 4]", out.decode('utf-8')) @@ -1876,7 +1876,7 @@ class SparkSubmitTests(unittest.TestCase): # this will fail if you have different spark.executor.memory # in conf/spark-defaults.conf proc = subprocess.Popen( - [self.sparkSubmit, "--master", "local-cluster[1,1,512]", script], + [self.sparkSubmit, "--master", "local-cluster[1,1,1024]", script], stdout=subprocess.PIPE) out, err = proc.communicate() self.assertEqual(0, proc.returncode) |