From 009b0e37e7c284c531cc3c44d0e5b5b1476f1666 Mon Sep 17 00:00:00 2001 From: Matei Zaharia Date: Thu, 27 Sep 2012 18:45:44 -0700 Subject: Added an option to compress blocks in the block store --- docs/configuration.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'docs/configuration.md') diff --git a/docs/configuration.md b/docs/configuration.md index 4e47ca16e8..e4d5b21a12 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -111,6 +111,23 @@ Apart from these, the following properties are also available, and may be useful reduceByKey, etc) when not set by user. + + spark.blockManager.compress + false + + Set to "true" to have Spark compress map output files, RDDs that get cached on disk, + and RDDs that get cached in serialized form. Generally a good idea when dealing with + large datasets, but might add some CPU overhead. + + + + spark.broadcast.compress + false + + Set to "true" to have Spark compress broadcast variables before sending them. + Generally a good idea when broadcasting large values. + + spark.storage.memoryFraction 0.66 -- cgit v1.2.3