diff options
author | Matei Zaharia <matei.zaharia@gmail.com> | 2013-09-02 11:20:39 -0700 |
---|---|---|
committer | Matei Zaharia <matei.zaharia@gmail.com> | 2013-09-02 11:20:39 -0700 |
commit | 636fc0c89e837f1dec72bdacc749423b08e06126 (patch) | |
tree | e05ba450c54acfc9e322a58cb2f5691d4755b260 /core/src | |
parent | d9a53b94f81f6d706bece56c15a2f9e05143a350 (diff) | |
parent | 246bf67f58fab00df736abc20a0f3740920e0083 (diff) | |
download | spark-636fc0c89e837f1dec72bdacc749423b08e06126.tar.gz spark-636fc0c89e837f1dec72bdacc749423b08e06126.tar.bz2 spark-636fc0c89e837f1dec72bdacc749423b08e06126.zip |
Merge pull request #886 from mateiz/codec
Fix spark.io.compression.codec and change default codec to LZF
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/org/apache/spark/io/CompressionCodec.scala | 12 | ||||
-rw-r--r-- | core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala | 2 |
2 files changed, 5 insertions, 9 deletions
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala index 90a0420caf..570a979b56 100644 --- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala +++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala @@ -39,17 +39,13 @@ trait CompressionCodec { private[spark] object CompressionCodec { def createCodec(): CompressionCodec = { - // Set the default codec to Snappy since the LZF implementation initializes a pretty large - // buffer for every stream, which results in a lot of memory overhead when the number of - // shuffle reduce buckets are large. - createCodec(classOf[SnappyCompressionCodec].getName) + createCodec(System.getProperty( + "spark.io.compression.codec", classOf[LZFCompressionCodec].getName)) } def createCodec(codecName: String): CompressionCodec = { - Class.forName( - System.getProperty("spark.io.compression.codec", codecName), - true, - Thread.currentThread.getContextClassLoader).newInstance().asInstanceOf[CompressionCodec] + Class.forName(codecName, true, Thread.currentThread.getContextClassLoader) + .newInstance().asInstanceOf[CompressionCodec] } } diff --git a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala index fd6f69041a..ab81bfbe55 100644 --- a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala +++ b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala @@ -44,7 +44,7 @@ class CompressionCodecSuite extends FunSuite { test("default compression codec") { val codec = CompressionCodec.createCodec() - assert(codec.getClass === classOf[SnappyCompressionCodec]) + assert(codec.getClass === classOf[LZFCompressionCodec]) testCodec(codec) } |