diff options
author | Davies Liu <davies@databricks.com> | 2015-12-21 14:21:43 -0800 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2015-12-21 14:21:43 -0800 |
commit | 29cecd4a42f6969613e5b2a40f2724f99e7eec01 (patch) | |
tree | 261b40272c991649ce584b7cec2056dcd2b6cf1c /core/src/test/scala | |
parent | d655d37ddf59d7fb6db529324ac8044d53b2622a (diff) | |
download | spark-29cecd4a42f6969613e5b2a40f2724f99e7eec01.tar.gz spark-29cecd4a42f6969613e5b2a40f2724f99e7eec01.tar.bz2 spark-29cecd4a42f6969613e5b2a40f2724f99e7eec01.zip |
[SPARK-12388] change default compression to lz4
According the benchmark [1], LZ4-java could be 80% (or 30%) faster than Snappy.
After changing the compressor to LZ4, I saw 20% improvement on end-to-end time for a TPCDS query (Q4).
[1] https://github.com/ning/jvm-compressor-benchmark/wiki
cc rxin
Author: Davies Liu <davies@databricks.com>
Closes #10342 from davies/lz4.
Diffstat (limited to 'core/src/test/scala')
-rw-r--r-- | core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala | 8 |
1 files changed, 3 insertions, 5 deletions
diff --git a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala index 1553ab60bd..9e9c2b0165 100644 --- a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala +++ b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala @@ -46,7 +46,7 @@ class CompressionCodecSuite extends SparkFunSuite { test("default compression codec") { val codec = CompressionCodec.createCodec(conf) - assert(codec.getClass === classOf[SnappyCompressionCodec]) + assert(codec.getClass === classOf[LZ4CompressionCodec]) testCodec(codec) } @@ -62,12 +62,10 @@ class CompressionCodecSuite extends SparkFunSuite { testCodec(codec) } - test("lz4 does not support concatenation of serialized streams") { + test("lz4 supports concatenation of serialized streams") { val codec = CompressionCodec.createCodec(conf, classOf[LZ4CompressionCodec].getName) assert(codec.getClass === classOf[LZ4CompressionCodec]) - intercept[Exception] { - testConcatenationOfSerializedStreams(codec) - } + testConcatenationOfSerializedStreams(codec) } test("lzf compression codec") { |