diff options
author | Liang-Chi Hsieh <viirya@gmail.com> | 2016-01-26 11:36:00 +0000 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-01-26 11:36:00 +0000 |
commit | 5936bf9fa85ccf7f0216145356140161c2801682 (patch) | |
tree | 163661f0c68baa978decee2e0a79e668b1536c91 /core/src | |
parent | 6743de3a98e3f0d0e6064ca1872fa88c3aeaa143 (diff) | |
download | spark-5936bf9fa85ccf7f0216145356140161c2801682.tar.gz spark-5936bf9fa85ccf7f0216145356140161c2801682.tar.bz2 spark-5936bf9fa85ccf7f0216145356140161c2801682.zip |
[SPARK-12961][CORE] Prevent snappy-java memory leak
JIRA: https://issues.apache.org/jira/browse/SPARK-12961
To prevent memory leak in snappy-java, just call the method once and cache the result. After the library releases new version, we can remove this object.
JoshRosen
Author: Liang-Chi Hsieh <viirya@gmail.com>
Closes #10875 from viirya/prevent-snappy-memory-leak.
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/org/apache/spark/io/CompressionCodec.scala | 20 |
1 files changed, 14 insertions, 6 deletions
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala index 717804626f..ae014becef 100644 --- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala +++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala @@ -149,12 +149,7 @@ class LZFCompressionCodec(conf: SparkConf) extends CompressionCodec { */ @DeveloperApi class SnappyCompressionCodec(conf: SparkConf) extends CompressionCodec { - - try { - Snappy.getNativeLibraryVersion - } catch { - case e: Error => throw new IllegalArgumentException(e) - } + val version = SnappyCompressionCodec.version override def compressedOutputStream(s: OutputStream): OutputStream = { val blockSize = conf.getSizeAsBytes("spark.io.compression.snappy.blockSize", "32k").toInt @@ -165,6 +160,19 @@ class SnappyCompressionCodec(conf: SparkConf) extends CompressionCodec { } /** + * Object guards against memory leak bug in snappy-java library: + * (https://github.com/xerial/snappy-java/issues/131). + * Before a new version of the library, we only call the method once and cache the result. + */ +private final object SnappyCompressionCodec { + private lazy val version: String = try { + Snappy.getNativeLibraryVersion + } catch { + case e: Error => throw new IllegalArgumentException(e) + } +} + +/** * Wrapper over [[SnappyOutputStream]] which guards against write-after-close and double-close * issues. See SPARK-7660 for more details. This wrapping can be removed if we upgrade to a version * of snappy-java that contains the fix for https://github.com/xerial/snappy-java/issues/107. |