From 6402b539d0058bd2cfc600f348801e7279acadab Mon Sep 17 00:00:00 2001 From: Charles Reiss Date: Sat, 10 Aug 2013 21:27:40 -0700 Subject: Use new Configuration() instead of new JobConf() for ObjectWritable. JobConf's constructor loads default config files in some verisons of Hadoop, which is quite slow, and we only need the Configuration object to pass the correct ClassLoader. --- core/src/main/scala/spark/SerializableWritable.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'core') diff --git a/core/src/main/scala/spark/SerializableWritable.scala b/core/src/main/scala/spark/SerializableWritable.scala index 0236611ef9..936d8e6241 100644 --- a/core/src/main/scala/spark/SerializableWritable.scala +++ b/core/src/main/scala/spark/SerializableWritable.scala @@ -21,7 +21,7 @@ import java.io._ import org.apache.hadoop.io.ObjectWritable import org.apache.hadoop.io.Writable -import org.apache.hadoop.mapred.JobConf +import org.apache.hadoop.conf.Configuration class SerializableWritable[T <: Writable](@transient var t: T) extends Serializable { def value = t @@ -35,7 +35,7 @@ class SerializableWritable[T <: Writable](@transient var t: T) extends Serializa private def readObject(in: ObjectInputStream) { in.defaultReadObject() val ow = new ObjectWritable() - ow.setConf(new JobConf()) + ow.setConf(new Configuration()) ow.readFields(in) t = ow.get().asInstanceOf[T] } -- cgit v1.2.3