diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-01 10:33:37 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-03-01 10:33:37 -0800 |
commit | a789e9aaea2d44273b0089496ea38b764fd95e70 (patch) | |
tree | 3ec87bb9d96e0d3cde268fbe98aef2aa4d21fc07 /core/src | |
parent | 9e59afd71082c709aa0f4f4a95ec1de982179aee (diff) | |
parent | 021c50a8d4f00a008c7af24893fde702ea004523 (diff) | |
download | spark-a789e9aaea2d44273b0089496ea38b764fd95e70.tar.gz spark-a789e9aaea2d44273b0089496ea38b764fd95e70.tar.bz2 spark-a789e9aaea2d44273b0089496ea38b764fd95e70.zip |
Merge remote branch 'origin/master' into new-rdds
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/spark/HadoopFile.scala | 21 |
1 files changed, 6 insertions, 15 deletions
diff --git a/core/src/main/scala/spark/HadoopFile.scala b/core/src/main/scala/spark/HadoopFile.scala index 754466204b..0a7996c7bd 100644 --- a/core/src/main/scala/spark/HadoopFile.scala +++ b/core/src/main/scala/spark/HadoopFile.scala @@ -35,7 +35,7 @@ class HadoopFile[K, V]( keyClass: Class[K], valueClass: Class[V]) extends RDD[(K, V)](sc) { - @transient val splits_ : Array[Split] = ConfigureLock.synchronized { + @transient val splits_ : Array[Split] = { val conf = new JobConf() FileInputFormat.setInputPaths(conf, path) val inputFormat = createInputFormat(conf) @@ -57,13 +57,11 @@ extends RDD[(K, V)](sc) { val split = theSplit.asInstanceOf[HadoopSplit] var reader: RecordReader[K, V] = null - ConfigureLock.synchronized { - val conf = new JobConf() - val bufferSize = System.getProperty("spark.buffer.size", "65536") - conf.set("io.file.buffer.size", bufferSize) - val fmt = createInputFormat(conf) - reader = fmt.getRecordReader(split.inputSplit.value, conf, Reporter.NULL) - } + val conf = new JobConf() + val bufferSize = System.getProperty("spark.buffer.size", "65536") + conf.set("io.file.buffer.size", bufferSize) + val fmt = createInputFormat(conf) + reader = fmt.getRecordReader(split.inputSplit.value, conf, Reporter.NULL) val key: K = keyClass.newInstance() val value: V = valueClass.newInstance() @@ -118,10 +116,3 @@ extends MappedRDD[String, (LongWritable, Text)]( classOf[LongWritable], classOf[Text]), { pair: (LongWritable, Text) => pair._2.toString } ) - - -/** - * Object used to ensure that only one thread at a time is configuring Hadoop - * InputFormat classes. Apparently configuring them is not thread safe! - */ -object ConfigureLock {} |