diff options
| author | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2012-12-29 16:23:43 -0800 |
|---|---|---|
| committer | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2013-01-07 15:57:33 -0800 |
| commit | aed368a970bbaee4bdf297ba3f6f1b0fa131452c (patch) | |
| tree | 354e07b9addf245e8dd2d1db9ba25463a59172b9 | |
| parent | f8d579a0c05b7d29b59e541b483ded471d14ec17 (diff) | |
| download | spark-aed368a970bbaee4bdf297ba3f6f1b0fa131452c.tar.gz spark-aed368a970bbaee4bdf297ba3f6f1b0fa131452c.tar.bz2 spark-aed368a970bbaee4bdf297ba3f6f1b0fa131452c.zip | |
Update Hadoop dependency to 1.0.3 as 0.20 has Sun specific dependencies. Also
fix SequenceFileRDDFunctions to pick the right type conversion across Hadoop
versions
| -rw-r--r-- | core/src/main/scala/spark/SequenceFileRDDFunctions.scala | 8 | ||||
| -rw-r--r-- | project/SparkBuild.scala | 2 |
2 files changed, 8 insertions, 2 deletions
diff --git a/core/src/main/scala/spark/SequenceFileRDDFunctions.scala b/core/src/main/scala/spark/SequenceFileRDDFunctions.scala index a34aee69c1..6b4a11d6d3 100644 --- a/core/src/main/scala/spark/SequenceFileRDDFunctions.scala +++ b/core/src/main/scala/spark/SequenceFileRDDFunctions.scala @@ -42,7 +42,13 @@ class SequenceFileRDDFunctions[K <% Writable: ClassManifest, V <% Writable : Cla if (classOf[Writable].isAssignableFrom(classManifest[T].erasure)) { classManifest[T].erasure } else { - implicitly[T => Writable].getClass.getMethods()(0).getReturnType + // We get the type of the Writable class by looking at the apply method which converts + // from T to Writable. Since we have two apply methods we filter out the one which + // is of the form "java.lang.Object apply(java.lang.Object)" + implicitly[T => Writable].getClass.getDeclaredMethods().filter( + m => m.getReturnType().toString != "java.lang.Object" && + m.getName() == "apply")(0).getReturnType + } // TODO: use something like WritableConverter to avoid reflection } diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 842d0fa96b..7c7c33131a 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -10,7 +10,7 @@ import twirl.sbt.TwirlPlugin._ object SparkBuild extends Build { // Hadoop version to build against. For example, "0.20.2", "0.20.205.0", or // "1.0.3" for Apache releases, or "0.20.2-cdh3u5" for Cloudera Hadoop. - val HADOOP_VERSION = "0.20.205.0" + val HADOOP_VERSION = "1.0.3" val HADOOP_MAJOR_VERSION = "1" // For Hadoop 2 versions such as "2.0.0-mr1-cdh4.1.1", set the HADOOP_MAJOR_VERSION to "2" |