diff options
author | Kostas Sakellis <kostas@cloudera.com> | 2014-12-08 15:44:18 -0800 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2014-12-08 15:44:30 -0800 |
commit | 9ed5641a5a4425278283896928efa4e382fb74d8 (patch) | |
tree | 2ec2267d962df4b048f9f4b5eacb43c73add6caf /examples/src | |
parent | 6b9e8b081655f71f7ff2c4238254f7aaa110723c (diff) | |
download | spark-9ed5641a5a4425278283896928efa4e382fb74d8.tar.gz spark-9ed5641a5a4425278283896928efa4e382fb74d8.tar.bz2 spark-9ed5641a5a4425278283896928efa4e382fb74d8.zip |
[SPARK-4774] [SQL] Makes HiveFromSpark more portable
HiveFromSpark read the kv1.txt file from SPARK_HOME/examples/src/main/resources/kv1.txt which assumed
you had a source tree checked out. Now we copy the kv1.txt file to a temporary file and delete it when
the jvm shuts down. This allows us to run this example outside of a spark source tree.
Author: Kostas Sakellis <kostas@cloudera.com>
Closes #3628 from ksakellis/kostas-spark-4774 and squashes the following commits:
6770f83 [Kostas Sakellis] [SPARK-4774] [SQL] Makes HiveFromSpark more portable
(cherry picked from commit d6a972b3e4dc35a2d95df47d256462b325f4bda6)
Signed-off-by: Michael Armbrust <michael@databricks.com>
Diffstat (limited to 'examples/src')
-rw-r--r-- | examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala | 13 |
1 files changed, 11 insertions, 2 deletions
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala b/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala index 138923c4d7..5725da1848 100644 --- a/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala +++ b/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala @@ -17,6 +17,10 @@ package org.apache.spark.examples.sql.hive +import com.google.common.io.{ByteStreams, Files} + +import java.io.File + import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql._ import org.apache.spark.sql.hive.HiveContext @@ -24,10 +28,15 @@ import org.apache.spark.sql.hive.HiveContext object HiveFromSpark { case class Record(key: Int, value: String) + // Copy kv1.txt file from classpath to temporary directory + val kv1Stream = HiveFromSpark.getClass.getResourceAsStream("/kv1.txt") + val kv1File = File.createTempFile("kv1", "txt") + kv1File.deleteOnExit() + ByteStreams.copy(kv1Stream, Files.newOutputStreamSupplier(kv1File)) + def main(args: Array[String]) { val sparkConf = new SparkConf().setAppName("HiveFromSpark") val sc = new SparkContext(sparkConf) - val path = s"${System.getenv("SPARK_HOME")}/examples/src/main/resources/kv1.txt" // A hive context adds support for finding tables in the MetaStore and writing queries // using HiveQL. Users who do not have an existing Hive deployment can still create a @@ -37,7 +46,7 @@ object HiveFromSpark { import hiveContext._ sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)") - sql(s"LOAD DATA LOCAL INPATH '$path' INTO TABLE src") + sql(s"LOAD DATA LOCAL INPATH '${kv1File.getAbsolutePath}' INTO TABLE src") // Queries are expressed in HiveQL println("Result of 'SELECT *': ") |