aboutsummaryrefslogtreecommitdiff
path: root/sql/hive
diff options
context:
space:
mode:
authorMichael Armbrust <michael@databricks.com>2014-12-30 11:24:46 -0800
committerMichael Armbrust <michael@databricks.com>2014-12-30 11:24:46 -0800
commit480bd1d2edd1de06af607b0cf3ff3c0b16089add (patch)
treea5f7db35d353a8e55274585d1477fca824e0400c /sql/hive
parentefa80a531ecd485f6cf0cdc24ffa42ba17eea46d (diff)
downloadspark-480bd1d2edd1de06af607b0cf3ff3c0b16089add.tar.gz
spark-480bd1d2edd1de06af607b0cf3ff3c0b16089add.tar.bz2
spark-480bd1d2edd1de06af607b0cf3ff3c0b16089add.zip
[SPARK-4908][SQL] Prevent multiple concurrent hive native commands
This is just a quick fix that locks when calling `runHive`. If we can find a way to avoid the error without a global lock that would be better. Author: Michael Armbrust <michael@databricks.com> Closes #3834 from marmbrus/hiveConcurrency and squashes the following commits: bf25300 [Michael Armbrust] prevent multiple concurrent hive native commands
Diffstat (limited to 'sql/hive')
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala2
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala7
2 files changed, 8 insertions, 1 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 56fe27a77b..982e0593fc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -284,7 +284,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
* Execute the command using Hive and return the results as a sequence. Each element
* in the sequence is one row.
*/
- protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = {
+ protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = synchronized {
try {
val cmd_trimmed: String = cmd.trim()
val tokens: Array[String] = cmd_trimmed.split("\\s+")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 4d81acc753..fb6da33e88 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -56,6 +56,13 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
Locale.setDefault(originalLocale)
}
+ test("SPARK-4908: concurent hive native commands") {
+ (1 to 100).par.map { _ =>
+ sql("USE default")
+ sql("SHOW TABLES")
+ }
+ }
+
createQueryTest("constant object inspector for generic udf",
"""SELECT named_struct(
lower("AA"), "10",