aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLiwei Lin <lwlin7@gmail.com>2016-07-24 08:35:57 +0100
committerSean Owen <sowen@cloudera.com>2016-07-24 08:35:57 +0100
commitd6795c7a254b83d4ae4785f3add74981e5273c91 (patch)
treeb705111bb4f48adad23a1cee7743b8cc53b83816
parente3c7039b45763ca70c91432e7bdb8494ab010e62 (diff)
downloadspark-d6795c7a254b83d4ae4785f3add74981e5273c91.tar.gz
spark-d6795c7a254b83d4ae4785f3add74981e5273c91.tar.bz2
spark-d6795c7a254b83d4ae4785f3add74981e5273c91.zip
[SPARK-16515][SQL][FOLLOW-UP] Fix test `script` on OS X/Windows...
## Problem The current `sed` in `test_script.sh` is missing a `$`, leading to the failure of `script` test on OS X: ``` == Results == !== Correct Answer - 2 == == Spark Answer - 2 == ![x1_y1] [x1] ![x2_y2] [x2] ``` In addition, this `script` test would also fail on systems like Windows where we couldn't be able to invoke `bash` or `echo | sed`. ## What changes were proposed in this pull request? This patch - fixes `sed` in `test_script.sh` - adds command guards so that the `script` test would pass on systems like Windows ## How was this patch tested? - Jenkins - Manually verified tests pass on OS X Author: Liwei Lin <lwlin7@gmail.com> Closes #14280 from lw-lin/osx-sed.
-rwxr-xr-xsql/hive/src/test/resources/test_script.sh2
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala26
2 files changed, 19 insertions, 9 deletions
diff --git a/sql/hive/src/test/resources/test_script.sh b/sql/hive/src/test/resources/test_script.sh
index ab998c41b2..eb0c50e982 100755
--- a/sql/hive/src/test/resources/test_script.sh
+++ b/sql/hive/src/test/resources/test_script.sh
@@ -19,5 +19,5 @@
while read line
do
- echo "$line" | sed 's/\t/_/'
+ echo "$line" | sed $'s/\t/_/'
done < /dev/stdin
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 688260d27c..cba6aa53f1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution
import java.sql.{Date, Timestamp}
+import scala.sys.process.Process
+import scala.util.Try
+
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
@@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import spark.implicits._
test("script") {
- val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
- df.createOrReplaceTempView("script_table")
- val query1 = sql(
- """
- |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
- |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
- |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
- checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+ if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) {
+ val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
+ df.createOrReplaceTempView("script_table")
+ val query1 = sql(
+ """
+ |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
+ |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
+ |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
+ checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+ }
+ // else skip this test
}
test("UDTF") {
@@ -1766,4 +1772,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
}
+
+ def testCommandAvailable(command: String): Boolean = {
+ Try(Process(command) !!).isSuccess
+ }
}