diff options
author | Liwei Lin <lwlin7@gmail.com> | 2016-07-24 08:35:57 +0100 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-07-24 08:35:57 +0100 |
commit | d6795c7a254b83d4ae4785f3add74981e5273c91 (patch) | |
tree | b705111bb4f48adad23a1cee7743b8cc53b83816 /sql/hive | |
parent | e3c7039b45763ca70c91432e7bdb8494ab010e62 (diff) | |
download | spark-d6795c7a254b83d4ae4785f3add74981e5273c91.tar.gz spark-d6795c7a254b83d4ae4785f3add74981e5273c91.tar.bz2 spark-d6795c7a254b83d4ae4785f3add74981e5273c91.zip |
[SPARK-16515][SQL][FOLLOW-UP] Fix test `script` on OS X/Windows...
## Problem
The current `sed` in `test_script.sh` is missing a `$`, leading to the failure of `script` test on OS X:
```
== Results ==
!== Correct Answer - 2 == == Spark Answer - 2 ==
![x1_y1] [x1]
![x2_y2] [x2]
```
In addition, this `script` test would also fail on systems like Windows where we couldn't be able to invoke `bash` or `echo | sed`.
## What changes were proposed in this pull request?
This patch
- fixes `sed` in `test_script.sh`
- adds command guards so that the `script` test would pass on systems like Windows
## How was this patch tested?
- Jenkins
- Manually verified tests pass on OS X
Author: Liwei Lin <lwlin7@gmail.com>
Closes #14280 from lw-lin/osx-sed.
Diffstat (limited to 'sql/hive')
-rwxr-xr-x | sql/hive/src/test/resources/test_script.sh | 2 | ||||
-rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 26 |
2 files changed, 19 insertions, 9 deletions
diff --git a/sql/hive/src/test/resources/test_script.sh b/sql/hive/src/test/resources/test_script.sh index ab998c41b2..eb0c50e982 100755 --- a/sql/hive/src/test/resources/test_script.sh +++ b/sql/hive/src/test/resources/test_script.sh @@ -19,5 +19,5 @@ while read line do - echo "$line" | sed 's/\t/_/' + echo "$line" | sed $'s/\t/_/' done < /dev/stdin diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 688260d27c..cba6aa53f1 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution import java.sql.{Date, Timestamp} +import scala.sys.process.Process +import scala.util.Try + import org.apache.hadoop.fs.Path import org.apache.spark.sql._ @@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { import spark.implicits._ test("script") { - val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") - df.createOrReplaceTempView("script_table") - val query1 = sql( - """ - |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table - |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS - |(col1 STRING, col2 STRING)) script_test_table""".stripMargin) - checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil) + if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) { + val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") + df.createOrReplaceTempView("script_table") + val query1 = sql( + """ + |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table + |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS + |(col1 STRING, col2 STRING)) script_test_table""".stripMargin) + checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil) + } + // else skip this test } test("UDTF") { @@ -1766,4 +1772,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } } + + def testCommandAvailable(command: String): Boolean = { + Try(Process(command) !!).isSuccess + } } |