From 49b361298b09d415de1857846367913495aecfa6 Mon Sep 17 00:00:00 2001 From: Yin Huai Date: Thu, 31 Jul 2014 13:05:24 -0700 Subject: [SPARK-2523] [SQL] Hadoop table scan bug fixing (fix failing Jenkins maven test) This PR tries to resolve the broken Jenkins maven test issue introduced by #1439. Now, we create a single query test to run both the setup work and the test query. Author: Yin Huai Closes #1669 from yhuai/SPARK-2523-fixTest and squashes the following commits: 358af1a [Yin Huai] Make partition_based_table_scan_with_different_serde run atomically. --- ...ferent_serde-0-1436cccda63b78dd6e43a399da6cc474 | 0 ...ferent_serde-0-8caed2a6e80250a6d38a59388679c298 | 2 - ...ferent_serde-1-8d9bf54373f45bc35f8cb6e82771b154 | 0 ...ferent_serde-2-7816c17905012cf381abf93d230faa8d | 0 ...ferent_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc | 0 ...ferent_serde-4-8caed2a6e80250a6d38a59388679c298 | 2 + .../sql/hive/execution/HiveTableScanSuite.scala | 45 +++++++++------------- 7 files changed, 21 insertions(+), 28 deletions(-) create mode 100644 sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-1436cccda63b78dd6e43a399da6cc474 delete mode 100644 sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298 create mode 100644 sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-1-8d9bf54373f45bc35f8cb6e82771b154 create mode 100644 sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-2-7816c17905012cf381abf93d230faa8d create mode 100644 sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc create mode 100644 sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298 (limited to 'sql') diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-1436cccda63b78dd6e43a399da6cc474 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-1436cccda63b78dd6e43a399da6cc474 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298 deleted file mode 100644 index f369f21e18..0000000000 --- a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298 +++ /dev/null @@ -1,2 +0,0 @@ -100 100 2010-01-01 -200 200 2010-01-02 diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-1-8d9bf54373f45bc35f8cb6e82771b154 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-1-8d9bf54373f45bc35f8cb6e82771b154 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-2-7816c17905012cf381abf93d230faa8d b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-2-7816c17905012cf381abf93d230faa8d new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298 new file mode 100644 index 0000000000..f369f21e18 --- /dev/null +++ b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298 @@ -0,0 +1,2 @@ +100 100 2010-01-01 +200 200 2010-01-02 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala index bcb00f871d..c5736723b4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala @@ -17,32 +17,25 @@ package org.apache.spark.sql.hive.execution -import org.scalatest.{BeforeAndAfterAll, FunSuite} - -import org.apache.spark.{SparkConf, SparkContext} -import org.apache.spark.sql.hive.test.TestHive - class HiveTableScanSuite extends HiveComparisonTest { - // MINOR HACK: You must run a query before calling reset the first time. - TestHive.hql("SHOW TABLES") - TestHive.reset() - - TestHive.hql("""CREATE TABLE part_scan_test (key STRING, value STRING) PARTITIONED BY (ds STRING) - | ROW FORMAT SERDE - | 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' - | STORED AS RCFILE - """.stripMargin) - TestHive.hql("""FROM src - | INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-01') - | SELECT 100,100 LIMIT 1 - """.stripMargin) - TestHive.hql("""ALTER TABLE part_scan_test SET SERDE - | 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' - """.stripMargin) - TestHive.hql("""FROM src INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-02') - | SELECT 200,200 LIMIT 1 - """.stripMargin) - createQueryTest("partition_based_table_scan_with_different_serde", - "SELECT * from part_scan_test", false) + createQueryTest("partition_based_table_scan_with_different_serde", + """ + |CREATE TABLE part_scan_test (key STRING, value STRING) PARTITIONED BY (ds STRING) + |ROW FORMAT SERDE + |'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe' + |STORED AS RCFILE; + | + |FROM src + |INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-01') + |SELECT 100,100 LIMIT 1; + | + |ALTER TABLE part_scan_test SET SERDE + |'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'; + | + |FROM src INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-02') + |SELECT 200,200 LIMIT 1; + | + |SELECT * from part_scan_test; + """.stripMargin) } -- cgit v1.2.3