aboutsummaryrefslogtreecommitdiff
path: root/sql/hive/src
diff options
context:
space:
mode:
authorwitgo <witgo@qq.com>2014-07-23 18:17:05 -0700
committerPatrick Wendell <pwendell@gmail.com>2014-07-23 18:17:05 -0700
commit60f0ae3d87c84fd96e1f4d0abf5be1f51870e7ab (patch)
tree2fed1b435ebd3f6d4eac4e343586432eba3f24db /sql/hive/src
parent9b763329d968658d2dedee4485b7931646e6392e (diff)
downloadspark-60f0ae3d87c84fd96e1f4d0abf5be1f51870e7ab.tar.gz
spark-60f0ae3d87c84fd96e1f4d0abf5be1f51870e7ab.tar.bz2
spark-60f0ae3d87c84fd96e1f4d0abf5be1f51870e7ab.zip
[SPARK-2484][SQL] Build should not run hivecompatibility tests by default.
Author: witgo <witgo@qq.com> Closes #1403 from witgo/hive_compatibility and squashes the following commits: 4e5ecdb [witgo] The default does not run hive compatibility tests
Diffstat (limited to 'sql/hive/src')
-rw-r--r--sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala875
1 files changed, 0 insertions, 875 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
deleted file mode 100644
index 8b451973a4..0000000000
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ /dev/null
@@ -1,875 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.execution
-
-import java.io.File
-import java.util.TimeZone
-
-import org.scalatest.BeforeAndAfter
-
-import org.apache.spark.sql.hive.test.TestHive
-
-/**
- * Runs the test cases that are included in the hive distribution.
- */
-class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
- // TODO: bundle in jar files... get from classpath
- lazy val hiveQueryDir = TestHive.getHiveFile("ql" + File.separator + "src" +
- File.separator + "test" + File.separator + "queries" + File.separator + "clientpositive")
-
- var originalTimeZone: TimeZone = _
-
- def testCases = hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f)
-
- override def beforeAll() {
- TestHive.cacheTables = true
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- originalTimeZone = TimeZone.getDefault
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- }
-
- override def afterAll() {
- TestHive.cacheTables = false
- TimeZone.setDefault(originalTimeZone)
- }
-
- /** A list of tests deemed out of scope currently and thus completely disregarded. */
- override def blackList = Seq(
- // These tests use hooks that are not on the classpath and thus break all subsequent execution.
- "hook_order",
- "hook_context",
- "mapjoin_hook",
- "multi_sahooks",
- "overridden_confs",
- "query_properties",
- "sample10",
- "updateAccessTime",
- "index_compact_binary_search",
- "bucket_num_reducers",
- "column_access_stats",
- "concatenate_inherit_table_location",
- "describe_pretty",
- "describe_syntax",
- "orc_ends_with_nulls",
-
- // Setting a default property does not seem to get reset and thus changes the answer for many
- // subsequent tests.
- "create_default_prop",
-
- // User/machine specific test answers, breaks the caching mechanism.
- "authorization_3",
- "authorization_5",
- "keyword_1",
- "misc_json",
- "create_like_tbl_props",
- "load_overwrite",
- "alter_table_serde2",
- "alter_table_not_sorted",
- "alter_skewed_table",
- "alter_partition_clusterby_sortby",
- "alter_merge",
- "alter_concatenate_indexed_table",
- "protectmode2",
- //"describe_table",
- "describe_comment_nonascii",
- "udf5",
- "udf_java_method",
- "create_merge_compressed",
-
- // DFS commands
- "symlink_text_input_format",
-
- // Weird DDL differences result in failures on jenkins.
- "create_like2",
- "create_view_translate",
- "partitions_json",
-
- // This test is totally fine except that it includes wrong queries and expects errors, but error
- // message format in Hive and Spark SQL differ. Should workaround this later.
- "udf_to_unix_timestamp",
-
- // Cant run without local map/reduce.
- "index_auto_update",
- "index_auto_self_join",
- "index_stale.*",
- "index_compression",
- "index_bitmap_compression",
- "index_auto_multiple",
- "index_auto_mult_tables_compact",
- "index_auto_mult_tables",
- "index_auto_file_format",
- "index_auth",
- "index_auto_empty",
- "index_auto_partitioned",
- "index_auto_unused",
- "index_bitmap_auto_partitioned",
- "ql_rewrite_gbtoidx",
- "stats1.*",
- "stats20",
- "alter_merge_stats",
- "columnstats.*",
-
-
- // Hive seems to think 1.0 > NaN = true && 1.0 < NaN = false... which is wrong.
- // http://stackoverflow.com/a/1573715
- "ops_comparison",
-
- // Tests that seems to never complete on hive...
- "skewjoin",
- "database",
-
- // These tests fail and and exit the JVM.
- "auto_join18_multi_distinct",
- "join18_multi_distinct",
- "input44",
- "input42",
- "input_dfs",
- "metadata_export_drop",
- "repair",
-
- // Uses a serde that isn't on the classpath... breaks other tests.
- "bucketizedhiveinputformat",
-
- // Avro tests seem to change the output format permanently thus breaking the answer cache, until
- // we figure out why this is the case let just ignore all of avro related tests.
- ".*avro.*",
-
- // Unique joins are weird and will require a lot of hacks (see comments in hive parser).
- "uniquejoin",
-
- // Hive seems to get the wrong answer on some outer joins. MySQL agrees with catalyst.
- "auto_join29",
-
- // No support for multi-alias i.e. udf as (e1, e2, e3).
- "allcolref_in_udf",
-
- // No support for TestSerDe (not published afaik)
- "alter1",
- "input16",
-
- // No support for unpublished test udfs.
- "autogen_colalias",
-
- // Hive does not support buckets.
- ".*bucket.*",
-
- // No window support yet
- ".*window.*",
-
- // Fails in hive with authorization errors.
- "alter_rename_partition_authorization",
- "authorization.*",
-
- // Hadoop version specific tests
- "archive_corrupt",
-
- // No support for case sensitivity is resolution using hive properties atm.
- "case_sensitivity",
-
- // Flaky test, Hive sometimes returns different set of 10 rows.
- "lateral_view_outer",
-
- // After stop taking the `stringOrError` route, exceptions are thrown from these cases.
- // See SPARK-2129 for details.
- "join_view",
- "mergejoins_mixed",
-
- // Returning the result of a describe state as a JSON object is not supported.
- "describe_table_json",
- "describe_database_json",
- "describe_formatted_view_partitioned_json",
-
- // Hive returns the results of describe as plain text. Comments with multiple lines
- // introduce extra lines in the Hive results, which make the result comparison fail.
- "describe_comment_indent"
- )
-
- /**
- * The set of tests that are believed to be working in catalyst. Tests not on whiteList or
- * blacklist are implicitly marked as ignored.
- */
- override def whiteList = Seq(
- "add_part_exist",
- "add_part_multiple",
- "add_partition_no_whitelist",
- "add_partition_with_whitelist",
- "alias_casted_column",
- "alter2",
- "alter3",
- "alter4",
- "alter5",
- "alter_index",
- "alter_merge_2",
- "alter_partition_format_loc",
- "alter_partition_protect_mode",
- "alter_partition_with_whitelist",
- "alter_rename_partition",
- "alter_table_serde",
- "alter_varchar1",
- "alter_varchar2",
- "alter_view_as_select",
- "ambiguous_col",
- "auto_join0",
- "auto_join1",
- "auto_join10",
- "auto_join11",
- "auto_join12",
- "auto_join13",
- "auto_join14",
- "auto_join14_hadoop20",
- "auto_join15",
- "auto_join17",
- "auto_join18",
- "auto_join19",
- "auto_join2",
- "auto_join20",
- "auto_join21",
- "auto_join22",
- "auto_join23",
- "auto_join24",
- "auto_join25",
- "auto_join26",
- "auto_join27",
- "auto_join28",
- "auto_join3",
- "auto_join30",
- "auto_join31",
- "auto_join32",
- "auto_join4",
- "auto_join5",
- "auto_join6",
- "auto_join7",
- "auto_join8",
- "auto_join9",
- "auto_join_filters",
- "auto_join_nulls",
- "auto_join_reordering_values",
- "auto_smb_mapjoin_14",
- "auto_sortmerge_join_1",
- "auto_sortmerge_join_10",
- "auto_sortmerge_join_11",
- "auto_sortmerge_join_12",
- "auto_sortmerge_join_13",
- "auto_sortmerge_join_14",
- "auto_sortmerge_join_15",
- "auto_sortmerge_join_2",
- "auto_sortmerge_join_3",
- "auto_sortmerge_join_4",
- "auto_sortmerge_join_5",
- "auto_sortmerge_join_6",
- "auto_sortmerge_join_7",
- "auto_sortmerge_join_8",
- "auto_sortmerge_join_9",
- "binary_constant",
- "binarysortable_1",
- "cast1",
- "cluster",
- "combine1",
- "compute_stats_binary",
- "compute_stats_boolean",
- "compute_stats_double",
- "compute_stats_empty_table",
- "compute_stats_long",
- "compute_stats_string",
- "compute_stats_table",
- "convert_enum_to_string",
- "correlationoptimizer1",
- "correlationoptimizer10",
- "correlationoptimizer11",
- "correlationoptimizer13",
- "correlationoptimizer14",
- "correlationoptimizer15",
- "correlationoptimizer2",
- "correlationoptimizer3",
- "correlationoptimizer4",
- "correlationoptimizer6",
- "correlationoptimizer7",
- "correlationoptimizer8",
- "correlationoptimizer9",
- "count",
- "cp_mj_rc",
- "create_insert_outputformat",
- "create_like_view",
- "create_nested_type",
- "create_skewed_table1",
- "create_struct_table",
- "ct_case_insensitive",
- "database_location",
- "database_properties",
- "decimal_1",
- "decimal_4",
- "decimal_join",
- "default_partition_name",
- "delimiter",
- "desc_non_existent_tbl",
- "describe_formatted_view_partitioned",
- "diff_part_input_formats",
- "disable_file_format_check",
- "disallow_incompatible_type_change_off",
- "drop_function",
- "drop_index",
- "drop_multi_partitions",
- "drop_partitions_filter",
- "drop_partitions_filter2",
- "drop_partitions_filter3",
- "drop_partitions_ignore_protection",
- "drop_table",
- "drop_table2",
- "drop_view",
- "escape_clusterby1",
- "escape_distributeby1",
- "escape_orderby1",
- "escape_sortby1",
- "fetch_aggregation",
- "fileformat_sequencefile",
- "fileformat_text",
- "filter_join_breaktask",
- "filter_join_breaktask2",
- "groupby1",
- "groupby11",
- "groupby1_map",
- "groupby1_map_nomap",
- "groupby1_map_skew",
- "groupby1_noskew",
- "groupby2",
- "groupby2_map",
- "groupby2_map_skew",
- "groupby2_noskew",
- "groupby4",
- "groupby4_map",
- "groupby4_map_skew",
- "groupby4_noskew",
- "groupby5",
- "groupby5_map",
- "groupby5_map_skew",
- "groupby5_noskew",
- "groupby6",
- "groupby6_map",
- "groupby6_map_skew",
- "groupby6_noskew",
- "groupby7",
- "groupby7_map",
- "groupby7_map_multi_single_reducer",
- "groupby7_map_skew",
- "groupby7_noskew",
- "groupby8",
- "groupby8_map",
- "groupby8_map_skew",
- "groupby8_noskew",
- "groupby9",
- "groupby_distinct_samekey",
- "groupby_map_ppr",
- "groupby_multi_insert_common_distinct",
- "groupby_multi_single_reducer2",
- "groupby_multi_single_reducer3",
- "groupby_mutli_insert_common_distinct",
- "groupby_neg_float",
- "groupby_ppd",
- "groupby_ppr",
- "groupby_sort_10",
- "groupby_sort_2",
- "groupby_sort_3",
- "groupby_sort_4",
- "groupby_sort_5",
- "groupby_sort_6",
- "groupby_sort_7",
- "groupby_sort_8",
- "groupby_sort_9",
- "groupby_sort_test_1",
- "having",
- "having1",
- "implicit_cast1",
- "innerjoin",
- "inoutdriver",
- "input",
- "input0",
- "input1",
- "input10",
- "input11",
- "input11_limit",
- "input12",
- "input12_hadoop20",
- "input14",
- "input15",
- "input19",
- "input1_limit",
- "input2",
- "input21",
- "input22",
- "input23",
- "input24",
- "input25",
- "input26",
- "input28",
- "input2_limit",
- "input3",
- "input4",
- "input40",
- "input41",
- "input49",
- "input4_cb_delim",
- "input6",
- "input7",
- "input8",
- "input9",
- "input_limit",
- "input_part0",
- "input_part1",
- "input_part10",
- "input_part10_win",
- "input_part2",
- "input_part3",
- "input_part4",
- "input_part5",
- "input_part6",
- "input_part7",
- "input_part8",
- "input_part9",
- "input_testsequencefile",
- "inputddl1",
- "inputddl2",
- "inputddl3",
- "inputddl4",
- "inputddl6",
- "inputddl7",
- "inputddl8",
- "insert1",
- "insert2_overwrite_partitions",
- "insert_compressed",
- "join0",
- "join1",
- "join10",
- "join11",
- "join12",
- "join13",
- "join14",
- "join14_hadoop20",
- "join15",
- "join16",
- "join17",
- "join18",
- "join19",
- "join_1to1",
- "join2",
- "join20",
- "join21",
- "join22",
- "join23",
- "join24",
- "join25",
- "join26",
- "join27",
- "join28",
- "join29",
- "join3",
- "join30",
- "join31",
- "join32",
- "join32_lessSize",
- "join33",
- "join34",
- "join35",
- "join36",
- "join37",
- "join38",
- "join39",
- "join4",
- "join40",
- "join41",
- "join5",
- "join6",
- "join7",
- "join8",
- "join9",
- "join_array",
- "join_casesensitive",
- "join_empty",
- "join_filters",
- "join_hive_626",
- "join_map_ppr",
- "join_nulls",
- "join_rc",
- "join_reorder2",
- "join_reorder3",
- "join_reorder4",
- "join_star",
- "lateral_view",
- "lateral_view_cp",
- "lateral_view_ppd",
- "leftsemijoin",
- "leftsemijoin_mr",
- "limit_pushdown_negative",
- "lineage1",
- "literal_double",
- "literal_ints",
- "literal_string",
- "load_dyn_part7",
- "load_file_with_space_in_the_name",
- "loadpart1",
- "louter_join_ppr",
- "mapjoin_distinct",
- "mapjoin_filter_on_outerjoin",
- "mapjoin_mapjoin",
- "mapjoin_subquery",
- "mapjoin_subquery2",
- "mapjoin_test_outer",
- "mapreduce1",
- "mapreduce2",
- "mapreduce3",
- "mapreduce4",
- "mapreduce5",
- "mapreduce6",
- "mapreduce7",
- "mapreduce8",
- "merge1",
- "merge2",
- "mergejoins",
- "multigroupby_singlemr",
- "multi_insert_gby",
- "multi_insert_gby3",
- "multi_insert_lateral_view",
- "multi_join_union",
- "multiMapJoin1",
- "multiMapJoin2",
- "noalias_subq1",
- "nomore_ambiguous_table_col",
- "nonblock_op_deduplicate",
- "notable_alias1",
- "notable_alias2",
- "nullgroup",
- "nullgroup2",
- "nullgroup3",
- "nullgroup4",
- "nullgroup4_multi_distinct",
- "nullgroup5",
- "nullinput",
- "nullinput2",
- "nullscript",
- "optional_outer",
- "orc_dictionary_threshold",
- "orc_empty_files",
- "order",
- "order2",
- "outer_join_ppr",
- "parallel",
- "parenthesis_star_by",
- "partcols1",
- "part_inherit_tbl_props",
- "part_inherit_tbl_props_empty",
- "part_inherit_tbl_props_with_star",
- "partition_schema1",
- "partition_serde_format",
- "partition_varchar1",
- "partition_wise_fileformat4",
- "partition_wise_fileformat5",
- "partition_wise_fileformat6",
- "partition_wise_fileformat7",
- "partition_wise_fileformat9",
- "plan_json",
- "ppd1",
- "ppd2",
- "ppd_clusterby",
- "ppd_constant_expr",
- "ppd_constant_where",
- "ppd_gby",
- "ppd_gby2",
- "ppd_gby_join",
- "ppd_join",
- "ppd_join2",
- "ppd_join3",
- "ppd_join_filter",
- "ppd_outer_join1",
- "ppd_outer_join2",
- "ppd_outer_join3",
- "ppd_outer_join4",
- "ppd_outer_join5",
- "ppd_random",
- "ppd_repeated_alias",
- "ppd_transform",
- "ppd_udf_col",
- "ppd_union",
- "ppr_allchildsarenull",
- "ppr_pushdown",
- "ppr_pushdown2",
- "ppr_pushdown3",
- "progress_1",
- "protectmode",
- "push_or",
- "query_with_semi",
- "quote1",
- "quote2",
- "rcfile_columnar",
- "rcfile_lazydecompress",
- "rcfile_null_value",
- "rcfile_toleratecorruptions",
- "rcfile_union",
- "reduce_deduplicate",
- "reduce_deduplicate_exclude_gby",
- "reduce_deduplicate_exclude_join",
- "reduce_deduplicate_extended",
- "reducesink_dedup",
- "rename_column",
- "router_join_ppr",
- "select_as_omitted",
- "select_unquote_and",
- "select_unquote_not",
- "select_unquote_or",
- "semijoin",
- "serde_regex",
- "serde_reported_schema",
- "set_variable_sub",
- "show_describe_func_quotes",
- "show_functions",
- "show_partitions",
- "skewjoinopt13",
- "skewjoinopt18",
- "skewjoinopt9",
- "smb_mapjoin9",
- "smb_mapjoin_1",
- "smb_mapjoin_10",
- "smb_mapjoin_13",
- "smb_mapjoin_14",
- "smb_mapjoin_15",
- "smb_mapjoin_16",
- "smb_mapjoin_17",
- "smb_mapjoin_2",
- "smb_mapjoin_21",
- "smb_mapjoin_25",
- "smb_mapjoin_3",
- "smb_mapjoin_4",
- "smb_mapjoin_5",
- "smb_mapjoin_6",
- "smb_mapjoin_7",
- "smb_mapjoin_8",
- "sort",
- "sort_merge_join_desc_1",
- "sort_merge_join_desc_2",
- "sort_merge_join_desc_3",
- "sort_merge_join_desc_4",
- "sort_merge_join_desc_5",
- "sort_merge_join_desc_6",
- "sort_merge_join_desc_7",
- "stats0",
- "stats_aggregator_error_1",
- "stats_empty_partition",
- "stats_publisher_error_1",
- "subq2",
- "tablename_with_select",
- "timestamp_1",
- "timestamp_2",
- "timestamp_3",
- "timestamp_comparison",
- "timestamp_lazy",
- "timestamp_null",
- "timestamp_udf",
- "touch",
- "transform_ppr1",
- "transform_ppr2",
- "type_cast_1",
- "type_widening",
- "udaf_collect_set",
- "udaf_corr",
- "udaf_covar_pop",
- "udaf_covar_samp",
- "udaf_histogram_numeric",
- "udf_10_trims",
- "udf2",
- "udf6",
- "udf7",
- "udf8",
- "udf9",
- "udf_E",
- "udf_PI",
- "udf_abs",
- "udf_acos",
- "udf_add",
- "udf_array",
- "udf_array_contains",
- "udf_ascii",
- "udf_asin",
- "udf_atan",
- "udf_avg",
- "udf_bigint",
- "udf_bin",
- "udf_bitmap_and",
- "udf_bitmap_empty",
- "udf_bitmap_or",
- "udf_bitwise_and",
- "udf_bitwise_not",
- "udf_bitwise_or",
- "udf_bitwise_xor",
- "udf_boolean",
- "udf_case",
- "udf_ceil",
- "udf_ceiling",
- "udf_concat",
- "udf_concat_insert1",
- "udf_concat_insert2",
- "udf_concat_ws",
- "udf_conv",
- "udf_cos",
- "udf_count",
- "udf_date_add",
- "udf_datediff",
- "udf_date_sub",
- "udf_day",
- "udf_dayofmonth",
- "udf_degrees",
- "udf_div",
- "udf_double",
- "udf_E",
- "udf_elt",
- "udf_exp",
- "udf_field",
- "udf_find_in_set",
- "udf_float",
- "udf_floor",
- "udf_format_number",
- "udf_from_unixtime",
- "udf_greaterthan",
- "udf_greaterthanorequal",
- "udf_hash",
- "udf_hex",
- "udf_if",
- "udf_index",
- "udf_instr",
- "udf_int",
- "udf_isnotnull",
- "udf_isnull",
- "udf_java_method",
- "udf_lcase",
- "udf_length",
- "udf_lessthan",
- "udf_lessthanorequal",
- "udf_like",
- "udf_ln",
- "udf_locate",
- "udf_log",
- "udf_log10",
- "udf_log2",
- "udf_lower",
- "udf_lpad",
- "udf_ltrim",
- "udf_map",
- "udf_minute",
- "udf_modulo",
- "udf_month",
- "udf_negative",
- "udf_not",
- "udf_notequal",
- "udf_notop",
- "udf_nvl",
- "udf_or",
- "udf_parse_url",
- "udf_PI",
- "udf_positive",
- "udf_pow",
- "udf_power",
- "udf_radians",
- "udf_rand",
- "udf_regexp",
- "udf_regexp_extract",
- "udf_regexp_replace",
- "udf_repeat",
- "udf_rlike",
- "udf_round",
- "udf_round_3",
- "udf_rpad",
- "udf_rtrim",
- "udf_second",
- "udf_sign",
- "udf_sin",
- "udf_smallint",
- "udf_space",
- "udf_sqrt",
- "udf_std",
- "udf_stddev",
- "udf_stddev_pop",
- "udf_stddev_samp",
- "udf_string",
- "udf_substring",
- "udf_subtract",
- "udf_sum",
- "udf_tan",
- "udf_tinyint",
- "udf_to_byte",
- "udf_to_date",
- "udf_to_double",
- "udf_to_float",
- "udf_to_long",
- "udf_to_short",
- "udf_translate",
- "udf_trim",
- "udf_ucase",
- "udf_unix_timestamp",
- "udf_upper",
- "udf_var_pop",
- "udf_var_samp",
- "udf_variance",
- "udf_weekofyear",
- "udf_when",
- "udf_xpath",
- "udf_xpath_boolean",
- "udf_xpath_double",
- "udf_xpath_float",
- "udf_xpath_int",
- "udf_xpath_long",
- "udf_xpath_short",
- "udf_xpath_string",
- "unicode_notation",
- "union10",
- "union11",
- "union13",
- "union14",
- "union15",
- "union16",
- "union17",
- "union18",
- "union19",
- "union2",
- "union20",
- "union22",
- "union23",
- "union24",
- "union25",
- "union26",
- "union27",
- "union28",
- "union29",
- "union3",
- "union30",
- "union31",
- "union33",
- "union34",
- "union4",
- "union5",
- "union6",
- "union7",
- "union8",
- "union9",
- "union_lateralview",
- "union_ppr",
- "union_remove_11",
- "union_remove_3",
- "union_remove_6",
- "union_script",
- "varchar_2",
- "varchar_join1",
- "varchar_union1"
- )
-}