diff options
author | Cheng Lian <lian@databricks.com> | 2015-08-20 11:00:24 -0700 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-08-20 11:00:29 -0700 |
commit | 85f9a61357994da5023b08b0a8a2eb09388ce7f8 (patch) | |
tree | 8bbde85c5339c54d58f87b77dcfcde34eae7e659 /sql/core/src/test/avro/parquet-compat.avdl | |
parent | 39e91fe2fd43044cc734d55625a3c03284b69f09 (diff) | |
download | spark-85f9a61357994da5023b08b0a8a2eb09388ce7f8.tar.gz spark-85f9a61357994da5023b08b0a8a2eb09388ce7f8.tar.bz2 spark-85f9a61357994da5023b08b0a8a2eb09388ce7f8.zip |
[SPARK-10136] [SQL] Fixes Parquet support for Avro array of primitive array
I caught SPARK-10136 while adding more test cases to `ParquetAvroCompatibilitySuite`. Actual bug fix code lies in `CatalystRowConverter.scala`.
Author: Cheng Lian <lian@databricks.com>
Closes #8341 from liancheng/spark-10136/parquet-avro-nested-primitive-array.
Diffstat (limited to 'sql/core/src/test/avro/parquet-compat.avdl')
-rw-r--r-- | sql/core/src/test/avro/parquet-compat.avdl | 19 |
1 files changed, 18 insertions, 1 deletions
diff --git a/sql/core/src/test/avro/parquet-compat.avdl b/sql/core/src/test/avro/parquet-compat.avdl index 8070d0a917..c5eb5b5164 100644 --- a/sql/core/src/test/avro/parquet-compat.avdl +++ b/sql/core/src/test/avro/parquet-compat.avdl @@ -34,7 +34,7 @@ protocol CompatibilityTest { string nested_string_column; } - record ParquetAvroCompat { + record AvroPrimitives { boolean bool_column; int int_column; long long_column; @@ -42,7 +42,9 @@ protocol CompatibilityTest { double double_column; bytes binary_column; string string_column; + } + record AvroOptionalPrimitives { union { null, boolean } maybe_bool_column; union { null, int } maybe_int_column; union { null, long } maybe_long_column; @@ -50,7 +52,22 @@ protocol CompatibilityTest { union { null, double } maybe_double_column; union { null, bytes } maybe_binary_column; union { null, string } maybe_string_column; + } + + record AvroNonNullableArrays { + array<string> strings_column; + union { null, array<int> } maybe_ints_column; + } + record AvroArrayOfArray { + array<array<int>> int_arrays_column; + } + + record AvroMapOfArray { + map<array<int>> string_to_ints_column; + } + + record ParquetAvroCompat { array<string> strings_column; map<int> string_to_int_column; map<array<Nested>> complex_column; |