diff options
author | Michael Armbrust <michael@databricks.com> | 2014-04-02 18:14:31 -0700 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-04-02 18:14:31 -0700 |
commit | 47ebea5468df2e4f94ef493c5403fcdcda8c5eb2 (patch) | |
tree | 3d6957ea93d01a3b4164f0bcc64932c5b375649c /sql/catalyst | |
parent | 9c65fa76f9d413e311a80f29d35d3ff7722e9476 (diff) | |
download | spark-47ebea5468df2e4f94ef493c5403fcdcda8c5eb2.tar.gz spark-47ebea5468df2e4f94ef493c5403fcdcda8c5eb2.tar.bz2 spark-47ebea5468df2e4f94ef493c5403fcdcda8c5eb2.zip |
[SQL] SPARK-1364 Improve datatype and test coverage for ScalaReflection schema inference.
Author: Michael Armbrust <michael@databricks.com>
Closes #293 from marmbrus/reflectTypes and squashes the following commits:
f54e8e8 [Michael Armbrust] Improve datatype and test coverage for ScalaReflection schema inference.
Diffstat (limited to 'sql/catalyst')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 976dda8d7e..5aaa63bf3b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -43,15 +43,25 @@ object ScalaReflection { val params = t.member("<init>": TermName).asMethod.paramss StructType( params.head.map(p => StructField(p.name.toString, schemaFor(p.typeSignature), true))) + // Need to decide if we actually need a special type here. + case t if t <:< typeOf[Array[Byte]] => BinaryType + case t if t <:< typeOf[Array[_]] => + sys.error(s"Only Array[Byte] supported now, use Seq instead of $t") case t if t <:< typeOf[Seq[_]] => val TypeRef(_, _, Seq(elementType)) = t ArrayType(schemaFor(elementType)) + case t if t <:< typeOf[Map[_,_]] => + val TypeRef(_, _, Seq(keyType, valueType)) = t + MapType(schemaFor(keyType), schemaFor(valueType)) case t if t <:< typeOf[String] => StringType case t if t <:< definitions.IntTpe => IntegerType case t if t <:< definitions.LongTpe => LongType + case t if t <:< definitions.FloatTpe => FloatType case t if t <:< definitions.DoubleTpe => DoubleType case t if t <:< definitions.ShortTpe => ShortType case t if t <:< definitions.ByteTpe => ByteType + case t if t <:< definitions.BooleanTpe => BooleanType + case t if t <:< typeOf[BigDecimal] => DecimalType } implicit class CaseClassRelation[A <: Product : TypeTag](data: Seq[A]) { |