diff options
author | Cheng Lian <lian@databricks.com> | 2014-03-23 12:08:55 -0700 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-03-23 12:08:55 -0700 |
commit | 57a4379c031e5d5901ba580422207d6aa2f19749 (patch) | |
tree | bad7341671b0b9adeacc114432befe7f749d8f9d /sql/catalyst | |
parent | abf6714e27cf07a13819b35a4ca50ff9bb28b65c (diff) | |
download | spark-57a4379c031e5d5901ba580422207d6aa2f19749.tar.gz spark-57a4379c031e5d5901ba580422207d6aa2f19749.tar.bz2 spark-57a4379c031e5d5901ba580422207d6aa2f19749.zip |
[SPARK-1292] In-memory columnar representation for Spark SQL
This PR is rebased from the Catalyst repository, and contains the first version of in-memory columnar representation for Spark SQL. Compression support is not included yet and will be added later in a separate PR.
Author: Cheng Lian <lian@databricks.com>
Author: Cheng Lian <lian.cs.zju@gmail.com>
Closes #205 from liancheng/memColumnarSupport and squashes the following commits:
99dba41 [Cheng Lian] Restricted new objects/classes to `private[sql]'
0892ad8 [Cheng Lian] Addressed ScalaStyle issues
af1ad5e [Cheng Lian] Fixed some minor issues introduced during rebasing
0dbf2fb [Cheng Lian] Make necessary renaming due to rebase
a162d4d [Cheng Lian] Removed the unnecessary InMemoryColumnarRelation class
9bcae4b [Cheng Lian] Added Apache license
220ee1e [Cheng Lian] Added table scan operator for in-memory columnar support.
c701c7a [Cheng Lian] Using SparkSqlSerializer for generic object SerDe causes error, made a workaround
ed8608e [Cheng Lian] Added implicit conversion from DataType to ColumnType
b8a645a [Cheng Lian] Replaced KryoSerializer with an updated SparkSqlSerializer
b6c0a49 [Cheng Lian] Minor test suite refactoring
214be73 [Cheng Lian] Refactored BINARY and GENERIC to reduce duplicate code
da2f4d5 [Cheng Lian] Added Apache license
dbf7a38 [Cheng Lian] Added ColumnAccessor and test suite, refactored ColumnBuilder
c01a177 [Cheng Lian] Added column builder classes and test suite
f18ddc6 [Cheng Lian] Added ColumnTypes and test suite
2d09066 [Cheng Lian] Added KryoSerializer
34f3c19 [Cheng Lian] Added TypeTag field to all NativeTypes
acc5c48 [Cheng Lian] Added Hive test files to .gitignore
Diffstat (limited to 'sql/catalyst')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala | 14 |
1 files changed, 13 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala index 6eb2b62ecc..90a9f9f7e5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala @@ -19,7 +19,9 @@ package org.apache.spark.sql package catalyst package types -import expressions.Expression +import scala.reflect.runtime.universe.{typeTag, TypeTag} + +import org.apache.spark.sql.catalyst.expressions.Expression abstract class DataType { /** Matches any expression that evaluates to this DataType */ @@ -33,11 +35,13 @@ case object NullType extends DataType abstract class NativeType extends DataType { type JvmType + @transient val tag: TypeTag[JvmType] val ordering: Ordering[JvmType] } case object StringType extends NativeType { type JvmType = String + @transient lazy val tag = typeTag[JvmType] val ordering = implicitly[Ordering[JvmType]] } case object BinaryType extends DataType { @@ -45,6 +49,7 @@ case object BinaryType extends DataType { } case object BooleanType extends NativeType { type JvmType = Boolean + @transient lazy val tag = typeTag[JvmType] val ordering = implicitly[Ordering[JvmType]] } @@ -71,6 +76,7 @@ abstract class IntegralType extends NumericType { case object LongType extends IntegralType { type JvmType = Long + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[Long]] val integral = implicitly[Integral[Long]] val ordering = implicitly[Ordering[JvmType]] @@ -78,6 +84,7 @@ case object LongType extends IntegralType { case object IntegerType extends IntegralType { type JvmType = Int + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[Int]] val integral = implicitly[Integral[Int]] val ordering = implicitly[Ordering[JvmType]] @@ -85,6 +92,7 @@ case object IntegerType extends IntegralType { case object ShortType extends IntegralType { type JvmType = Short + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[Short]] val integral = implicitly[Integral[Short]] val ordering = implicitly[Ordering[JvmType]] @@ -92,6 +100,7 @@ case object ShortType extends IntegralType { case object ByteType extends IntegralType { type JvmType = Byte + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[Byte]] val integral = implicitly[Integral[Byte]] val ordering = implicitly[Ordering[JvmType]] @@ -110,6 +119,7 @@ abstract class FractionalType extends NumericType { case object DecimalType extends FractionalType { type JvmType = BigDecimal + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[BigDecimal]] val fractional = implicitly[Fractional[BigDecimal]] val ordering = implicitly[Ordering[JvmType]] @@ -117,6 +127,7 @@ case object DecimalType extends FractionalType { case object DoubleType extends FractionalType { type JvmType = Double + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[Double]] val fractional = implicitly[Fractional[Double]] val ordering = implicitly[Ordering[JvmType]] @@ -124,6 +135,7 @@ case object DoubleType extends FractionalType { case object FloatType extends FractionalType { type JvmType = Float + @transient lazy val tag = typeTag[JvmType] val numeric = implicitly[Numeric[Float]] val fractional = implicitly[Fractional[Float]] val ordering = implicitly[Ordering[JvmType]] |