aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorReynold Xin <rxin@databricks.com>2015-05-27 01:12:59 -0700
committerReynold Xin <rxin@databricks.com>2015-05-27 01:12:59 -0700
commit9f48bf6b3761d66c7dc50f076ed92aff21b7eea0 (patch)
treee55b83814eb8148c32fb17cc1828799c8cb096f1
parent4f98d7a7f1715273bc91f1903bb7e0f287cc7394 (diff)
downloadspark-9f48bf6b3761d66c7dc50f076ed92aff21b7eea0.tar.gz
spark-9f48bf6b3761d66c7dc50f076ed92aff21b7eea0.tar.bz2
spark-9f48bf6b3761d66c7dc50f076ed92aff21b7eea0.zip
[SPARK-7887][SQL] Remove EvaluatedType from SQL Expression.
This type is not really used. Might as well remove it. Author: Reynold Xin <rxin@databricks.com> Closes #6427 from rxin/evalutedType and squashes the following commits: 51a319a [Reynold Xin] [SPARK-7887][SQL] Remove EvaluatedType from SQL Expression.
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala10
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala8
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala4
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala16
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala6
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala3
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala1
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala6
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala1
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala6
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala2
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala4
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala8
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala12
-rw-r--r--sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala4
-rw-r--r--sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala9
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala2
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala2
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala2
-rw-r--r--sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala5
28 files changed, 32 insertions, 95 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 2999c2ef3e..bbb150c1e8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -67,7 +67,7 @@ case class UnresolvedAttribute(nameParts: Seq[String])
override def withName(newName: String): UnresolvedAttribute = UnresolvedAttribute.quoted(newName)
// Unresolved attributes are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"'$name"
@@ -85,7 +85,7 @@ case class UnresolvedFunction(name: String, children: Seq[Expression]) extends E
override lazy val resolved = false
// Unresolved functions are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"'$name(${children.mkString(",")})"
@@ -107,7 +107,7 @@ trait Star extends NamedExpression with trees.LeafNode[Expression] {
override lazy val resolved = false
// Star gets expanded at runtime so we never evaluate a Star.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
def expand(input: Seq[Attribute], resolver: Resolver): Seq[NamedExpression]
@@ -166,7 +166,7 @@ case class MultiAlias(child: Expression, names: Seq[String])
override lazy val resolved = false
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$child AS $names"
@@ -200,7 +200,7 @@ case class UnresolvedExtractValue(child: Expression, extraction: Expression)
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
override lazy val resolved = false
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$child[$extraction]"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
index c6217f07c4..1ffc95c676 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala
@@ -30,8 +30,6 @@ import org.apache.spark.sql.catalyst.trees
case class BoundReference(ordinal: Int, dataType: DataType, nullable: Boolean)
extends NamedExpression with trees.LeafNode[Expression] {
- type EvaluatedType = Any
-
override def toString: String = s"input[$ordinal]"
override def eval(input: Row): Any = input(ordinal)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index d8cf2b2e32..df3cdf2cdf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -105,8 +105,6 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
override def toString: String = s"CAST($child, $dataType)"
- type EvaluatedType = Any
-
// [[func]] assumes the input is no longer null because eval already does the null check.
@inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index c7ae9da7fc..d199287844 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -25,9 +25,6 @@ import org.apache.spark.sql.types._
abstract class Expression extends TreeNode[Expression] {
self: Product =>
- /** The narrowest possible type that is produced when this expression is evaluated. */
- type EvaluatedType <: Any
-
/**
* Returns true when an expression is a candidate for static evaluation before the query is
* executed.
@@ -44,7 +41,7 @@ abstract class Expression extends TreeNode[Expression] {
def references: AttributeSet = AttributeSet(children.flatMap(_.references.iterator))
/** Returns the result of evaluating this expression on a given input Row */
- def eval(input: Row = null): EvaluatedType
+ def eval(input: Row = null): Any
/**
* Returns `true` if this expression and all its children have been resolved to a specific schema
@@ -117,8 +114,7 @@ abstract class UnaryExpression extends Expression with trees.UnaryNode[Expressio
// not like a real expressions.
case class GroupExpression(children: Seq[Expression]) extends Expression {
self: Product =>
- type EvaluatedType = Seq[Any]
- override def eval(input: Row): EvaluatedType = throw new UnsupportedOperationException
+ override def eval(input: Row): Any = throw new UnsupportedOperationException
override def nullable: Boolean = false
override def foldable: Boolean = false
override def dataType: DataType = throw new UnsupportedOperationException
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
index e05926cbfe..b5f4e16745 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExtractValue.scala
@@ -92,8 +92,6 @@ object ExtractValue {
trait ExtractValue extends UnaryExpression {
self: Product =>
-
- type EvaluatedType = Any
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
index fe2873e0be..5b45347872 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUdf.scala
@@ -27,8 +27,6 @@ import org.apache.spark.sql.types.DataType
case class ScalaUdf(function: AnyRef, dataType: DataType, children: Seq[Expression])
extends Expression {
- type EvaluatedType = Any
-
override def nullable: Boolean = true
override def toString: String = s"scalaUDF(${children.mkString(",")})"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
index 83074eb1e6..195eec8e5c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala
@@ -36,7 +36,7 @@ case class SortOrder(child: Expression, direction: SortDirection) extends Expres
override def nullable: Boolean = child.nullable
// SortOrder itself is never evaluated.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$child ${if (direction == Ascending) "ASC" else "DESC"}"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
index f3830c6d3b..72eff5fe96 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala
@@ -37,7 +37,7 @@ abstract class AggregateExpression extends Expression {
* [[AggregateExpression.eval]] should never be invoked because [[AggregateExpression]]'s are
* replaced with a physical aggregate operator at runtime.
*/
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
@@ -74,8 +74,6 @@ abstract class AggregateFunction
extends AggregateExpression with Serializable with trees.LeafNode[Expression] {
self: Product =>
- override type EvaluatedType = Any
-
/** Base should return the generic aggregate expression that this function is computing */
val base: AggregateExpression
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index c7a37ad966..34c833b260 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -22,7 +22,6 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.types._
case class UnaryMinus(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = child.dataType
override def foldable: Boolean = child.foldable
@@ -45,7 +44,6 @@ case class UnaryMinus(child: Expression) extends UnaryExpression {
}
case class Sqrt(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = DoubleType
override def foldable: Boolean = child.foldable
@@ -72,8 +70,6 @@ case class Sqrt(child: Expression) extends UnaryExpression {
abstract class BinaryArithmetic extends BinaryExpression {
self: Product =>
- type EvaluatedType = Any
-
override lazy val resolved =
left.resolved && right.resolved &&
left.dataType == right.dataType &&
@@ -101,7 +97,7 @@ abstract class BinaryArithmetic extends BinaryExpression {
}
}
- def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any =
+ def evalInternal(evalE1: Any, evalE2: Any): Any =
sys.error(s"BinaryExpressions must either override eval or evalInternal")
}
@@ -244,7 +240,7 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme
case other => sys.error(s"Unsupported bitwise & operation on $other")
}
- override def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any = and(evalE1, evalE2)
+ override def evalInternal(evalE1: Any, evalE2: Any): Any = and(evalE1, evalE2)
}
/**
@@ -265,7 +261,7 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet
case other => sys.error(s"Unsupported bitwise | operation on $other")
}
- override def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any = or(evalE1, evalE2)
+ override def evalInternal(evalE1: Any, evalE2: Any): Any = or(evalE1, evalE2)
}
/**
@@ -286,14 +282,13 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme
case other => sys.error(s"Unsupported bitwise ^ operation on $other")
}
- override def evalInternal(evalE1: EvaluatedType, evalE2: EvaluatedType): Any = xor(evalE1, evalE2)
+ override def evalInternal(evalE1: Any, evalE2: Any): Any = xor(evalE1, evalE2)
}
/**
* A function that calculates bitwise not(~) of a number.
*/
case class BitwiseNot(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = child.dataType
override def foldable: Boolean = child.foldable
@@ -323,7 +318,6 @@ case class BitwiseNot(child: Expression) extends UnaryExpression {
}
case class MaxOf(left: Expression, right: Expression) extends Expression {
- type EvaluatedType = Any
override def foldable: Boolean = left.foldable && right.foldable
@@ -368,7 +362,6 @@ case class MaxOf(left: Expression, right: Expression) extends Expression {
}
case class MinOf(left: Expression, right: Expression) extends Expression {
- type EvaluatedType = Any
override def foldable: Boolean = left.foldable && right.foldable
@@ -416,7 +409,6 @@ case class MinOf(left: Expression, right: Expression) extends Expression {
* A function that get the absolute value of the numeric value.
*/
case class Abs(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def dataType: DataType = child.dataType
override def foldable: Boolean = child.foldable
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
index 956a2429b0..e7cd7131a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
@@ -24,8 +24,7 @@ import org.apache.spark.sql.types._
* Returns an Array containing the evaluation of all children expressions.
*/
case class CreateArray(children: Seq[Expression]) extends Expression {
- override type EvaluatedType = Any
-
+
override def foldable: Boolean = children.forall(_.foldable)
lazy val childTypes = children.map(_.dataType).distinct
@@ -54,7 +53,6 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
* TODO: [[CreateStruct]] does not support codegen.
*/
case class CreateStruct(children: Seq[NamedExpression]) extends Expression {
- override type EvaluatedType = Row
override def foldable: Boolean = children.forall(_.foldable)
@@ -71,7 +69,7 @@ case class CreateStruct(children: Seq[NamedExpression]) extends Expression {
override def nullable: Boolean = false
- override def eval(input: Row): EvaluatedType = {
+ override def eval(input: Row): Any = {
Row(children.map(_.eval(input)): _*)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
index adb94df7d1..65ba18924a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalFunctions.scala
@@ -21,7 +21,6 @@ import org.apache.spark.sql.types._
/** Return the unscaled Long value of a Decimal, assuming it fits in a Long */
case class UnscaledValue(child: Expression) extends UnaryExpression {
- override type EvaluatedType = Any
override def dataType: DataType = LongType
override def foldable: Boolean = child.foldable
@@ -40,7 +39,6 @@ case class UnscaledValue(child: Expression) extends UnaryExpression {
/** Create a Decimal from an unscaled Long value */
case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends UnaryExpression {
- override type EvaluatedType = Decimal
override def dataType: DataType = DecimalType(precision, scale)
override def foldable: Boolean = child.foldable
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 747a47bdde..cab40feb72 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -40,8 +40,6 @@ import org.apache.spark.sql.types._
abstract class Generator extends Expression {
self: Product =>
- override type EvaluatedType = TraversableOnce[Row]
-
// TODO ideally we should return the type of ArrayType(StructType),
// however, we don't keep the output field names in the Generator.
override def dataType: DataType = throw new UnsupportedOperationException
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 5f8c7354ae..d3ca3d9a4b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -78,14 +78,12 @@ case class Literal protected (value: Any, dataType: DataType) extends LeafExpres
override def toString: String = if (value != null) value.toString else "null"
- type EvaluatedType = Any
override def eval(input: Row): Any = value
}
// TODO: Specialize
case class MutableLiteral(var value: Any, dataType: DataType, nullable: Boolean = true)
extends LeafExpression {
- type EvaluatedType = Any
def update(expression: Expression, input: Row): Unit = {
value = expression.eval(input)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
index fcc06d3aa1..d5be44626f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.types._
*/
abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
extends BinaryExpression with Serializable with ExpectsInputTypes { self: Product =>
- type EvaluatedType = Any
+
override def symbol: String = null
override def expectedChildTypes: Seq[DataType] = Seq(DoubleType, DoubleType)
@@ -68,6 +68,7 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
case class Atan2(
left: Expression,
right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") {
+
override def eval(input: Row): Any = {
val evalE1 = left.eval(input)
if (evalE1 == null) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
index dc68469e06..cdcb8e2840 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.types._
abstract class MathematicalExpression(f: Double => Double, name: String)
extends UnaryExpression with Serializable with ExpectsInputTypes {
self: Product =>
- type EvaluatedType = Any
override def expectedChildTypes: Seq[DataType] = Seq(DoubleType)
override def dataType: DataType = DoubleType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 50be26d0b0..00565ec651 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -111,7 +111,6 @@ case class Alias(child: Expression, name: String)(
val explicitMetadata: Option[Metadata] = None)
extends NamedExpression with trees.UnaryNode[Expression] {
- override type EvaluatedType = Any
// Alias(Generator, xx) need to be transformed into Generate(generator, ...)
override lazy val resolved = childrenResolved && !child.isInstanceOf[Generator]
@@ -229,7 +228,7 @@ case class AttributeReference(
}
// Unresolved attributes are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"$name#${exprId.id}$typeSuffix"
@@ -240,7 +239,6 @@ case class AttributeReference(
* expression id or the unresolved indicator.
*/
case class PrettyAttribute(name: String) extends Attribute with trees.LeafNode[Expression] {
- type EvaluatedType = Any
override def toString: String = name
@@ -252,7 +250,7 @@ case class PrettyAttribute(name: String) extends Attribute with trees.LeafNode[E
override def withName(newName: String): Attribute = throw new UnsupportedOperationException
override def qualifiers: Seq[String] = throw new UnsupportedOperationException
override def exprId: ExprId = throw new UnsupportedOperationException
- override def eval(input: Row): EvaluatedType = throw new UnsupportedOperationException
+ override def eval(input: Row): Any = throw new UnsupportedOperationException
override def nullable: Boolean = throw new UnsupportedOperationException
override def dataType: DataType = NullType
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
index f9161cf34f..5070570b47 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullFunctions.scala
@@ -22,7 +22,6 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedException
import org.apache.spark.sql.types.DataType
case class Coalesce(children: Seq[Expression]) extends Expression {
- type EvaluatedType = Any
/** Coalesce is nullable if all of its children are nullable, or if it has no children. */
override def nullable: Boolean = !children.exists(!_.nullable)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 1d72a9eb83..e2d1c8115e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -35,8 +35,6 @@ trait Predicate extends Expression {
self: Product =>
override def dataType: DataType = BooleanType
-
- type EvaluatedType = Any
}
trait PredicateHelper {
@@ -341,8 +339,6 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
trueValue.dataType
}
- type EvaluatedType = Any
-
override def eval(input: Row): Any = {
if (true == predicate.eval(input)) {
trueValue.eval(input)
@@ -357,8 +353,6 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
trait CaseWhenLike extends Expression {
self: Product =>
- type EvaluatedType = Any
-
// Note that `branches` are considered in consecutive pairs (cond, val), and the optional last
// element is the value for the default catch-all case (if provided).
// Hence, `branches` consists of at least two elements, and can have an odd or even length.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
index 66d7c8b07c..de82c15680 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala
@@ -38,8 +38,6 @@ abstract class RDG(seed: Long) extends LeafExpression with Serializable {
*/
@transient protected lazy val rng = new XORShiftRandom(seed + TaskContext.get().partitionId())
- override type EvaluatedType = Double
-
override def nullable: Boolean = false
override def dataType: DataType = DoubleType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
index 4c44182278..b65bf165f2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/sets.scala
@@ -51,7 +51,6 @@ private[sql] class OpenHashSetUDT(
* Creates a new set of the specified type
*/
case class NewSet(elementType: DataType) extends LeafExpression {
- type EvaluatedType = Any
override def nullable: Boolean = false
@@ -69,7 +68,6 @@ case class NewSet(elementType: DataType) extends LeafExpression {
* For performance, this expression mutates its input during evaluation.
*/
case class AddItemToSet(item: Expression, set: Expression) extends Expression {
- type EvaluatedType = Any
override def children: Seq[Expression] = item :: set :: Nil
@@ -101,7 +99,6 @@ case class AddItemToSet(item: Expression, set: Expression) extends Expression {
* For performance, this expression mutates its left input set during evaluation.
*/
case class CombineSets(left: Expression, right: Expression) extends BinaryExpression {
- type EvaluatedType = Any
override def nullable: Boolean = left.nullable || right.nullable
@@ -133,7 +130,6 @@ case class CombineSets(left: Expression, right: Expression) extends BinaryExpres
* Returns the number of elements in the input set.
*/
case class CountSet(child: Expression) extends UnaryExpression {
- type EvaluatedType = Any
override def nullable: Boolean = child.nullable
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
index 7683e0990c..5da93fe9c6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
@@ -25,8 +25,6 @@ import org.apache.spark.sql.types._
trait StringRegexExpression extends ExpectsInputTypes {
self: BinaryExpression =>
- type EvaluatedType = Any
-
def escape(v: String): String
def matches(regex: Pattern, str: String): Boolean
@@ -114,8 +112,6 @@ case class RLike(left: Expression, right: Expression)
trait CaseConversionExpression extends ExpectsInputTypes {
self: UnaryExpression =>
- type EvaluatedType = Any
-
def convert(v: UTF8String): UTF8String
override def foldable: Boolean = child.foldable
@@ -159,8 +155,6 @@ trait StringComparison extends ExpectsInputTypes {
def compare(l: UTF8String, r: UTF8String): Boolean
- override type EvaluatedType = Any
-
override def nullable: Boolean = left.nullable || right.nullable
override def expectedChildTypes: Seq[DataType] = Seq(StringType, StringType)
@@ -211,8 +205,6 @@ case class EndsWith(left: Expression, right: Expression)
*/
case class Substring(str: Expression, pos: Expression, len: Expression)
extends Expression with ExpectsInputTypes {
-
- type EvaluatedType = Any
override def foldable: Boolean = str.foldable && pos.foldable && len.foldable
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index 099d67ca7f..2729b34a08 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -66,8 +66,6 @@ case class WindowSpecDefinition(
}
}
- type EvaluatedType = Any
-
override def children: Seq[Expression] = partitionSpec ++ orderSpec
override lazy val resolved: Boolean =
@@ -76,7 +74,7 @@ case class WindowSpecDefinition(
override def toString: String = simpleString
- override def eval(input: Row): EvaluatedType = throw new UnsupportedOperationException
+ override def eval(input: Row): Any = throw new UnsupportedOperationException
override def nullable: Boolean = true
override def foldable: Boolean = false
override def dataType: DataType = throw new UnsupportedOperationException
@@ -299,7 +297,7 @@ case class UnresolvedWindowFunction(
override def get(index: Int): Any =
throw new UnresolvedException(this, "get")
// Unresolved functions are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def toString: String = s"'$name(${children.mkString(",")})"
@@ -311,25 +309,25 @@ case class UnresolvedWindowFunction(
case class UnresolvedWindowExpression(
child: UnresolvedWindowFunction,
windowSpec: WindowSpecReference) extends UnaryExpression {
+
override def dataType: DataType = throw new UnresolvedException(this, "dataType")
override def foldable: Boolean = throw new UnresolvedException(this, "foldable")
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
override lazy val resolved = false
// Unresolved functions are transient at compile time and don't get evaluated during execution.
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
case class WindowExpression(
windowFunction: WindowFunction,
windowSpec: WindowSpecDefinition) extends Expression {
- override type EvaluatedType = Any
override def children: Seq[Expression] =
windowFunction :: windowSpec :: Nil
- override def eval(input: Row): EvaluatedType =
+ override def eval(input: Row): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
override def dataType: DataType = windowFunction.dataType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
index fb4217a448..80ba57a082 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala
@@ -169,7 +169,7 @@ case class HashPartitioning(expressions: Seq[Expression], numPartitions: Int)
override def keyExpressions: Seq[Expression] = expressions
- override def eval(input: Row = null): EvaluatedType =
+ override def eval(input: Row = null): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
@@ -213,6 +213,6 @@ case class RangePartitioning(ordering: Seq[SortOrder], numPartitions: Int)
override def keyExpressions: Seq[Expression] = ordering.map(_.child)
- override def eval(input: Row): EvaluatedType =
+ override def eval(input: Row): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index 3d10dab5ba..e5f77dcd96 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -25,12 +25,11 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.{IntegerType, StringType, NullType}
case class Dummy(optKey: Option[Expression]) extends Expression {
- def children: Seq[Expression] = optKey.toSeq
- def nullable: Boolean = true
- def dataType: NullType = NullType
+ override def children: Seq[Expression] = optKey.toSeq
+ override def nullable: Boolean = true
+ override def dataType: NullType = NullType
override lazy val resolved = true
- type EvaluatedType = Any
- def eval(input: Row): Any = null.asInstanceOf[Any]
+ override def eval(input: Row): Any = null.asInstanceOf[Any]
}
class TreeNodeSuite extends FunSuite {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
index 9ac732b55b..e228a60c90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/MonotonicallyIncreasingID.scala
@@ -39,8 +39,6 @@ private[sql] case class MonotonicallyIncreasingID() extends LeafExpression {
*/
@transient private[this] var count: Long = 0L
- override type EvaluatedType = Long
-
override def nullable: Boolean = false
override def dataType: DataType = LongType
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
index c2c6cbd491..1272793f88 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/expressions/SparkPartitionID.scala
@@ -27,8 +27,6 @@ import org.apache.spark.sql.types.{IntegerType, DataType}
*/
private[sql] case object SparkPartitionID extends LeafExpression {
- override type EvaluatedType = Int
-
override def nullable: Boolean = false
override def dataType: DataType = IntegerType
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
index 11b2897f76..55f3ff4709 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUdfs.scala
@@ -56,7 +56,7 @@ private[spark] case class PythonUDF(
def nullable: Boolean = true
- override def eval(input: Row): PythonUDF.this.EvaluatedType = {
+ override def eval(input: Row): Any = {
sys.error("PythonUDFs can not be directly evaluated.")
}
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
index bc6b3a2d58..7ec4f73325 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
@@ -75,7 +75,7 @@ private[hive] abstract class HiveFunctionRegistry
private[hive] case class HiveSimpleUdf(funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
extends Expression with HiveInspectors with Logging {
- type EvaluatedType = Any
+
type UDFType = UDF
override def nullable: Boolean = true
@@ -139,7 +139,6 @@ private[hive] class DeferredObjectAdapter(oi: ObjectInspector)
private[hive] case class HiveGenericUdf(funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
extends Expression with HiveInspectors with Logging {
type UDFType = GenericUDF
- type EvaluatedType = Any
override def nullable: Boolean = true
@@ -336,8 +335,6 @@ private[hive] case class HiveWindowFunction(
def nullable: Boolean = true
- override type EvaluatedType = Any
-
override def eval(input: Row): Any =
throw new TreeNodeException(this, s"No function to evaluate expression. type: ${this.nodeName}")