diff options
| author | Takuya UESHIN <ueshin@happy-camper.st> | 2016-11-21 05:50:35 -0800 |
|---|---|---|
| committer | Herman van Hovell <hvanhovell@databricks.com> | 2016-11-21 05:50:35 -0800 |
| commit | 9f262ae163b6dca6526665b3ad12b3b2ea8fb873 (patch) | |
| tree | 63d29941f8769405f2d9e99c0c7213b43f30b15f | |
| parent | e811fbf9ed131bccbc46f3c5701c4ff317222fd9 (diff) | |
| download | spark-9f262ae163b6dca6526665b3ad12b3b2ea8fb873.tar.gz spark-9f262ae163b6dca6526665b3ad12b3b2ea8fb873.tar.bz2 spark-9f262ae163b6dca6526665b3ad12b3b2ea8fb873.zip | |
[SPARK-18398][SQL] Fix nullabilities of MapObjects and ExternalMapToCatalyst.
## What changes were proposed in this pull request?
The nullabilities of `MapObject` can be made more strict by relying on `inputObject.nullable` and `lambdaFunction.nullable`.
Also `ExternalMapToCatalyst.dataType` can be made more strict by relying on `valueConverter.nullable`.
## How was this patch tested?
Existing tests.
Author: Takuya UESHIN <ueshin@happy-camper.st>
Closes #15840 from ueshin/issues/SPARK-18398.
| -rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 0b36091ece..5c27179ec3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -461,14 +461,15 @@ case class MapObjects private( lambdaFunction: Expression, inputData: Expression) extends Expression with NonSQLExpression { - override def nullable: Boolean = true + override def nullable: Boolean = inputData.nullable override def children: Seq[Expression] = lambdaFunction :: inputData :: Nil override def eval(input: InternalRow): Any = throw new UnsupportedOperationException("Only code-generated evaluation is supported") - override def dataType: DataType = ArrayType(lambdaFunction.dataType) + override def dataType: DataType = + ArrayType(lambdaFunction.dataType, containsNull = lambdaFunction.nullable) override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val elementJavaType = ctx.javaType(loopVarDataType) @@ -642,7 +643,8 @@ case class ExternalMapToCatalyst private( override def foldable: Boolean = false - override def dataType: MapType = MapType(keyConverter.dataType, valueConverter.dataType) + override def dataType: MapType = MapType( + keyConverter.dataType, valueConverter.dataType, valueContainsNull = valueConverter.nullable) override def eval(input: InternalRow): Any = throw new UnsupportedOperationException("Only code-generated evaluation is supported") |