diff options
author | gatorsmile <gatorsmile@gmail.com> | 2016-05-06 11:43:07 -0700 |
---|---|---|
committer | Yin Huai <yhuai@databricks.com> | 2016-05-06 11:43:07 -0700 |
commit | 5c8fad7b9bfd6677111a8e27e2574f82b04ec479 (patch) | |
tree | 5beee2fc0016634be8461e12d1617c7edfb7b3f9 /sql/catalyst | |
parent | 76ad04d9a0a7d4dfb762318d9c7be0d7720f4e1a (diff) | |
download | spark-5c8fad7b9bfd6677111a8e27e2574f82b04ec479.tar.gz spark-5c8fad7b9bfd6677111a8e27e2574f82b04ec479.tar.bz2 spark-5c8fad7b9bfd6677111a8e27e2574f82b04ec479.zip |
[SPARK-15108][SQL] Describe Permanent UDTF
#### What changes were proposed in this pull request?
When Describe a UDTF, the command returns a wrong result. The command is unable to find the function, which has been created and cataloged in the catalog but not in the functionRegistry.
This PR is to correct it. If the function is not in the functionRegistry, we will check the catalog for collecting the information of the UDTF function.
#### How was this patch tested?
Added test cases to verify the results
Author: gatorsmile <gatorsmile@gmail.com>
Closes #12885 from gatorsmile/showFunction.
Diffstat (limited to 'sql/catalyst')
6 files changed, 25 insertions, 15 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala index 2412ec46c7..ff13bcec43 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/NoSuchItemException.scala @@ -37,5 +37,10 @@ class NoSuchPartitionException( extends AnalysisException( s"Partition not found in table '$table' database '$db':\n" + spec.mkString("\n")) -class NoSuchFunctionException(db: String, func: String) +class NoSuchPermanentFunctionException(db: String, func: String) extends AnalysisException(s"Function '$func' not found in database '$db'") + +class NoSuchFunctionException(db: String, func: String) + extends AnalysisException( + s"Undefined function: '$func'. This function is neither a registered temporary function nor " + + s"a permanent function registered in the database '$db'.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 712770784b..9918bce455 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -28,7 +28,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchFunctionException, SimpleFunctionRegistry} +import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchFunctionException, NoSuchPermanentFunctionException, SimpleFunctionRegistry} import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias} @@ -644,9 +644,7 @@ class SessionCatalog( } protected def failFunctionLookup(name: String): Nothing = { - throw new AnalysisException(s"Undefined function: '$name'. This function is " + - s"neither a registered temporary function nor " + - s"a permanent function registered in the database '$currentDb'.") + throw new NoSuchFunctionException(db = currentDb, func = name) } /** @@ -709,7 +707,7 @@ class SessionCatalog( externalCatalog.getFunction(currentDb, name.funcName) } catch { case e: AnalysisException => failFunctionLookup(name.funcName) - case e: NoSuchFunctionException => failFunctionLookup(name.funcName) + case e: NoSuchPermanentFunctionException => failFunctionLookup(name.funcName) } loadFunctionResources(catalogFunction.resources) // Please note that qualifiedName is provided by the user. However, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 1d4e1ec3b8..2d7d0f9032 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -103,11 +103,11 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { import ctx._ val functionName = if (describeFuncName.STRING() != null) { - string(describeFuncName.STRING()) + FunctionIdentifier(string(describeFuncName.STRING()), database = None) } else if (describeFuncName.qualifiedName() != null) { - visitFunctionName(describeFuncName.qualifiedName).unquotedString + visitFunctionName(describeFuncName.qualifiedName) } else { - describeFuncName.getText + FunctionIdentifier(describeFuncName.getText, database = None) } DescribeFunction(functionName, EXTENDED != null) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala index fcffdbaaf0..0ec3ff3c25 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.plans.logical +import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.types.StringType @@ -29,11 +30,12 @@ trait Command /** * Returned for the "DESCRIBE FUNCTION [EXTENDED] functionName" command. + * * @param functionName The function to be described. * @param isExtended True if "DESCRIBE FUNCTION EXTENDED" is used. Otherwise, false. */ private[sql] case class DescribeFunction( - functionName: String, + functionName: FunctionIdentifier, isExtended: Boolean) extends LogicalPlan with Command { override def children: Seq[LogicalPlan] = Seq.empty diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala index ce00a03e76..50baebe8bf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.analysis import org.apache.spark.SparkFunSuite import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder @@ -68,7 +69,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite { // Commands assertNotSupportedInStreamingPlan( "commmands", - DescribeFunction("func", true), + DescribeFunction(FunctionIdentifier("func", database = None), true), outputMode = Append, expectedMsgs = "commands" :: Nil) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala index aaf84268af..f25e3fb404 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala @@ -57,10 +57,14 @@ class PlanParserSuite extends PlanTest { } test("describe function") { - assertEqual("describe function bar", DescribeFunction("bar", isExtended = false)) - assertEqual("describe function extended bar", DescribeFunction("bar", isExtended = true)) - assertEqual("describe function foo.bar", DescribeFunction("foo.bar", isExtended = false)) - assertEqual("describe function extended f.bar", DescribeFunction("f.bar", isExtended = true)) + assertEqual("describe function bar", + DescribeFunction(FunctionIdentifier("bar", database = None), isExtended = false)) + assertEqual("describe function extended bar", + DescribeFunction(FunctionIdentifier("bar", database = None), isExtended = true)) + assertEqual("describe function foo.bar", + DescribeFunction(FunctionIdentifier("bar", database = Option("foo")), isExtended = false)) + assertEqual("describe function extended f.bar", + DescribeFunction(FunctionIdentifier("bar", database = Option("f")), isExtended = true)) } test("set operations") { |