diff options
author | Davies Liu <davies@databricks.com> | 2015-10-21 13:38:30 -0700 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-10-21 13:38:30 -0700 |
commit | f8c6bec65784de89b47e96a367d3f9790c1b3115 (patch) | |
tree | 29ebe356ff5dddfabf67bc0bca0b9fc7120e6d0b /sql/catalyst | |
parent | 7c74ebca05f40a2d8fe8f10f24a10486ce4f76c0 (diff) | |
download | spark-f8c6bec65784de89b47e96a367d3f9790c1b3115.tar.gz spark-f8c6bec65784de89b47e96a367d3f9790c1b3115.tar.bz2 spark-f8c6bec65784de89b47e96a367d3f9790c1b3115.zip |
[SPARK-11197][SQL] run SQL on files directly
This PR introduce a new feature to run SQL directly on files without create a table, for example:
```
select id from json.`path/to/json/files` as j
```
Author: Davies Liu <davies@databricks.com>
Closes #9173 from davies/source.
Diffstat (limited to 'sql/catalyst')
3 files changed, 12 insertions, 3 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 016dc293f4..beabacfc88 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -256,7 +256,7 @@ class Analyzer( catalog.lookupRelation(u.tableIdentifier, u.alias) } catch { case _: NoSuchTableException => - u.failAnalysis(s"Table Not Found: ${u.tableName}") + u.failAnalysis(s"Table not found: ${u.tableName}") } } @@ -264,7 +264,13 @@ class Analyzer( case i @ InsertIntoTable(u: UnresolvedRelation, _, _, _, _) => i.copy(table = EliminateSubQueries(getTable(u))) case u: UnresolvedRelation => - getTable(u) + try { + getTable(u) + } catch { + case _: AnalysisException if u.tableIdentifier.database.isDefined => + // delay the exception into CheckAnalysis, then it could be resolved as data source. + u + } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 7701fd0451..ab215407f7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -49,6 +49,9 @@ trait CheckAnalysis { plan.foreachUp { case p if p.analyzed => // Skip already analyzed sub-plans + case u: UnresolvedRelation => + u.failAnalysis(s"Table not found: ${u.tableIdentifier}") + case operator: LogicalPlan => operator transformExpressionsUp { case a: Attribute if !a.resolved => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 24af8483a7..0a1fa74bed 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -78,7 +78,7 @@ class AnalysisSuite extends AnalysisTest { test("resolve relations") { assertAnalysisError( - UnresolvedRelation(TableIdentifier("tAbLe"), None), Seq("Table Not Found: tAbLe")) + UnresolvedRelation(TableIdentifier("tAbLe"), None), Seq("Table not found: tAbLe")) checkAnalysis(UnresolvedRelation(TableIdentifier("TaBlE"), None), testRelation) |