diff options
author | Cheng Lian <lian@databricks.com> | 2015-04-08 18:47:39 -0700 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2015-04-08 18:47:39 -0700 |
commit | 891ada5be1e7fdd796380e2626d80843f2ef6017 (patch) | |
tree | b5cdc95af70d2d4d4bb25d27bc082aa3efe1cf2d /python/pyspark/sql/context.py | |
parent | 7d7384c781ea72e1eabab3daca2e237e3b0fc666 (diff) | |
download | spark-891ada5be1e7fdd796380e2626d80843f2ef6017.tar.gz spark-891ada5be1e7fdd796380e2626d80843f2ef6017.tar.bz2 spark-891ada5be1e7fdd796380e2626d80843f2ef6017.zip |
[SPARK-6696] [SQL] Adds HiveContext.refreshTable to PySpark
<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/5349)
<!-- Reviewable:end -->
Author: Cheng Lian <lian@databricks.com>
Closes #5349 from liancheng/py-refresh-table and squashes the following commits:
004bec0 [Cheng Lian] Adds HiveContext.refreshTable to PySpark
Diffstat (limited to 'python/pyspark/sql/context.py')
-rw-r--r-- | python/pyspark/sql/context.py | 9 |
1 files changed, 9 insertions, 0 deletions
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py index 93e2d176a5..e8529a8f8e 100644 --- a/python/pyspark/sql/context.py +++ b/python/pyspark/sql/context.py @@ -574,6 +574,15 @@ class HiveContext(SQLContext): def _get_hive_ctx(self): return self._jvm.HiveContext(self._jsc.sc()) + def refreshTable(self, tableName): + """Invalidate and refresh all the cached the metadata of the given + table. For performance reasons, Spark SQL or the external data source + library it uses might cache certain metadata about a table, such as the + location of blocks. When those change outside of Spark SQL, users should + call this function to invalidate the cache. + """ + self._ssql_ctx.refreshTable(tableName) + class UDFRegistration(object): """Wrapper for user-defined function registration.""" |