aboutsummaryrefslogtreecommitdiff
path: root/python
diff options
context:
space:
mode:
authorCheng Lian <lian@databricks.com>2015-04-08 18:47:39 -0700
committerMichael Armbrust <michael@databricks.com>2015-04-08 18:47:39 -0700
commit891ada5be1e7fdd796380e2626d80843f2ef6017 (patch)
treeb5cdc95af70d2d4d4bb25d27bc082aa3efe1cf2d /python
parent7d7384c781ea72e1eabab3daca2e237e3b0fc666 (diff)
downloadspark-891ada5be1e7fdd796380e2626d80843f2ef6017.tar.gz
spark-891ada5be1e7fdd796380e2626d80843f2ef6017.tar.bz2
spark-891ada5be1e7fdd796380e2626d80843f2ef6017.zip
[SPARK-6696] [SQL] Adds HiveContext.refreshTable to PySpark
<!-- Reviewable:start --> [<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/5349) <!-- Reviewable:end --> Author: Cheng Lian <lian@databricks.com> Closes #5349 from liancheng/py-refresh-table and squashes the following commits: 004bec0 [Cheng Lian] Adds HiveContext.refreshTable to PySpark
Diffstat (limited to 'python')
-rw-r--r--python/pyspark/sql/context.py9
1 files changed, 9 insertions, 0 deletions
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 93e2d176a5..e8529a8f8e 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -574,6 +574,15 @@ class HiveContext(SQLContext):
def _get_hive_ctx(self):
return self._jvm.HiveContext(self._jsc.sc())
+ def refreshTable(self, tableName):
+ """Invalidate and refresh all the cached the metadata of the given
+ table. For performance reasons, Spark SQL or the external data source
+ library it uses might cache certain metadata about a table, such as the
+ location of blocks. When those change outside of Spark SQL, users should
+ call this function to invalidate the cache.
+ """
+ self._ssql_ctx.refreshTable(tableName)
+
class UDFRegistration(object):
"""Wrapper for user-defined function registration."""