aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src/main/scala
diff options
context:
space:
mode:
authorwindpiger <songjun@outlook.com>2017-02-28 00:16:49 -0800
committerWenchen Fan <wenchen@databricks.com>2017-02-28 00:16:49 -0800
commita350bc16d36c58b48ac01f0258678ffcdb77e793 (patch)
tree5d294d922463cc1e37ef7db78d103913fda25a19 /sql/core/src/main/scala
parent73530383538ad72fdc3dd4c670485192f12ebc4e (diff)
downloadspark-a350bc16d36c58b48ac01f0258678ffcdb77e793.tar.gz
spark-a350bc16d36c58b48ac01f0258678ffcdb77e793.tar.bz2
spark-a350bc16d36c58b48ac01f0258678ffcdb77e793.zip
[SPARK-19748][SQL] refresh function has a wrong order to do cache invalidate and regenerate the inmemory var for InMemoryFileIndex with FileStatusCache
## What changes were proposed in this pull request? If we refresh a InMemoryFileIndex with a FileStatusCache, it will first use the FileStatusCache to re-generate the cachedLeafFiles etc, then call FileStatusCache.invalidateAll. While the order to do these two actions is wrong, this lead to the refresh action does not take effect. ``` override def refresh(): Unit = { refresh0() fileStatusCache.invalidateAll() } private def refresh0(): Unit = { val files = listLeafFiles(rootPaths) cachedLeafFiles = new mutable.LinkedHashMap[Path, FileStatus]() ++= files.map(f => f.getPath -> f) cachedLeafDirToChildrenFiles = files.toArray.groupBy(_.getPath.getParent) cachedPartitionSpec = null } ``` ## How was this patch tested? unit test added Author: windpiger <songjun@outlook.com> Closes #17079 from windpiger/fixInMemoryFileIndexRefresh.
Diffstat (limited to 'sql/core/src/main/scala')
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala2
1 files changed, 1 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
index 7531f0ae02..ee4d0863d9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
@@ -66,8 +66,8 @@ class InMemoryFileIndex(
}
override def refresh(): Unit = {
- refresh0()
fileStatusCache.invalidateAll()
+ refresh0()
}
private def refresh0(): Unit = {