diff options
author | Yanbo Liang <ybliang8@gmail.com> | 2015-03-21 10:53:04 +0800 |
---|---|---|
committer | Cheng Lian <lian@databricks.com> | 2015-03-21 10:53:04 +0800 |
commit | bc37c9743e065a0c756363c7b70e88f22a6e6edd (patch) | |
tree | 2cb3b4b332e336cff562aa30c691baf1469d7c7c | |
parent | 25e271d9fbb3394931d23822a1b2020e9d9b46b3 (diff) | |
download | spark-bc37c9743e065a0c756363c7b70e88f22a6e6edd.tar.gz spark-bc37c9743e065a0c756363c7b70e88f22a6e6edd.tar.bz2 spark-bc37c9743e065a0c756363c7b70e88f22a6e6edd.zip |
[SPARK-5821] [SQL] ParquetRelation2 CTAS should check if delete is successful
Do the same check as #4610 for ParquetRelation2.
Author: Yanbo Liang <ybliang8@gmail.com>
Closes #5107 from yanboliang/spark-5821-parquet and squashes the following commits:
7092c8d [Yanbo Liang] ParquetRelation2 CTAS should check if delete is successful
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala | 19 |
1 files changed, 14 insertions, 5 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala index 10b8876c1d..fbe7a419fe 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala @@ -611,13 +611,22 @@ private[sql] case class ParquetRelation2( val destinationPath = new Path(paths.head) if (overwrite) { - try { - destinationPath.getFileSystem(conf).delete(destinationPath, true) - } catch { - case e: IOException => + val fs = destinationPath.getFileSystem(conf) + if (fs.exists(destinationPath)) { + var success: Boolean = false + try { + success = fs.delete(destinationPath, true) + } catch { + case e: IOException => + throw new IOException( + s"Unable to clear output directory ${destinationPath.toString} prior" + + s" to writing to Parquet table:\n${e.toString}") + } + if (!success) { throw new IOException( s"Unable to clear output directory ${destinationPath.toString} prior" + - s" to writing to Parquet file:\n${e.toString}") + s" to writing to Parquet table.") + } } } |