diff options
author | Andrew Or <andrew@databricks.com> | 2015-07-21 23:00:13 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-07-21 23:00:13 -0700 |
commit | f4785f5b82c57bce41d3dc26ed9e3c9e794c7558 (patch) | |
tree | 11ec9a823484baa02368244c09bf612d2ddf9c1d | |
parent | 63f4bcc73f5a09c1790cc3c333f08b18609de6a4 (diff) | |
download | spark-f4785f5b82c57bce41d3dc26ed9e3c9e794c7558.tar.gz spark-f4785f5b82c57bce41d3dc26ed9e3c9e794c7558.tar.bz2 spark-f4785f5b82c57bce41d3dc26ed9e3c9e794c7558.zip |
[SPARK-9232] [SQL] Duplicate code in JSONRelation
Author: Andrew Or <andrew@databricks.com>
Closes #7576 from andrewor14/clean-up-json-relation and squashes the following commits:
ea80803 [Andrew Or] Clean up duplicate code
-rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala | 50 |
1 files changed, 21 insertions, 29 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala index 25802d054a..922794ac9a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.json import java.io.IOException -import org.apache.hadoop.fs.Path +import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.rdd.RDD import org.apache.spark.sql.AnalysisException @@ -87,20 +87,7 @@ private[sql] class DefaultSource case SaveMode.Append => sys.error(s"Append mode is not supported by ${this.getClass.getCanonicalName}") case SaveMode.Overwrite => { - var success: Boolean = false - try { - success = fs.delete(filesystemPath, true) - } catch { - case e: IOException => - throw new IOException( - s"Unable to clear output directory ${filesystemPath.toString} prior" - + s" to writing to JSON table:\n${e.toString}") - } - if (!success) { - throw new IOException( - s"Unable to clear output directory ${filesystemPath.toString} prior" - + s" to writing to JSON table.") - } + JSONRelation.delete(filesystemPath, fs) true } case SaveMode.ErrorIfExists => @@ -195,20 +182,7 @@ private[sql] class JSONRelation( if (overwrite) { if (fs.exists(filesystemPath)) { - var success: Boolean = false - try { - success = fs.delete(filesystemPath, true) - } catch { - case e: IOException => - throw new IOException( - s"Unable to clear output directory ${filesystemPath.toString} prior" - + s" to writing to JSON table:\n${e.toString}") - } - if (!success) { - throw new IOException( - s"Unable to clear output directory ${filesystemPath.toString} prior" - + s" to writing to JSON table.") - } + JSONRelation.delete(filesystemPath, fs) } // Write the data. data.toJSON.saveAsTextFile(filesystemPath.toString) @@ -228,3 +202,21 @@ private[sql] class JSONRelation( case _ => false } } + +private object JSONRelation { + + /** Delete the specified directory to overwrite it with new JSON data. */ + def delete(dir: Path, fs: FileSystem): Unit = { + var success: Boolean = false + val failMessage = s"Unable to clear output directory $dir prior to writing to JSON table" + try { + success = fs.delete(dir, true /* recursive */) + } catch { + case e: IOException => + throw new IOException(s"$failMessage\n${e.toString}") + } + if (!success) { + throw new IOException(failMessage) + } + } +} |