diff options
| -rw-r--r-- | project/MimaExcludes.scala | 3 | ||||
| -rw-r--r-- | python/pyspark/sql/readwriter.py | 4 | ||||
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala | 33 |
3 files changed, 6 insertions, 34 deletions
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index b0d862d006..69161e0d61 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -349,6 +349,9 @@ object MimaExcludes { // [SPARK-13686][MLLIB][STREAMING] Add a constructor parameter `reqParam` to (Streaming)LinearRegressionWithSGD ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.regression.LinearRegressionWithSGD.this") ) ++ Seq( + // SPARK-15250 Remove deprecated json API in DataFrameReader + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.DataFrameReader.json") + ) ++ Seq( // SPARK-13920: MIMA checks should apply to @Experimental and @DeveloperAPI APIs ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.Aggregator.combineCombinersByKey"), ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.Aggregator.combineValuesByKey"), diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py index 20250b431b..7e79df33e8 100644 --- a/python/pyspark/sql/readwriter.py +++ b/python/pyspark/sql/readwriter.py @@ -241,8 +241,8 @@ class DataFrameReader(object): if columnNameOfCorruptRecord is not None: self.option("columnNameOfCorruptRecord", columnNameOfCorruptRecord) if isinstance(path, basestring): - return self._df(self._jreader.json(path)) - elif type(path) == list: + path = [path] + if type(path) == list: return self._df(self._jreader.json(self._sqlContext._sc._jvm.PythonUtils.toSeq(path))) elif isinstance(path, RDD): def func(iterator): diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala index 15d09e3edd..e1a64dfc5e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala @@ -285,38 +285,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * * You can set the following JSON-specific options to deal with non-standard JSON files: * <li>`primitivesAsString` (default `false`): infers all primitive values as a string type</li> - * <li>`allowComments` (default `false`): ignores Java/C++ style comment in JSON records</li> - * <li>`allowUnquotedFieldNames` (default `false`): allows unquoted JSON field names</li> - * <li>`allowSingleQuotes` (default `true`): allows single quotes in addition to double quotes - * </li> - * <li>`allowNumericLeadingZeros` (default `false`): allows leading zeros in numbers - * (e.g. 00012)</li> - * <li>`mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records - * during parsing.</li> - * <ul> - * <li>`PERMISSIVE` : sets other fields to `null` when it meets a corrupted record, and puts the - * malformed string into a new field configured by `columnNameOfCorruptRecord`. When - * a schema is set by user, it sets `null` for extra fields.</li> - * <li>`DROPMALFORMED` : ignores the whole corrupted records.</li> - * <li>`FAILFAST` : throws an exception when it meets corrupted records.</li> - * </ul> - * <li>`columnNameOfCorruptRecord` (default `_corrupt_record`): allows renaming the new field - * having malformed string created by `PERMISSIVE` mode. This overrides - * `spark.sql.columnNameOfCorruptRecord`.</li> - * - * @since 1.4.0 - */ - // TODO: Remove this one in Spark 2.0. - def json(path: String): DataFrame = format("json").load(path) - - /** - * Loads a JSON file (one object per line) and returns the result as a [[DataFrame]]. - * - * This function goes through the input once to determine the input schema. If you know the - * schema in advance, use the version that specifies the schema to avoid the extra scan. - * - * You can set the following JSON-specific options to deal with non-standard JSON files: - * <li>`primitivesAsString` (default `false`): infers all primitive values as a string type</li> * <li>`prefersDecimal` (default `false`): infers all floating-point values as a decimal * type. If the values do not fit in decimal, then it infers them as doubles.</li> * <li>`allowComments` (default `false`): ignores Java/C++ style comment in JSON records</li> @@ -342,6 +310,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * * @since 1.6.0 */ + @scala.annotation.varargs def json(paths: String*): DataFrame = format("json").load(paths : _*) /** |