aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
diff options
context:
space:
mode:
Diffstat (limited to 'sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala')
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala28
1 files changed, 0 insertions, 28 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index d437c16a25..864a9cd3eb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -35,89 +35,73 @@ import org.apache.spark.sql.types.StructType
@Experimental
final class DataStreamReader private[sql](sparkSession: SparkSession) extends Logging {
/**
- * :: Experimental ::
* Specifies the input data source format.
*
* @since 2.0.0
*/
- @Experimental
def format(source: String): DataStreamReader = {
this.source = source
this
}
/**
- * :: Experimental ::
* Specifies the input schema. Some data sources (e.g. JSON) can infer the input schema
* automatically from data. By specifying the schema here, the underlying data source can
* skip the schema inference step, and thus speed up data loading.
*
* @since 2.0.0
*/
- @Experimental
def schema(schema: StructType): DataStreamReader = {
this.userSpecifiedSchema = Option(schema)
this
}
/**
- * :: Experimental ::
* Adds an input option for the underlying data source.
*
* @since 2.0.0
*/
- @Experimental
def option(key: String, value: String): DataStreamReader = {
this.extraOptions += (key -> value)
this
}
/**
- * :: Experimental ::
* Adds an input option for the underlying data source.
*
* @since 2.0.0
*/
- @Experimental
def option(key: String, value: Boolean): DataStreamReader = option(key, value.toString)
/**
- * :: Experimental ::
* Adds an input option for the underlying data source.
*
* @since 2.0.0
*/
- @Experimental
def option(key: String, value: Long): DataStreamReader = option(key, value.toString)
/**
- * :: Experimental ::
* Adds an input option for the underlying data source.
*
* @since 2.0.0
*/
- @Experimental
def option(key: String, value: Double): DataStreamReader = option(key, value.toString)
/**
- * :: Experimental ::
* (Scala-specific) Adds input options for the underlying data source.
*
* @since 2.0.0
*/
- @Experimental
def options(options: scala.collection.Map[String, String]): DataStreamReader = {
this.extraOptions ++= options
this
}
/**
- * :: Experimental ::
* Adds input options for the underlying data source.
*
* @since 2.0.0
*/
- @Experimental
def options(options: java.util.Map[String, String]): DataStreamReader = {
this.options(options.asScala)
this
@@ -125,13 +109,11 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
/**
- * :: Experimental ::
* Loads input data stream in as a [[DataFrame]], for data streams that don't require a path
* (e.g. external key-value stores).
*
* @since 2.0.0
*/
- @Experimental
def load(): DataFrame = {
val dataSource =
DataSource(
@@ -143,18 +125,15 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
}
/**
- * :: Experimental ::
* Loads input in as a [[DataFrame]], for data streams that read from some path.
*
* @since 2.0.0
*/
- @Experimental
def load(path: String): DataFrame = {
option("path", path).load()
}
/**
- * :: Experimental ::
* Loads a JSON file stream (one object per line) and returns the result as a [[DataFrame]].
*
* This function goes through the input once to determine the input schema. If you know the
@@ -198,11 +177,9 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
*
* @since 2.0.0
*/
- @Experimental
def json(path: String): DataFrame = format("json").load(path)
/**
- * :: Experimental ::
* Loads a CSV file stream and returns the result as a [[DataFrame]].
*
* This function will go through the input once to determine the input schema if `inferSchema`
@@ -262,11 +239,9 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
*
* @since 2.0.0
*/
- @Experimental
def csv(path: String): DataFrame = format("csv").load(path)
/**
- * :: Experimental ::
* Loads a Parquet file stream, returning the result as a [[DataFrame]].
*
* You can set the following Parquet-specific option(s) for reading Parquet files:
@@ -281,13 +256,11 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
*
* @since 2.0.0
*/
- @Experimental
def parquet(path: String): DataFrame = {
format("parquet").load(path)
}
/**
- * :: Experimental ::
* Loads text files and returns a [[DataFrame]] whose schema starts with a string column named
* "value", and followed by partitioned columns if there are any.
*
@@ -308,7 +281,6 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
*
* @since 2.0.0
*/
- @Experimental
def text(path: String): DataFrame = format("text").load(path)