diff options
3 files changed, 13 insertions, 11 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala index 64f2f00484..f9e24167a1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala @@ -84,14 +84,11 @@ class FileStreamSinkLog( private implicit val formats = Serialization.formats(NoTypeHints) - protected override val fileCleanupDelayMs = - sparkSession.conf.get(SQLConf.FILE_SINK_LOG_CLEANUP_DELAY) + protected override val fileCleanupDelayMs = sparkSession.sessionState.conf.fileSinkLogCleanupDelay - protected override val isDeletingExpiredLog = - sparkSession.conf.get(SQLConf.FILE_SINK_LOG_DELETION) + protected override val isDeletingExpiredLog = sparkSession.sessionState.conf.fileSinkLogDeletion - protected override val compactInterval = - sparkSession.conf.get(SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL) + protected override val compactInterval = sparkSession.sessionState.conf.fileSinkLogCompactInterval require(compactInterval > 0, s"Please set ${SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL.key} (was $compactInterval) " + "to a positive value.") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala index 8103309aff..4681f2ba08 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala @@ -39,16 +39,15 @@ class FileStreamSourceLog( // Configurations about metadata compaction protected override val compactInterval = - sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL) + sparkSession.sessionState.conf.fileSourceLogCompactInterval require(compactInterval > 0, s"Please set ${SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL.key} (was $compactInterval) to a " + s"positive value.") protected override val fileCleanupDelayMs = - sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_CLEANUP_DELAY) + sparkSession.sessionState.conf.fileSourceLogCleanupDelay - protected override val isDeletingExpiredLog = - sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_DELETION) + protected override val isDeletingExpiredLog = sparkSession.sessionState.conf.fileSourceLogDeletion private implicit val formats = Serialization.formats(NoTypeHints) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index f8b7a7f8ef..e67140fefe 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -620,10 +620,16 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION) - def fileSinkLogCompatInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) + def fileSinkLogCompactInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) def fileSinkLogCleanupDelay: Long = getConf(FILE_SINK_LOG_CLEANUP_DELAY) + def fileSourceLogDeletion: Boolean = getConf(FILE_SOURCE_LOG_DELETION) + + def fileSourceLogCompactInterval: Int = getConf(FILE_SOURCE_LOG_COMPACT_INTERVAL) + + def fileSourceLogCleanupDelay: Long = getConf(FILE_SOURCE_LOG_CLEANUP_DELAY) + def streamingSchemaInference: Boolean = getConf(STREAMING_SCHEMA_INFERENCE) def streamingPollingDelay: Long = getConf(STREAMING_POLLING_DELAY) |