diff options
Diffstat (limited to 'sql/core/src')
4 files changed, 261 insertions, 1 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala new file mode 100644 index 0000000000..e90a042431 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +/** + * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`. + * + * @since 2.0.0 + */ +abstract class RuntimeConfig { + + /** + * Sets the given Spark runtime configuration property. + * + * @since 2.0.0 + */ + def set(key: String, value: String): RuntimeConfig + + /** + * Sets the given Spark runtime configuration property. + * + * @since 2.0.0 + */ + def set(key: String, value: Boolean): RuntimeConfig + + /** + * Sets the given Spark runtime configuration property. + * + * @since 2.0.0 + */ + def set(key: String, value: Long): RuntimeConfig + + /** + * Returns the value of Spark runtime configuration property for the given key. + * + * @throws NoSuchElementException if the key is not set and does not have a default value + * @since 2.0.0 + */ + @throws[NoSuchElementException]("if the key is not set") + def get(key: String): String + + /** + * Returns the value of Spark runtime configuration property for the given key. + * + * @since 2.0.0 + */ + def getOption(key: String): Option[String] + + /** + * Resets the configuration property for the given key. + * + * @since 2.0.0 + */ + def unset(key: String): Unit + + /** + * Sets the given Hadoop configuration property. This is passed directly to Hadoop during I/O. + * + * @since 2.0.0 + */ + def setHadoop(key: String, value: String): RuntimeConfig + + /** + * Returns the value of the Hadoop configuration property. + * + * @throws NoSuchElementException if the key is not set + * @since 2.0.0 + */ + @throws[NoSuchElementException]("if the key is not set") + def getHadoop(key: String): String + + /** + * Returns the value of the Hadoop configuration property. + * + * @since 2.0.0 + */ + def getHadoopOption(key: String): Option[String] + + /** + * Resets the Hadoop configuration property for the given key. + * + * @since 2.0.0 + */ + def unsetHadoop(key: String): Unit +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala new file mode 100644 index 0000000000..058df1e3c1 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/RuntimeConfigImpl.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal + +import org.apache.spark.sql.RuntimeConfig + +/** + * Implementation for [[RuntimeConfig]]. + */ +class RuntimeConfigImpl extends RuntimeConfig { + + private val conf = new SQLConf + + private val hadoopConf = java.util.Collections.synchronizedMap( + new java.util.HashMap[String, String]()) + + override def set(key: String, value: String): RuntimeConfig = { + conf.setConfString(key, value) + this + } + + override def set(key: String, value: Boolean): RuntimeConfig = set(key, value.toString) + + override def set(key: String, value: Long): RuntimeConfig = set(key, value.toString) + + @throws[NoSuchElementException]("if the key is not set") + override def get(key: String): String = conf.getConfString(key) + + override def getOption(key: String): Option[String] = { + try Option(get(key)) catch { + case _: NoSuchElementException => None + } + } + + override def unset(key: String): Unit = conf.unsetConf(key) + + override def setHadoop(key: String, value: String): RuntimeConfig = { + hadoopConf.put(key, value) + this + } + + @throws[NoSuchElementException]("if the key is not set") + override def getHadoop(key: String): String = hadoopConf.synchronized { + if (hadoopConf.containsKey(key)) { + hadoopConf.get(key) + } else { + throw new NoSuchElementException(key) + } + } + + override def getHadoopOption(key: String): Option[String] = { + try Option(getHadoop(key)) catch { + case _: NoSuchElementException => None + } + } + + override def unsetHadoop(key: String): Unit = hadoopConf.remove(key) +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index c1e3f386b2..9a50ef77ef 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.internal -import java.util.Properties +import java.util.{NoSuchElementException, Properties} import scala.collection.JavaConverters._ import scala.collection.immutable @@ -649,6 +649,7 @@ class SQLConf extends Serializable with CatalystConf with ParserConf with Loggin } /** Return the value of Spark SQL configuration property for the given key. */ + @throws[NoSuchElementException]("if key is not set") def getConfString(key: String): String = { Option(settings.get(key)). orElse { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala new file mode 100644 index 0000000000..f809e01169 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/RuntimeConfigSuite.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.internal + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.RuntimeConfig + +class RuntimeConfigSuite extends SparkFunSuite { + + private def newConf(): RuntimeConfig = new RuntimeConfigImpl + + test("set and get") { + val conf = newConf() + conf + .set("k1", "v1") + .set("k2", 2) + .set("k3", value = false) + + assert(conf.get("k1") == "v1") + assert(conf.get("k2") == "2") + assert(conf.get("k3") == "false") + + intercept[NoSuchElementException] { + conf.get("notset") + } + } + + test("getOption") { + val conf = newConf().set("k1", "v1") + assert(conf.getOption("k1") == Some("v1")) + assert(conf.getOption("notset") == None) + } + + test("unset") { + val conf = newConf().set("k1", "v1") + assert(conf.get("k1") == "v1") + conf.unset("k1") + intercept[NoSuchElementException] { + conf.get("k1") + } + } + + test("set and get hadoop configuration") { + val conf = newConf() + conf + .setHadoop("k1", "v1") + .setHadoop("k2", "v2") + + assert(conf.getHadoop("k1") == "v1") + assert(conf.getHadoop("k2") == "v2") + + intercept[NoSuchElementException] { + conf.get("notset") + } + } + + test("getHadoopOption") { + val conf = newConf().setHadoop("k1", "v1") + assert(conf.getHadoopOption("k1") == Some("v1")) + assert(conf.getHadoopOption("notset") == None) + } + + test("unsetHadoop") { + val conf = newConf().setHadoop("k1", "v1") + assert(conf.getHadoop("k1") == "v1") + conf.unsetHadoop("k1") + intercept[NoSuchElementException] { + conf.getHadoop("k1") + } + } +} |