aboutsummaryrefslogtreecommitdiff
path: root/core/src/main
diff options
context:
space:
mode:
authorSean Owen <sowen@cloudera.com>2016-07-30 04:42:38 -0700
committerSean Owen <sowen@cloudera.com>2016-07-30 04:42:38 -0700
commit0dc4310b470c7e4355c0da67ca3373c3013cc9dd (patch)
tree9a8ac5aefbb25188958e9ae028c7ffdc117b705a /core/src/main
parentbbc247548ac6faeca15afc05c266cee37ef13416 (diff)
downloadspark-0dc4310b470c7e4355c0da67ca3373c3013cc9dd.tar.gz
spark-0dc4310b470c7e4355c0da67ca3373c3013cc9dd.tar.bz2
spark-0dc4310b470c7e4355c0da67ca3373c3013cc9dd.zip
[SPARK-16694][CORE] Use for/foreach rather than map for Unit expressions whose side effects are required
## What changes were proposed in this pull request? Use foreach/for instead of map where operation requires execution of body, not actually defining a transformation ## How was this patch tested? Jenkins Author: Sean Owen <sowen@cloudera.com> Closes #14332 from srowen/SPARK-16694.
Diffstat (limited to 'core/src/main')
-rw-r--r--core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala4
-rw-r--r--core/src/main/scala/org/apache/spark/deploy/Client.scala14
-rw-r--r--core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala14
-rw-r--r--core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala4
-rw-r--r--core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala4
5 files changed, 20 insertions, 20 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
index c416e835a9..7d5348266b 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
@@ -198,7 +198,7 @@ private[r] class RBackendHandler(server: RBackend)
args: Array[Object]): Option[Int] = {
val numArgs = args.length
- for (index <- 0 until parameterTypesOfMethods.length) {
+ for (index <- parameterTypesOfMethods.indices) {
val parameterTypes = parameterTypesOfMethods(index)
if (parameterTypes.length == numArgs) {
@@ -240,7 +240,7 @@ private[r] class RBackendHandler(server: RBackend)
// Convert args if needed
val parameterTypes = parameterTypesOfMethods(index)
- (0 until numArgs).map { i =>
+ for (i <- 0 until numArgs) {
if (parameterTypes(i) == classOf[Seq[Any]] && args(i).getClass.isArray) {
// Convert a Java array to scala Seq
args(i) = args(i).asInstanceOf[Array[_]].toSeq
diff --git a/core/src/main/scala/org/apache/spark/deploy/Client.scala b/core/src/main/scala/org/apache/spark/deploy/Client.scala
index 640f25f504..bf2dab6e71 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Client.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Client.scala
@@ -116,7 +116,7 @@ private class ClientEndpoint(
}
/* Find out driver status then exit the JVM */
- def pollAndReportStatus(driverId: String) {
+ def pollAndReportStatus(driverId: String): Unit = {
// Since ClientEndpoint is the only RpcEndpoint in the process, blocking the event loop thread
// is fine.
logInfo("... waiting before polling master for driver state")
@@ -137,12 +137,14 @@ private class ClientEndpoint(
case _ =>
}
// Exception, if present
- statusResponse.exception.map { e =>
- logError(s"Exception from cluster was: $e")
- e.printStackTrace()
- System.exit(-1)
+ statusResponse.exception match {
+ case Some(e) =>
+ logError(s"Exception from cluster was: $e")
+ e.printStackTrace()
+ System.exit(-1)
+ case _ =>
+ System.exit(0)
}
- System.exit(0)
}
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala
index 2ec9846e33..9c198a61f3 100644
--- a/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala
@@ -183,14 +183,14 @@ private class DefaultPartitionCoalescer(val balanceSlack: Double = 0.10)
getAllPrefLocs(prev)
- // gets all the preffered locations of the previous RDD and splits them into partitions
+ // gets all the preferred locations of the previous RDD and splits them into partitions
// with preferred locations and ones without
- def getAllPrefLocs(prev: RDD[_]) {
+ def getAllPrefLocs(prev: RDD[_]): Unit = {
val tmpPartsWithLocs = mutable.LinkedHashMap[Partition, Seq[String]]()
// first get the locations for each partition, only do this once since it can be expensive
prev.partitions.foreach(p => {
val locs = currPrefLocs(p, prev)
- if (locs.size > 0) {
+ if (locs.nonEmpty) {
tmpPartsWithLocs.put(p, locs)
} else {
partsWithoutLocs += p
@@ -198,13 +198,13 @@ private class DefaultPartitionCoalescer(val balanceSlack: Double = 0.10)
}
)
// convert it into an array of host to partition
- (0 to 2).map(x =>
- tmpPartsWithLocs.foreach(parts => {
+ for (x <- 0 to 2) {
+ tmpPartsWithLocs.foreach { parts =>
val p = parts._1
val locs = parts._2
if (locs.size > x) partsWithLocs += ((locs(x), p))
- } )
- )
+ }
+ }
}
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 515fd6f4e2..99afe0250c 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -155,7 +155,7 @@ class HadoopRDD[K, V](
logDebug("Cloning Hadoop Configuration")
val newJobConf = new JobConf(conf)
if (!conf.isInstanceOf[JobConf]) {
- initLocalJobConfFuncOpt.map(f => f(newJobConf))
+ initLocalJobConfFuncOpt.foreach(f => f(newJobConf))
}
newJobConf
}
@@ -174,7 +174,7 @@ class HadoopRDD[K, V](
HadoopRDD.CONFIGURATION_INSTANTIATION_LOCK.synchronized {
logDebug("Creating new JobConf and caching it for later re-use")
val newJobConf = new JobConf(conf)
- initLocalJobConfFuncOpt.map(f => f(newJobConf))
+ initLocalJobConfFuncOpt.foreach(f => f(newJobConf))
HadoopRDD.putCachedMetadata(jobConfCacheKey, newJobConf)
newJobConf
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index 959d6fd46d..263e6197a6 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -220,9 +220,7 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
command.addUris(CommandInfo.URI.newBuilder().setValue(uri.get))
}
- conf.getOption("spark.mesos.uris").map { uris =>
- setupUris(uris, command)
- }
+ conf.getOption("spark.mesos.uris").foreach(setupUris(_, command))
command.build()
}