aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrew Or <andrew@databricks.com>2015-05-13 16:28:37 -0700
committerAndrew Or <andrew@databricks.com>2015-05-13 16:28:37 -0700
commitf88ac701552a1a854247509db49d78f13515eae4 (patch)
treed5b59f95d778279142a70084b21ef9a50aec8427
parentf6e18388d993d99f768c6d547327e0720ec64224 (diff)
downloadspark-f88ac701552a1a854247509db49d78f13515eae4.tar.gz
spark-f88ac701552a1a854247509db49d78f13515eae4.tar.bz2
spark-f88ac701552a1a854247509db49d78f13515eae4.zip
[SPARK-7399] Spark compilation error for scala 2.11
Subsequent fix following #5966. I tried this out locally. Author: Andrew Or <andrew@databricks.com> Closes #6129 from andrewor14/211-compilation and squashes the following commits: 713868f [Andrew Or] Fix compilation issue for scala 2.11
-rw-r--r--core/src/main/scala/org/apache/spark/rdd/RDD.scala2
-rw-r--r--core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala4
-rw-r--r--core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala20
3 files changed, 14 insertions, 12 deletions
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 02a94baf37..f7fa37e4cd 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -1524,7 +1524,7 @@ abstract class RDD[T: ClassTag](
* doCheckpoint() is called recursively on the parent RDDs.
*/
private[spark] def doCheckpoint(): Unit = {
- RDDOperationScope.withScope(sc, "checkpoint", false, true) {
+ RDDOperationScope.withScope(sc, "checkpoint", allowNesting = false, ignoreParent = true) {
if (!doCheckpointCalled) {
doCheckpointCalled = true
if (checkpointData.isDefined) {
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
index 93ec606f2d..2725826f42 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
@@ -96,7 +96,7 @@ private[spark] object RDDOperationScope {
sc: SparkContext,
allowNesting: Boolean = false)(body: => T): T = {
val callerMethodName = Thread.currentThread.getStackTrace()(3).getMethodName
- withScope[T](sc, callerMethodName, allowNesting)(body)
+ withScope[T](sc, callerMethodName, allowNesting, ignoreParent = false)(body)
}
/**
@@ -116,7 +116,7 @@ private[spark] object RDDOperationScope {
sc: SparkContext,
name: String,
allowNesting: Boolean,
- ignoreParent: Boolean = false)(body: => T): T = {
+ ignoreParent: Boolean)(body: => T): T = {
// Save the old scope to restore it later
val scopeKey = SparkContext.RDD_SCOPE_KEY
val noOverrideKey = SparkContext.RDD_SCOPE_NO_OVERRIDE_KEY
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
index d75ecbf1f0..db465a6a9e 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDOperationScopeSuite.scala
@@ -61,11 +61,11 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter {
var rdd1: MyCoolRDD = null
var rdd2: MyCoolRDD = null
var rdd3: MyCoolRDD = null
- RDDOperationScope.withScope(sc, "scope1", allowNesting = false) {
+ RDDOperationScope.withScope(sc, "scope1", allowNesting = false, ignoreParent = false) {
rdd1 = new MyCoolRDD(sc)
- RDDOperationScope.withScope(sc, "scope2", allowNesting = false) {
+ RDDOperationScope.withScope(sc, "scope2", allowNesting = false, ignoreParent = false) {
rdd2 = new MyCoolRDD(sc)
- RDDOperationScope.withScope(sc, "scope3", allowNesting = false) {
+ RDDOperationScope.withScope(sc, "scope3", allowNesting = false, ignoreParent = false) {
rdd3 = new MyCoolRDD(sc)
}
}
@@ -84,11 +84,13 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter {
var rdd1: MyCoolRDD = null
var rdd2: MyCoolRDD = null
var rdd3: MyCoolRDD = null
- RDDOperationScope.withScope(sc, "scope1", allowNesting = true) { // allow nesting here
+ // allow nesting here
+ RDDOperationScope.withScope(sc, "scope1", allowNesting = true, ignoreParent = false) {
rdd1 = new MyCoolRDD(sc)
- RDDOperationScope.withScope(sc, "scope2", allowNesting = false) { // stop nesting here
+ // stop nesting here
+ RDDOperationScope.withScope(sc, "scope2", allowNesting = false, ignoreParent = false) {
rdd2 = new MyCoolRDD(sc)
- RDDOperationScope.withScope(sc, "scope3", allowNesting = false) {
+ RDDOperationScope.withScope(sc, "scope3", allowNesting = false, ignoreParent = false) {
rdd3 = new MyCoolRDD(sc)
}
}
@@ -107,11 +109,11 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter {
var rdd1: MyCoolRDD = null
var rdd2: MyCoolRDD = null
var rdd3: MyCoolRDD = null
- RDDOperationScope.withScope(sc, "scope1", allowNesting = true) {
+ RDDOperationScope.withScope(sc, "scope1", allowNesting = true, ignoreParent = false) {
rdd1 = new MyCoolRDD(sc)
- RDDOperationScope.withScope(sc, "scope2", allowNesting = true) {
+ RDDOperationScope.withScope(sc, "scope2", allowNesting = true, ignoreParent = false) {
rdd2 = new MyCoolRDD(sc)
- RDDOperationScope.withScope(sc, "scope3", allowNesting = true) {
+ RDDOperationScope.withScope(sc, "scope3", allowNesting = true, ignoreParent = false) {
rdd3 = new MyCoolRDD(sc)
}
}