diff options
author | Davies Liu <davies@databricks.com> | 2015-10-30 15:47:40 -0700 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2015-10-30 15:47:40 -0700 |
commit | 45029bfdea42eb8964f2ba697859687393d2a558 (patch) | |
tree | 45173a40ba6548f69f797d307ffb3a299bf6872e /core/src/test | |
parent | bb5a2af034196620d869fc9b1a400e014e718b8c (diff) | |
download | spark-45029bfdea42eb8964f2ba697859687393d2a558.tar.gz spark-45029bfdea42eb8964f2ba697859687393d2a558.tar.bz2 spark-45029bfdea42eb8964f2ba697859687393d2a558.zip |
[SPARK-11423] remove MapPartitionsWithPreparationRDD
Since we do not need to preserve a page before calling compute(), MapPartitionsWithPreparationRDD is not needed anymore.
This PR basically revert #8543, #8511, #8038, #8011
Author: Davies Liu <davies@databricks.com>
Closes #9381 from davies/remove_prepare2.
Diffstat (limited to 'core/src/test')
-rw-r--r-- | core/src/test/scala/org/apache/spark/rdd/MapPartitionsWithPreparationRDDSuite.scala | 66 |
1 files changed, 0 insertions, 66 deletions
diff --git a/core/src/test/scala/org/apache/spark/rdd/MapPartitionsWithPreparationRDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/MapPartitionsWithPreparationRDDSuite.scala deleted file mode 100644 index e281e817e4..0000000000 --- a/core/src/test/scala/org/apache/spark/rdd/MapPartitionsWithPreparationRDDSuite.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.rdd - -import scala.collection.mutable - -import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite, TaskContext} - -class MapPartitionsWithPreparationRDDSuite extends SparkFunSuite with LocalSparkContext { - - test("prepare called before parent partition is computed") { - sc = new SparkContext("local", "test") - - // Have the parent partition push a number to the list - val parent = sc.parallelize(1 to 100, 1).mapPartitions { iter => - TestObject.things.append(20) - iter - } - - // Push a different number during the prepare phase - val preparePartition = () => { TestObject.things.append(10) } - - // Push yet another number during the execution phase - val executePartition = ( - taskContext: TaskContext, - partitionIndex: Int, - notUsed: Unit, - parentIterator: Iterator[Int]) => { - TestObject.things.append(30) - TestObject.things.iterator - } - - // Verify that the numbers are pushed in the order expected - val rdd = new MapPartitionsWithPreparationRDD[Int, Int, Unit]( - parent, preparePartition, executePartition) - val result = rdd.collect() - assert(result === Array(10, 20, 30)) - - TestObject.things.clear() - // Zip two of these RDDs, both should be prepared before the parent is executed - val rdd2 = new MapPartitionsWithPreparationRDD[Int, Int, Unit]( - parent, preparePartition, executePartition) - val result2 = rdd.zipPartitions(rdd2)((a, b) => a).collect() - assert(result2 === Array(10, 10, 20, 30, 20, 30)) - } - -} - -private object TestObject { - val things = new mutable.ListBuffer[Int] -} |