diff options
author | Mosharaf Chowdhury <mosharaf@cs.berkeley.edu> | 2013-10-14 14:50:49 -0700 |
---|---|---|
committer | Mosharaf Chowdhury <mosharaf@cs.berkeley.edu> | 2013-10-16 21:33:33 -0700 |
commit | feb45d391f8d09c120d7d43e72e96e9bf9784fa0 (patch) | |
tree | 6c0464b06ee4f1854593c6985b931f172a364a62 /examples | |
parent | 6e5a60fab46daf5525749d9e71f1ccfaffddd34d (diff) | |
download | spark-feb45d391f8d09c120d7d43e72e96e9bf9784fa0.tar.gz spark-feb45d391f8d09c120d7d43e72e96e9bf9784fa0.tar.bz2 spark-feb45d391f8d09c120d7d43e72e96e9bf9784fa0.zip |
Default blockSize is 4MB.
BroadcastTest2 example added for testing broadcasts.
Diffstat (limited to 'examples')
-rw-r--r-- | examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala | 59 |
1 files changed, 59 insertions, 0 deletions
diff --git a/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala b/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala new file mode 100644 index 0000000000..4b96d0c9dd --- /dev/null +++ b/examples/src/main/scala/org/apache/spark/examples/BroadcastTest2.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.examples + +import org.apache.spark.SparkContext + +object BroadcastTest2 { + def main(args: Array[String]) { + if (args.length == 0) { + System.err.println("Usage: BroadcastTest2 <master> [slices] [numElem] [broadcastAlgo] [blockSize]") + System.exit(1) + } + + val bcName = if (args.length > 3) args(3) else "Http" + val blockSize = if (args.length > 4) args(4) else "4096" + + System.setProperty("spark.broadcast.factory", "org.apache.spark.broadcast." + bcName + "BroadcastFactory") + System.setProperty("spark.broadcast.blockSize", blockSize) + + val sc = new SparkContext(args(0), "Broadcast Test 2", + System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_EXAMPLES_JAR"))) + + val slices = if (args.length > 1) args(1).toInt else 2 + val num = if (args.length > 2) args(2).toInt else 1000000 + + var arr1 = new Array[Int](num) + for (i <- 0 until arr1.length) { + arr1(i) = i + } + + for (i <- 0 until 3) { + println("Iteration " + i) + println("===========") + val startTime = System.nanoTime + val barr1 = sc.broadcast(arr1) + sc.parallelize(1 to 10, slices).foreach { + i => println(barr1.value.size) + } + println("Iteration %d took %.0f milliseconds".format(i, (System.nanoTime - startTime) / 1E6)) + } + + System.exit(0) + } +} |