diff options
author | Aaron Davidson <aaron@databricks.com> | 2013-09-07 09:28:39 -0700 |
---|---|---|
committer | Aaron Davidson <aaron@databricks.com> | 2013-09-07 09:34:07 -0700 |
commit | 8001687af597056f630fb81f1edbcaf354c5388a (patch) | |
tree | 175599cec3e8cbcc3c4cd97ad962bc1408dd9027 /core | |
parent | b8a0b6ea5ee409dc51e121915794bccce92d457c (diff) | |
download | spark-8001687af597056f630fb81f1edbcaf354c5388a.tar.gz spark-8001687af597056f630fb81f1edbcaf354c5388a.tar.bz2 spark-8001687af597056f630fb81f1edbcaf354c5388a.zip |
Remove reflection, hard-code StorageLevels
The sc.StorageLevel -> StorageLevel pathway is a bit janky, but otherwise
the shell would have to call a private method of SparkContext. Having
StorageLevel available in sc also doesn't seem like the end of the world.
There may be a better solution, though.
As for creating the StorageLevel object itself, this seems to be the best
way in Python 2 for creating singleton, enum-like objects:
http://stackoverflow.com/questions/36932/how-can-i-represent-an-enum-in-python
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala | 11 |
1 files changed, 0 insertions, 11 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala index c3b770a42c..ccd3833964 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala @@ -28,7 +28,6 @@ import org.apache.spark.broadcast.Broadcast import org.apache.spark._ import org.apache.spark.rdd.RDD import org.apache.spark.rdd.PipedRDD -import org.apache.spark.storage.StorageLevel import org.apache.spark.util.Utils @@ -271,16 +270,6 @@ private[spark] object PythonRDD { JavaRDD.fromRDD(sc.sc.parallelize(objs, parallelism)) } - /** - * Returns the StorageLevel with the given string name. - * Throws an exception if the name is not a valid StorageLevel. - */ - def getStorageLevelByName(name: String) : StorageLevel = { - // In Scala, "val MEMORY_ONLY" produces a public getter by the same name. - val storageLevelGetter = StorageLevel.getClass().getDeclaredMethod(name) - return storageLevelGetter.invoke(StorageLevel).asInstanceOf[StorageLevel] - } - def writeIteratorToPickleFile[T](items: java.util.Iterator[T], filename: String) { import scala.collection.JavaConverters._ writeIteratorToPickleFile(items.asScala, filename) |