diff options
author | Matei Zaharia <matei@databricks.com> | 2013-12-31 18:23:14 -0500 |
---|---|---|
committer | Matei Zaharia <matei@databricks.com> | 2013-12-31 18:23:14 -0500 |
commit | ba9338f104ccc71d4f342a3f96624a9b36895f48 (patch) | |
tree | 5a5b2bf6b34565a594a68fff59b9dd2af234be7f /python/pyspark | |
parent | 0fa5809768cf60ec62b4277f04e23a44dc1582e2 (diff) | |
parent | 63b411dd8664c27ac55586d8345733afad80961f (diff) | |
download | spark-ba9338f104ccc71d4f342a3f96624a9b36895f48.tar.gz spark-ba9338f104ccc71d4f342a3f96624a9b36895f48.tar.bz2 spark-ba9338f104ccc71d4f342a3f96624a9b36895f48.zip |
Merge remote-tracking branch 'apache/master' into conf2
Conflicts:
core/src/main/scala/org/apache/spark/rdd/CheckpointRDD.scala
streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala
streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
Diffstat (limited to 'python/pyspark')
-rw-r--r-- | python/pyspark/context.py | 9 | ||||
-rw-r--r-- | python/pyspark/tests.py | 4 |
2 files changed, 4 insertions, 9 deletions
diff --git a/python/pyspark/context.py b/python/pyspark/context.py index ee2f5cbe34..d77dd76765 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -345,17 +345,12 @@ class SparkContext(object): self._python_includes.append(filename) sys.path.append(os.path.join(SparkFiles.getRootDirectory(), filename)) # for tests in local mode - def setCheckpointDir(self, dirName, useExisting=False): + def setCheckpointDir(self, dirName): """ Set the directory under which RDDs are going to be checkpointed. The directory must be a HDFS path if running on a cluster. - - If the directory does not exist, it will be created. If the directory - exists and C{useExisting} is set to true, then the exisiting directory - will be used. Otherwise an exception will be thrown to prevent - accidental overriding of checkpoint files in the existing directory. """ - self._jsc.sc().setCheckpointDir(dirName, useExisting) + self._jsc.sc().setCheckpointDir(dirName) def _getJavaStorageLevel(self, storageLevel): """ diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 3987642bf4..7acb6eaf10 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -73,8 +73,8 @@ class TestCheckpoint(PySparkTestCase): time.sleep(1) # 1 second self.assertTrue(flatMappedRDD.isCheckpointed()) self.assertEqual(flatMappedRDD.collect(), result) - self.assertEqual(self.checkpointDir.name, - os.path.dirname(flatMappedRDD.getCheckpointFile())) + self.assertEqual("file:" + self.checkpointDir.name, + os.path.dirname(os.path.dirname(flatMappedRDD.getCheckpointFile()))) def test_checkpoint_and_restore(self): parCollection = self.sc.parallelize([1, 2, 3, 4]) |