diff options
| author | Felix Cheung <felixcheung_m@hotmail.com> | 2017-03-19 22:34:18 -0700 |
|---|---|---|
| committer | Felix Cheung <felixcheung@apache.org> | 2017-03-19 22:34:18 -0700 |
| commit | c40597720e8e66a6b11ca241b1ad387154a8fe72 (patch) | |
| tree | 59fbba7991666313502bd2b80d2d41d6d37ee593 /R/pkg/inst/tests | |
| parent | 0cdcf9114527a2c359c25e46fd6556b3855bfb28 (diff) | |
| download | spark-c40597720e8e66a6b11ca241b1ad387154a8fe72.tar.gz spark-c40597720e8e66a6b11ca241b1ad387154a8fe72.tar.bz2 spark-c40597720e8e66a6b11ca241b1ad387154a8fe72.zip | |
[SPARK-20020][SPARKR] DataFrame checkpoint API
## What changes were proposed in this pull request?
Add checkpoint, setCheckpointDir API to R
## How was this patch tested?
unit tests, manual tests
Author: Felix Cheung <felixcheung_m@hotmail.com>
Closes #17351 from felixcheung/rdfcheckpoint.
Diffstat (limited to 'R/pkg/inst/tests')
| -rw-r--r-- | R/pkg/inst/tests/testthat/test_rdd.R | 4 | ||||
| -rw-r--r-- | R/pkg/inst/tests/testthat/test_sparkSQL.R | 11 |
2 files changed, 13 insertions, 2 deletions
diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/inst/tests/testthat/test_rdd.R index 787ef51c50..b72c801dd9 100644 --- a/R/pkg/inst/tests/testthat/test_rdd.R +++ b/R/pkg/inst/tests/testthat/test_rdd.R @@ -143,8 +143,8 @@ test_that("PipelinedRDD support actions: cache(), persist(), unpersist(), checkp expect_false(rdd2@env$isCached) tempDir <- tempfile(pattern = "checkpoint") - setCheckpointDir(sc, tempDir) - checkpoint(rdd2) + setCheckpointDirSC(sc, tempDir) + checkpointRDD(rdd2) expect_true(rdd2@env$isCheckpointed) rdd2 <- lapply(rdd2, function(x) x) diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R index 9c38e0d866..cbc3569795 100644 --- a/R/pkg/inst/tests/testthat/test_sparkSQL.R +++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R @@ -841,6 +841,17 @@ test_that("cache(), storageLevel(), persist(), and unpersist() on a DataFrame", expect_true(is.data.frame(collect(df))) }) +test_that("setCheckpointDir(), checkpoint() on a DataFrame", { + checkpointDir <- file.path(tempdir(), "cproot") + expect_true(length(list.files(path = checkpointDir, all.files = TRUE)) == 0) + + setCheckpointDir(checkpointDir) + df <- read.json(jsonPath) + df <- checkpoint(df) + expect_is(df, "SparkDataFrame") + expect_false(length(list.files(path = checkpointDir, all.files = TRUE)) == 0) +}) + test_that("schema(), dtypes(), columns(), names() return the correct values/format", { df <- read.json(jsonPath) testSchema <- schema(df) |