diff options
Diffstat (limited to 'R')
-rw-r--r-- | R/pkg/R/DataFrame.R | 169 | ||||
-rw-r--r-- | R/pkg/R/SQLContext.R | 47 | ||||
-rw-r--r-- | R/pkg/R/mllib.R | 6 | ||||
-rw-r--r-- | R/pkg/R/schema.R | 24 | ||||
-rw-r--r-- | R/pkg/R/sparkR.R | 7 |
5 files changed, 92 insertions, 161 deletions
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index f3a3eff46d..583d3aefd4 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -35,12 +35,11 @@ setOldClass("structType") #' @slot env An R environment that stores bookkeeping states of the SparkDataFrame #' @slot sdf A Java object reference to the backing Scala DataFrame #' @seealso \link{createDataFrame}, \link{read.json}, \link{table} -#' @seealso \url{https://spark.apache.org/docs/latest/sparkr.html#sparkr-dataframes} +#' @seealso \url{https://spark.apache.org/docs/latest/sparkr.html#sparkdataframe} #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df <- createDataFrame(faithful) #'} setClass("SparkDataFrame", @@ -77,8 +76,7 @@ dataFrame <- function(sdf, isCached = FALSE) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' printSchema(df) @@ -102,8 +100,7 @@ setMethod("printSchema", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' dfSchema <- schema(df) @@ -126,8 +123,7 @@ setMethod("schema", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' explain(df, TRUE) @@ -157,8 +153,7 @@ setMethod("explain", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' isLocal(df) @@ -182,8 +177,7 @@ setMethod("isLocal", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' showDF(df) @@ -207,8 +201,7 @@ setMethod("showDF", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' df @@ -234,8 +227,7 @@ setMethod("show", "SparkDataFrame", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' dtypes(df) @@ -261,8 +253,7 @@ setMethod("dtypes", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' columns(df) @@ -396,8 +387,7 @@ setMethod("coltypes", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' coltypes(df) <- c("character", "integer") @@ -432,7 +422,7 @@ setMethod("coltypes<-", #' Creates a temporary view using the given name. #' -#' Creates a new temporary view using a SparkDataFrame in the SQLContext. If a +#' Creates a new temporary view using a SparkDataFrame in the Spark Session. If a #' temporary view with the same name already exists, replaces it. #' #' @param x A SparkDataFrame @@ -444,8 +434,7 @@ setMethod("coltypes<-", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' createOrReplaceTempView(df, "json_df") @@ -469,8 +458,7 @@ setMethod("createOrReplaceTempView", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' registerTempTable(df, "json_df") @@ -498,8 +486,7 @@ setMethod("registerTempTable", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df <- read.df(path, "parquet") #' df2 <- read.df(path2, "parquet") #' createOrReplaceTempView(df, "table1") @@ -526,8 +513,7 @@ setMethod("insertInto", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' cache(df) @@ -554,8 +540,7 @@ setMethod("cache", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' persist(df, "MEMORY_AND_DISK") @@ -582,8 +567,7 @@ setMethod("persist", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' persist(df, "MEMORY_AND_DISK") @@ -617,8 +601,7 @@ setMethod("unpersist", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' newDF <- repartition(df, 2L) @@ -660,8 +643,7 @@ setMethod("repartition", #' @noRd #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' newRDD <- toJSON(df) @@ -688,8 +670,7 @@ setMethod("toJSON", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' write.json(df, "/tmp/sparkr-tmp/") @@ -742,8 +723,7 @@ setMethod("write.orc", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' write.parquet(df, "/tmp/sparkr-tmp1/") @@ -781,8 +761,7 @@ setMethod("saveAsParquetFile", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.txt" #' df <- read.text(path) #' write.text(df, "/tmp/sparkr-tmp/") @@ -806,8 +785,7 @@ setMethod("write.text", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' distinctDF <- distinct(df) @@ -842,8 +820,7 @@ setMethod("unique", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' collect(sample(df, FALSE, 0.5)) @@ -883,8 +860,7 @@ setMethod("sample_frac", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' count(df) @@ -913,8 +889,7 @@ setMethod("nrow", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' ncol(df) @@ -936,8 +911,7 @@ setMethod("ncol", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' dim(df) @@ -960,8 +934,7 @@ setMethod("dim", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' collected <- collect(df) @@ -1032,8 +1005,7 @@ setMethod("collect", #' @export #' @examples #' \dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' limitedDF <- limit(df, 10) @@ -1053,8 +1025,7 @@ setMethod("limit", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' take(df, 2) @@ -1082,8 +1053,7 @@ setMethod("take", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' head(df) @@ -1105,8 +1075,7 @@ setMethod("head", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' first(df) @@ -1126,8 +1095,7 @@ setMethod("first", #' @noRd #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' rdd <- toRDD(df) @@ -1662,8 +1630,7 @@ setMethod("select", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' selectExpr(df, "col1", "(col2 * 5) as newCol") @@ -1692,8 +1659,7 @@ setMethod("selectExpr", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' newDF <- withColumn(df, "newCol", df$col1 * 5) @@ -1721,8 +1687,7 @@ setMethod("withColumn", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' newDF <- mutate(df, newCol = df$col1 * 5, newCol2 = df$col1 * 2) @@ -1812,8 +1777,7 @@ setMethod("transform", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' newDF <- withColumnRenamed(df, "col1", "newCol1") @@ -1837,8 +1801,7 @@ setMethod("withColumnRenamed", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' newDF <- rename(df, col1 = df$newCol1) @@ -1881,8 +1844,7 @@ setClassUnion("characterOrColumn", c("character", "Column")) #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' arrange(df, df$col1) @@ -1957,8 +1919,7 @@ setMethod("orderBy", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' filter(df, "col1 > 0") @@ -1998,8 +1959,7 @@ setMethod("where", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' dropDuplicates(df) @@ -2045,8 +2005,7 @@ setMethod("dropDuplicates", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- read.json(path) #' df2 <- read.json(path2) #' join(df1, df2) # Performs a Cartesian @@ -2103,8 +2062,7 @@ setMethod("join", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- read.json(path) #' df2 <- read.json(path2) #' merge(df1, df2) # Performs a Cartesian @@ -2235,8 +2193,7 @@ generateAliasesForIntersectedCols <- function (x, intersectedColNames, suffix) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- read.json(path) #' df2 <- read.json(path2) #' unioned <- unionAll(df, df2) @@ -2279,8 +2236,7 @@ setMethod("rbind", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- read.json(path) #' df2 <- read.json(path2) #' intersectDF <- intersect(df, df2) @@ -2306,8 +2262,7 @@ setMethod("intersect", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- read.json(path) #' df2 <- read.json(path2) #' exceptDF <- except(df, df2) @@ -2349,8 +2304,7 @@ setMethod("except", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' write.df(df, "myfile", "parquet", "overwrite") @@ -2408,8 +2362,7 @@ setMethod("saveDF", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' saveAsTable(df, "myfile") @@ -2445,8 +2398,7 @@ setMethod("saveAsTable", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' describe(df) @@ -2501,10 +2453,9 @@ setMethod("summary", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlCtx <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" -#' df <- read.json(sqlCtx, path) +#' df <- read.json(path) #' dropna(df) #' } setMethod("dropna", @@ -2555,10 +2506,9 @@ setMethod("na.omit", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlCtx <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" -#' df <- read.json(sqlCtx, path) +#' df <- read.json(path) #' fillna(df, 1) #' fillna(df, list("age" = 20, "name" = "unknown")) #' } @@ -2769,10 +2719,9 @@ setMethod("str", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlCtx <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" -#' df <- read.json(sqlCtx, path) +#' df <- read.json(path) #' drop(df, "col1") #' drop(df, c("col1", "col2")) #' drop(df, df$col1) @@ -2943,8 +2892,7 @@ setMethod("histogram", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' jdbcUrl <- "jdbc:mysql://localhost:3306/databasename" #' write.jdbc(df, jdbcUrl, "table", user = "username", password = "password") #' } @@ -2972,8 +2920,7 @@ setMethod("write.jdbc", #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df <- createDataFrame(data.frame(id = 1:1000)) #' df_list <- randomSplit(df, c(2, 3, 5), 0) #' # df_list contains 3 SparkDataFrames with each having about 200, 300 and 500 rows respectively diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R index b7e1c062c7..b846b3dec2 100644 --- a/R/pkg/R/SQLContext.R +++ b/R/pkg/R/SQLContext.R @@ -126,8 +126,7 @@ getDefaultSqlSource <- function() { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- as.DataFrame(iris) #' df2 <- as.DataFrame(list(3,4,5,6)) #' df3 <- createDataFrame(iris) @@ -238,8 +237,7 @@ as.DataFrame <- function(x, ...) { #' @noRd #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' rdd <- lapply(parallelize(sc, 1:10), function(x) list(a=x, b=as.character(x))) #' df <- toDF(rdd) #'} @@ -261,8 +259,7 @@ setMethod("toDF", signature(x = "RDD"), #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' df <- jsonFile(path) @@ -309,8 +306,7 @@ jsonFile <- function(x, ...) { #' @noRd #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' rdd <- texFile(sc, "path/to/json") #' df <- jsonRDD(sqlContext, rdd) #'} @@ -401,8 +397,7 @@ parquetFile <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.txt" #' df <- read.text(path) #' } @@ -432,8 +427,7 @@ read.text <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' createOrReplaceTempView(df, "table") @@ -464,8 +458,7 @@ sql <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' createOrReplaceTempView(df, "table") @@ -489,8 +482,7 @@ tableToDF <- function(tableName) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' tables("hive") #' } #' @name tables @@ -516,8 +508,7 @@ tables <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' tableNames("hive") #' } #' @name tableNames @@ -545,8 +536,7 @@ tableNames <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' createOrReplaceTempView(df, "table") @@ -575,8 +565,7 @@ cacheTable <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' path <- "path/to/file.json" #' df <- read.json(path) #' createOrReplaceTempView(df, "table") @@ -629,8 +618,7 @@ clearCache <- function() { #' @export #' @examples #' \dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df <- read.df(path, "parquet") #' createOrReplaceTempView(df, "table") #' dropTempTable("table") @@ -694,8 +682,7 @@ dropTempView <- function(viewName) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' df1 <- read.df("path/to/file.json", source = "json") #' schema <- structType(structField("name", "string"), #' structField("info", "map<string,double>")) @@ -758,9 +745,8 @@ loadDF <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) -#' df <- sparkRSQL.createExternalTable("myjson", path="path/to/json", source="json") +#' sparkR.session() +#' df <- createExternalTable("myjson", path="path/to/json", source="json") #' } #' @name createExternalTable #' @method createExternalTable default @@ -806,8 +792,7 @@ createExternalTable <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' jdbcUrl <- "jdbc:mysql://localhost:3306/databasename" #' df <- read.jdbc(jdbcUrl, "table", predicates = list("field<=123"), user = "username") #' df2 <- read.jdbc(jdbcUrl, "table2", partitionColumn = "index", lowerBound = 0, diff --git a/R/pkg/R/mllib.R b/R/pkg/R/mllib.R index ba2eee2fca..2127daee18 100644 --- a/R/pkg/R/mllib.R +++ b/R/pkg/R/mllib.R @@ -67,8 +67,7 @@ setClass("KMeansModel", representation(jobj = "jobj")) #' @export #' @examples #' \dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' data(iris) #' df <- createDataFrame(iris) #' model <- spark.glm(df, Sepal_Length ~ Sepal_Width, family="gaussian") @@ -115,8 +114,7 @@ setMethod( #' @export #' @examples #' \dontrun{ -#' sc <- sparkR.init() -#' sqlContext <- sparkRSQL.init(sc) +#' sparkR.session() #' data(iris) #' df <- createDataFrame(iris) #' model <- glm(Sepal_Length ~ Sepal_Width, df, family="gaussian") diff --git a/R/pkg/R/schema.R b/R/pkg/R/schema.R index 039aa008b3..89a2cfac3c 100644 --- a/R/pkg/R/schema.R +++ b/R/pkg/R/schema.R @@ -29,11 +29,11 @@ #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlCtx <- sparkRSQL.init(sc) -#' rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) }) -#' schema <- structType(structField("a", "integer"), structField("b", "string")) -#' df <- createDataFrame(sqlCtx, rdd, schema) +#' schema <- structType(structField("a", "integer"), structField("c", "string"), +#' structField("avg", "double")) +#' df1 <- gapply(df, list("a", "c"), +#' function(key, x) { y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE) }, +#' schema) #' } structType <- function(x, ...) { UseMethod("structType", x) @@ -90,13 +90,13 @@ print.structType <- function(x, ...) { #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' sqlCtx <- sparkRSQL.init(sc) -#' rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) }) -#' field1 <- structField("a", "integer", TRUE) -#' field2 <- structField("b", "string", TRUE) -#' schema <- structType(field1, field2) -#' df <- createDataFrame(sqlCtx, rdd, schema) +#' field1 <- structField("a", "integer") +#' field2 <- structField("c", "string") +#' field3 <- structField("avg", "double") +#' schema <- structType(field1, field2, field3) +#' df1 <- gapply(df, list("a", "c"), +#' function(key, x) { y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE) }, +#' schema) #' } structField <- function(x, ...) { diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 0dfd7b7530..d05660ccf6 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -89,9 +89,7 @@ sparkR.session.stop <- function() { #' (Deprecated) Initialize a new Spark Context. #' -#' This function initializes a new SparkContext. For details on how to initialize -#' and use SparkR, refer to SparkR programming guide at -#' \url{http://spark.apache.org/docs/latest/sparkr.html#starting-up-sparkcontext-sqlcontext}. +#' This function initializes a new SparkContext. #' #' @param master The Spark master URL #' @param appName Application name to register with cluster manager @@ -312,6 +310,9 @@ sparkRHive.init <- function(jsc = NULL) { #' Additional Spark properties can be set (...), and these named parameters take priority over #' over values in master, appName, named lists of sparkConfig. #' +#' For details on how to initialize and use SparkR, refer to SparkR programming guide at +#' \url{http://spark.apache.org/docs/latest/sparkr.html#starting-up-sparksession}. +#' #' @param master The Spark master URL #' @param appName Application name to register with cluster manager #' @param sparkHome Spark Home directory |