+ Vignettes and other documentation
+
+
+
+
+
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.R b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.R
new file mode 100644
index 0000000..89a25b9
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.R
@@ -0,0 +1,579 @@
+## ----dynamic-chunk-options, include=FALSE-------------------------------------
+# In GitHub lint job, we don't have full JVM build
+# SparkR vignette fails to evaluate
+GITHUB_ACTIONS <- tolower(Sys.getenv("GITHUB_ACTIONS")) == "true"
+EVAL_CHUNK <- !GITHUB_ACTIONS
+
+## ----setup, include=FALSE, eval=EVAL_CHUNK------------------------------------
+library(knitr)
+opts_hooks$set(eval = function(options) {
+ # override eval to FALSE only on windows
+ if (.Platform$OS.type == "windows") {
+ options$eval = FALSE
+ }
+ options
+})
+r_tmp_dir <- tempdir()
+tmp_arg <- paste0("-Djava.io.tmpdir=", r_tmp_dir)
+sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg,
+ spark.executor.extraJavaOptions = tmp_arg)
+old_java_opt <- Sys.getenv("_JAVA_OPTIONS")
+Sys.setenv("_JAVA_OPTIONS" = paste("-XX:-UsePerfData", old_java_opt, sep = " "))
+
+## ---- message=FALSE, eval=EVAL_CHUNK------------------------------------------
+library(SparkR)
+
+## ---- include=FALSE, eval=EVAL_CHUNK------------------------------------------
+# disable eval if java version not supported
+override_eval <- tryCatch(!is.numeric(SparkR:::checkJavaVersion()),
+ error = function(e) { TRUE },
+ warning = function(e) { TRUE })
+
+if (override_eval) {
+ opts_hooks$set(eval = function(options) {
+ options$eval = FALSE
+ options
+ })
+}
+
+## ---- include=FALSE, eval=EVAL_CHUNK------------------------------------------
+install.spark()
+sparkR.session(master = "local[1]", sparkConfig = sparkSessionConfig, enableHiveSupport = FALSE)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+sparkR.session()
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+cars <- cbind(model = rownames(mtcars), mtcars)
+carsDF <- createDataFrame(cars)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+head(carsDF)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsSubDF <- select(carsDF, "model", "mpg", "hp")
+carsSubDF <- filter(carsSubDF, carsSubDF$hp >= 200)
+head(carsSubDF)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsGPDF <- summarize(groupBy(carsDF, carsDF$gear), count = n(carsDF$gear))
+head(carsGPDF)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsGP <- collect(carsGPDF)
+class(carsGP)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+model <- spark.glm(carsDF, mpg ~ wt + cyl)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+summary(model)
+
+## ---- eval=FALSE--------------------------------------------------------------
+# write.ml(model, path = "/HOME/tmp/mlModel/glmModel")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# sparkR.session.stop()
+
+## ---- eval=FALSE--------------------------------------------------------------
+# install.spark()
+
+## ---- eval=FALSE--------------------------------------------------------------
+# sparkR.session(sparkHome = "/HOME/spark")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# spark_warehouse_path <- file.path(path.expand('~'), "spark-warehouse")
+# sparkR.session(spark.sql.warehouse.dir = spark_warehouse_path)
+
+## ---- echo=FALSE, tidy = TRUE, eval=EVAL_CHUNK--------------------------------
+paste("Spark", packageVersion("SparkR"))
+
+## ---- eval=FALSE--------------------------------------------------------------
+# sparkR.session(master = "spark://local:7077")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# sparkR.session(master = "yarn")
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+df <- as.DataFrame(faithful)
+head(df)
+
+## ---- eval=FALSE--------------------------------------------------------------
+# sparkR.session(sparkPackages = "com.databricks:spark-avro_2.12:3.0.0")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# df <- read.df(csvPath, "csv", header = "true", inferSchema = "true", na.strings = "NA")
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+filePath <- paste0(sparkR.conf("spark.home"),
+ "/examples/src/main/resources/people.json")
+readLines(filePath, n = 2L)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+people <- read.df(filePath, "json")
+count(people)
+head(people)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+printSchema(people)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+people <- read.json(paste0(Sys.getenv("SPARK_HOME"),
+ c("/examples/src/main/resources/people.json",
+ "/examples/src/main/resources/people.json")))
+count(people)
+
+## ---- eval=FALSE--------------------------------------------------------------
+# write.df(people, path = "people.parquet", source = "parquet", mode = "overwrite")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
+#
+# txtPath <- paste0(sparkR.conf("spark.home"), "/examples/src/main/resources/kv1.txt")
+# sqlCMD <- sprintf("LOAD DATA LOCAL INPATH '%s' INTO TABLE src", txtPath)
+# sql(sqlCMD)
+#
+# results <- sql("FROM src SELECT key, value")
+#
+# # results is now a SparkDataFrame
+# head(results)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsDF
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+printSchema(carsDF)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+head(select(carsDF, "mpg"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+head(filter(carsDF, carsDF$mpg < 20))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+numCyl <- summarize(groupBy(carsDF, carsDF$cyl), count = n(carsDF$cyl))
+head(numCyl)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+mean(cube(carsDF, "cyl", "gear", "am"), "mpg")
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+mean(rollup(carsDF, "cyl", "gear", "am"), "mpg")
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsDF_km <- carsDF
+carsDF_km$kmpg <- carsDF_km$mpg * 1.61
+head(select(carsDF_km, "model", "mpg", "kmpg"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsSubDF <- select(carsDF, "model", "mpg", "cyl")
+ws <- orderBy(windowPartitionBy("cyl"), "mpg")
+carsRank <- withColumn(carsSubDF, "rank", over(rank(), ws))
+head(carsRank, n = 20L)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+carsSubDF <- select(carsDF, "model", "mpg")
+schema <- "model STRING, mpg DOUBLE, kmpg DOUBLE"
+out <- dapply(carsSubDF, function(x) { x <- cbind(x, x$mpg * 1.61) }, schema)
+head(collect(out))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+out <- dapplyCollect(
+ carsSubDF,
+ function(x) {
+ x <- cbind(x, "kmpg" = x$mpg * 1.61)
+ })
+head(out, 3)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+schema <- structType(structField("cyl", "double"), structField("max_mpg", "double"))
+result <- gapply(
+ carsDF,
+ "cyl",
+ function(key, x) {
+ y <- data.frame(key, max(x$mpg))
+ },
+ schema)
+head(arrange(result, "max_mpg", decreasing = TRUE))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+result <- gapplyCollect(
+ carsDF,
+ "cyl",
+ function(key, x) {
+ y <- data.frame(key, max(x$mpg))
+ colnames(y) <- c("cyl", "max_mpg")
+ y
+ })
+head(result[order(result$max_mpg, decreasing = TRUE), ])
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+costs <- exp(seq(from = log(1), to = log(1000), length.out = 5))
+train <- function(cost) {
+ stopifnot(requireNamespace("e1071", quietly = TRUE))
+ model <- e1071::svm(Species ~ ., data = iris, cost = cost)
+ summary(model)
+}
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+model.summaries <- spark.lapply(costs, train)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+class(model.summaries)
+
+## ---- include=FALSE, eval=EVAL_CHUNK------------------------------------------
+ops <- options()
+options(max.print=40)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+print(model.summaries[[2]])
+
+## ---- include=FALSE, eval=EVAL_CHUNK------------------------------------------
+options(ops)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+people <- read.df(paste0(sparkR.conf("spark.home"),
+ "/examples/src/main/resources/people.json"), "json")
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+createOrReplaceTempView(people, "people")
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+teenagers <- sql("SELECT name FROM people WHERE age >= 13 AND age <= 19")
+head(teenagers)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+splitDF_list <- randomSplit(carsDF, c(0.7, 0.3), seed = 0)
+carsDF_train <- splitDF_list[[1]]
+carsDF_test <- splitDF_list[[2]]
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+count(carsDF_train)
+head(carsDF_train)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+count(carsDF_test)
+head(carsDF_test)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+# load training data and create a DataFrame
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# fit a Linear SVM classifier model
+model <- spark.svmLinear(training, Survived ~ ., regParam = 0.01, maxIter = 10)
+summary(model)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+prediction <- predict(model, training)
+head(select(prediction, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+model <- spark.logit(training, Survived ~ ., regParam = 0.04741301)
+summary(model)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+fitted <- predict(model, training)
+head(select(fitted, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# Note in this case, Spark infers it is multinomial logistic regression, so family = "multinomial" is optional.
+model <- spark.logit(training, Class ~ ., regParam = 0.07815179)
+summary(model)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# fit a Multilayer Perceptron Classification Model
+model <- spark.mlp(training, Survived ~ Age + Sex, blockSize = 128, layers = c(2, 2), solver = "l-bfgs", maxIter = 100, tol = 0.5, stepSize = 1, seed = 1, initialWeights = c( 0, 0, 5, 5, 9, 9))
+
+## ---- include=FALSE, eval=EVAL_CHUNK------------------------------------------
+ops <- options()
+options(max.print=5)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+# check the summary of the fitted model
+summary(model)
+
+## ---- include=FALSE, eval=EVAL_CHUNK------------------------------------------
+options(ops)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+# make predictions use the fitted model
+predictions <- predict(model, training)
+head(select(predictions, predictions$prediction))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+titanic <- as.data.frame(Titanic)
+titanicDF <- createDataFrame(titanic[titanic$Freq > 0, -5])
+naiveBayesModel <- spark.naiveBayes(titanicDF, Survived ~ Class + Sex + Age)
+summary(naiveBayesModel)
+naiveBayesPrediction <- predict(naiveBayesModel, titanicDF)
+head(select(naiveBayesPrediction, "Class", "Sex", "Age", "Survived", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+
+model <- spark.fmClassifier(training, Survived ~ Age + Sex)
+summary(model)
+
+predictions <- predict(model, training)
+head(select(predictions, predictions$prediction))
+
+## ---- warning=FALSE, eval=EVAL_CHUNK------------------------------------------
+library(survival)
+ovarianDF <- createDataFrame(ovarian)
+aftModel <- spark.survreg(ovarianDF, Surv(futime, fustat) ~ ecog_ps + rx)
+summary(aftModel)
+aftPredictions <- predict(aftModel, ovarianDF)
+head(aftPredictions)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+gaussianGLM <- spark.glm(carsDF, mpg ~ wt + hp)
+summary(gaussianGLM)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+gaussianFitted <- predict(gaussianGLM, carsDF)
+head(select(gaussianFitted, "model", "prediction", "mpg", "wt", "hp"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+tweedieGLM1 <- spark.glm(carsDF, mpg ~ wt + hp, family = "tweedie", var.power = 0.0)
+summary(tweedieGLM1)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+tweedieGLM2 <- spark.glm(carsDF, mpg ~ wt + hp, family = "tweedie",
+ var.power = 1.2, link.power = 0.0)
+summary(tweedieGLM2)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+y <- c(3.0, 6.0, 8.0, 5.0, 7.0)
+x <- c(1.0, 2.0, 3.5, 3.0, 4.0)
+w <- rep(1.0, 5)
+data <- data.frame(y = y, x = x, w = w)
+df <- createDataFrame(data)
+isoregModel <- spark.isoreg(df, y ~ x, weightCol = "w")
+isoregFitted <- predict(isoregModel, df)
+head(select(isoregFitted, "x", "y", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+newDF <- createDataFrame(data.frame(x = c(1.5, 3.2)))
+head(predict(isoregModel, newDF))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+model <- spark.lm(carsDF, mpg ~ wt + hp)
+
+summary(model)
+predictions <- predict(model, carsDF)
+head(select(predictions, predictions$prediction))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+model <- spark.fmRegressor(carsDF, mpg ~ wt + hp)
+summary(model)
+predictions <- predict(model, carsDF)
+head(select(predictions, predictions$prediction))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+dtModel <- spark.decisionTree(df, Survived ~ ., type = "classification", maxDepth = 2)
+summary(dtModel)
+predictions <- predict(dtModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+gbtModel <- spark.gbt(df, Survived ~ ., type = "classification", maxDepth = 2, maxIter = 2)
+summary(gbtModel)
+predictions <- predict(gbtModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+rfModel <- spark.randomForest(df, Survived ~ ., type = "classification", maxDepth = 2, numTrees = 2)
+summary(rfModel)
+predictions <- predict(rfModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+model <- spark.bisectingKmeans(training, Class ~ Survived, k = 4)
+summary(model)
+fitted <- predict(model, training)
+head(select(fitted, "Class", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+X1 <- data.frame(V1 = rnorm(4), V2 = rnorm(4))
+X2 <- data.frame(V1 = rnorm(6, 3), V2 = rnorm(6, 4))
+data <- rbind(X1, X2)
+df <- createDataFrame(data)
+gmmModel <- spark.gaussianMixture(df, ~ V1 + V2, k = 2)
+summary(gmmModel)
+gmmFitted <- predict(gmmModel, df)
+head(select(gmmFitted, "V1", "V2", "prediction"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+kmeansModel <- spark.kmeans(carsDF, ~ mpg + hp + wt, k = 3)
+summary(kmeansModel)
+kmeansPredictions <- predict(kmeansModel, carsDF)
+head(select(kmeansPredictions, "model", "mpg", "hp", "wt", "prediction"), n = 20L)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+corpus <- data.frame(features = c(
+ "1 2 6 0 2 3 1 1 0 0 3",
+ "1 3 0 1 3 0 0 2 0 0 1",
+ "1 4 1 0 0 4 9 0 1 2 0",
+ "2 1 0 3 0 0 5 0 2 3 9",
+ "3 1 1 9 3 0 2 0 0 1 3",
+ "4 2 0 3 4 5 1 1 1 4 0",
+ "2 1 0 3 0 0 5 0 2 2 9",
+ "1 1 1 9 2 1 2 0 0 1 3",
+ "4 4 0 3 4 2 1 3 0 0 0",
+ "2 8 2 0 3 0 2 0 2 7 2",
+ "1 1 1 9 0 2 2 0 0 3 3",
+ "4 1 0 0 4 5 1 3 0 1 0"))
+corpusDF <- createDataFrame(corpus)
+model <- spark.lda(data = corpusDF, k = 5, optimizer = "em")
+summary(model)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+posterior <- spark.posterior(model, corpusDF)
+head(posterior)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+perplexity <- spark.perplexity(model, corpusDF)
+perplexity
+
+## ---- eval=FALSE--------------------------------------------------------------
+# ratings <- list(list(0, 0, 4.0), list(0, 1, 2.0), list(1, 1, 3.0), list(1, 2, 4.0),
+# list(2, 1, 1.0), list(2, 2, 5.0))
+# df <- createDataFrame(ratings, c("user", "item", "rating"))
+# model <- spark.als(df, "rating", "user", "item", rank = 10, reg = 0.1, nonnegative = TRUE)
+
+## ---- eval=FALSE--------------------------------------------------------------
+# stats <- summary(model)
+# userFactors <- stats$userFactors
+# itemFactors <- stats$itemFactors
+# head(userFactors)
+# head(itemFactors)
+
+## ---- eval=FALSE--------------------------------------------------------------
+# predicted <- predict(model, df)
+# head(predicted)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+df <- createDataFrame(list(list(0L, 1L, 1.0), list(0L, 2L, 1.0),
+ list(1L, 2L, 1.0), list(3L, 4L, 1.0),
+ list(4L, 0L, 0.1)),
+ schema = c("src", "dst", "weight"))
+head(spark.assignClusters(df, initMode = "degree", weightCol = "weight"))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+df <- selectExpr(createDataFrame(data.frame(rawItems = c(
+ "T,R,U", "T,S", "V,R", "R,U,T,V", "R,S", "V,S,U", "U,R", "S,T", "V,R", "V,U,S",
+ "T,V,U", "R,V", "T,S", "T,S", "S,T", "S,U", "T,R", "V,R", "S,V", "T,S,U"
+))), "split(rawItems, ',') AS items")
+
+fpm <- spark.fpGrowth(df, minSupport = 0.2, minConfidence = 0.5)
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+head(spark.freqItemsets(fpm))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+head(spark.associationRules(fpm))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+head(predict(fpm, df))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+df <- createDataFrame(list(list(list(list(1L, 2L), list(3L))),
+ list(list(list(1L), list(3L, 2L), list(1L, 2L))),
+ list(list(list(1L, 2L), list(5L))),
+ list(list(list(6L)))),
+ schema = c("sequence"))
+head(spark.findFrequentSequentialPatterns(df, minSupport = 0.5, maxPatternLength = 5L))
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+freqStats <- head(select(df, mean(df$Freq), sd(df$Freq)))
+freqMean <- freqStats[1]
+freqStd <- freqStats[2]
+
+test <- spark.kstest(df, "Freq", "norm", c(freqMean, freqStd))
+testSummary <- summary(test)
+testSummary
+
+## ---- eval=EVAL_CHUNK---------------------------------------------------------
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+gaussianGLM <- spark.glm(training, Freq ~ Sex + Age, family = "gaussian")
+
+# Save and then load a fitted MLlib model
+modelPath <- tempfile(pattern = "ml", fileext = ".tmp")
+write.ml(gaussianGLM, modelPath)
+gaussianGLM2 <- read.ml(modelPath)
+
+# Check model summary
+summary(gaussianGLM2)
+
+# Check model prediction
+gaussianPredictions <- predict(gaussianGLM2, training)
+head(gaussianPredictions)
+
+unlink(modelPath)
+
+## ---- eval=FALSE--------------------------------------------------------------
+# # Create DataFrame representing the stream of input lines from connection
+# lines <- read.stream("socket", host = hostname, port = port)
+#
+# # Split the lines into words
+# words <- selectExpr(lines, "explode(split(value, ' ')) as word")
+#
+# # Generate running word count
+# wordCounts <- count(groupBy(words, "word"))
+#
+# # Start running the query that prints the running counts to the console
+# query <- write.stream(wordCounts, "console", outputMode = "complete")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# topic <- read.stream("kafka",
+# kafka.bootstrap.servers = "host1:port1,host2:port2",
+# subscribe = "topic1")
+# keyvalue <- selectExpr(topic, "CAST(key AS STRING)", "CAST(value AS STRING)")
+
+## ---- eval=FALSE--------------------------------------------------------------
+# noAggDF <- select(where(deviceDataStreamingDf, "signal > 10"), "device")
+#
+# # Print new data to console
+# write.stream(noAggDF, "console")
+#
+# # Write new data to Parquet files
+# write.stream(noAggDF,
+# "parquet",
+# path = "path/to/destination/dir",
+# checkpointLocation = "path/to/checkpoint/dir")
+#
+# # Aggregate
+# aggDF <- count(groupBy(noAggDF, "device"))
+#
+# # Print updated aggregations to console
+# write.stream(aggDF, "console", outputMode = "complete")
+#
+# # Have all the aggregates in an in memory table. The query name will be the table name
+# write.stream(aggDF, "memory", queryName = "aggregates", outputMode = "complete")
+#
+# head(sql("select * from aggregates"))
+
+## ---- echo=FALSE, eval=EVAL_CHUNK---------------------------------------------
+sparkR.session.stop()
+
+## ----cleanup, include=FALSE, eval=EVAL_CHUNK----------------------------------
+SparkR:::uninstallDownloadedSpark()
+
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.Rmd b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.Rmd
new file mode 100644
index 0000000..4a51076
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.Rmd
@@ -0,0 +1,1294 @@
+---
+title: "SparkR - Practical Guide"
+output:
+ rmarkdown::html_vignette:
+ toc: true
+ toc_depth: 4
+vignette: >
+ %\VignetteIndexEntry{SparkR - Practical Guide}
+ %\VignetteEngine{knitr::rmarkdown}
+ \usepackage[utf8]{inputenc}
+---
+
+
+
+```{r dynamic-chunk-options, include=FALSE}
+# In GitHub lint job, we don't have full JVM build
+# SparkR vignette fails to evaluate
+GITHUB_ACTIONS <- tolower(Sys.getenv("GITHUB_ACTIONS")) == "true"
+EVAL_CHUNK <- !GITHUB_ACTIONS
+```
+
+
+```{r setup, include=FALSE, eval=EVAL_CHUNK}
+library(knitr)
+opts_hooks$set(eval = function(options) {
+ # override eval to FALSE only on windows
+ if (.Platform$OS.type == "windows") {
+ options$eval = FALSE
+ }
+ options
+})
+r_tmp_dir <- tempdir()
+tmp_arg <- paste0("-Djava.io.tmpdir=", r_tmp_dir)
+sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg,
+ spark.executor.extraJavaOptions = tmp_arg)
+old_java_opt <- Sys.getenv("_JAVA_OPTIONS")
+Sys.setenv("_JAVA_OPTIONS" = paste("-XX:-UsePerfData", old_java_opt, sep = " "))
+```
+
+## Overview
+
+SparkR is an R package that provides a light-weight frontend to use Apache Spark from R. With Spark `r packageVersion("SparkR")`, SparkR provides a distributed data frame implementation that supports data processing operations like selection, filtering, aggregation etc. and distributed machine learning using [MLlib](https://spark.apache.org/mllib/).
+
+## Getting Started
+
+We begin with an example running on the local machine and provide an overview of the use of SparkR: data ingestion, data processing and machine learning.
+
+First, let's load and attach the package.
+```{r, message=FALSE, eval=EVAL_CHUNK}
+library(SparkR)
+```
+
+```{r, include=FALSE, eval=EVAL_CHUNK}
+# disable eval if java version not supported
+override_eval <- tryCatch(!is.numeric(SparkR:::checkJavaVersion()),
+ error = function(e) { TRUE },
+ warning = function(e) { TRUE })
+
+if (override_eval) {
+ opts_hooks$set(eval = function(options) {
+ options$eval = FALSE
+ options
+ })
+}
+```
+
+`SparkSession` is the entry point into SparkR which connects your R program to a Spark cluster. You can create a `SparkSession` using `sparkR.session` and pass in options such as the application name, any Spark packages depended on, etc.
+
+We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see [Spark Session](#SetupSparkSession).
+
+```{r, include=FALSE, eval=EVAL_CHUNK}
+install.spark()
+sparkR.session(master = "local[1]", sparkConfig = sparkSessionConfig, enableHiveSupport = FALSE)
+```
+
+```{r, eval=EVAL_CHUNK}
+sparkR.session()
+```
+
+The operations in SparkR are centered around an R class called `SparkDataFrame`. It is a distributed collection of data organized into named columns, which is conceptually equivalent to a table in a relational database or a data frame in R, but with richer optimizations under the hood.
+
+`SparkDataFrame` can be constructed from a wide array of sources such as: structured data files, tables in Hive, external databases, or existing local R data frames. For example, we create a `SparkDataFrame` from a local R data frame,
+
+```{r, eval=EVAL_CHUNK}
+cars <- cbind(model = rownames(mtcars), mtcars)
+carsDF <- createDataFrame(cars)
+```
+
+We can view the first few rows of the `SparkDataFrame` by `head` or `showDF` function.
+```{r, eval=EVAL_CHUNK}
+head(carsDF)
+```
+
+Common data processing operations such as `filter` and `select` are supported on the `SparkDataFrame`.
+```{r, eval=EVAL_CHUNK}
+carsSubDF <- select(carsDF, "model", "mpg", "hp")
+carsSubDF <- filter(carsSubDF, carsSubDF$hp >= 200)
+head(carsSubDF)
+```
+
+SparkR can use many common aggregation functions after grouping.
+
+```{r, eval=EVAL_CHUNK}
+carsGPDF <- summarize(groupBy(carsDF, carsDF$gear), count = n(carsDF$gear))
+head(carsGPDF)
+```
+
+The results `carsDF` and `carsSubDF` are `SparkDataFrame` objects. To convert back to R `data.frame`, we can use `collect`. **Caution**: This can cause your interactive environment to run out of memory, though, because `collect()` fetches the entire distributed `DataFrame` to your client, which is acting as a Spark driver.
+```{r, eval=EVAL_CHUNK}
+carsGP <- collect(carsGPDF)
+class(carsGP)
+```
+
+SparkR supports a number of commonly used machine learning algorithms. Under the hood, SparkR uses MLlib to train the model. Users can call `summary` to print a summary of the fitted model, `predict` to make predictions on new data, and `write.ml`/`read.ml` to save/load fitted models.
+
+SparkR supports a subset of R formula operators for model fitting, including ‘~’, ‘.’, ‘:’, ‘+’, and ‘-‘. We use linear regression as an example.
+```{r, eval=EVAL_CHUNK}
+model <- spark.glm(carsDF, mpg ~ wt + cyl)
+```
+
+The result matches that returned by R `glm` function applied to the corresponding `data.frame` `mtcars` of `carsDF`. In fact, for Generalized Linear Model, we specifically expose `glm` for `SparkDataFrame` as well so that the above is equivalent to `model <- glm(mpg ~ wt + cyl, data = carsDF)`.
+
+```{r, eval=EVAL_CHUNK}
+summary(model)
+```
+
+The model can be saved by `write.ml` and loaded back using `read.ml`.
+```{r, eval=FALSE}
+write.ml(model, path = "/HOME/tmp/mlModel/glmModel")
+```
+
+In the end, we can stop Spark Session by running
+```{r, eval=FALSE}
+sparkR.session.stop()
+```
+
+## Setup
+
+### Installation
+
+Different from many other R packages, to use SparkR, you need an additional installation of Apache Spark. The Spark installation will be used to run a backend process that will compile and execute SparkR programs.
+
+After installing the SparkR package, you can call `sparkR.session` as explained in the previous section to start and it will check for the Spark installation. If you are working with SparkR from an interactive shell (e.g. R, RStudio) then Spark is downloaded and cached automatically if it is not found. Alternatively, we provide an easy-to-use function `install.spark` for running this manually. If you don't have Spark installed on the computer, you may download it from [Apache Spark Website](https://spark.apache.org/downloads.html).
+
+```{r, eval=FALSE}
+install.spark()
+```
+
+If you already have Spark installed, you don't have to install again and can pass the `sparkHome` argument to `sparkR.session` to let SparkR know where the existing Spark installation is.
+
+```{r, eval=FALSE}
+sparkR.session(sparkHome = "/HOME/spark")
+```
+
+### Spark Session {#SetupSparkSession}
+
+
+In addition to `sparkHome`, many other options can be specified in `sparkR.session`. For a complete list, see [Starting up: SparkSession](https://spark.apache.org/docs/latest/sparkr.html#starting-up-sparksession) and [SparkR API doc](https://spark.apache.org/docs/latest/api/R/reference/sparkR.session.html).
+
+In particular, the following Spark driver properties can be set in `sparkConfig`.
+
+Property Name | Property group | spark-submit equivalent
+---------------- | ------------------ | ----------------------
+`spark.driver.memory` | Application Properties | `--driver-memory`
+`spark.driver.extraClassPath` | Runtime Environment | `--driver-class-path`
+`spark.driver.extraJavaOptions` | Runtime Environment | `--driver-java-options`
+`spark.driver.extraLibraryPath` | Runtime Environment | `--driver-library-path`
+`spark.kerberos.keytab` | Application Properties | `--keytab`
+`spark.kerberos.principal` | Application Properties | `--principal`
+
+**For Windows users**: Due to different file prefixes across operating systems, to avoid the issue of potential wrong prefix, a current workaround is to specify `spark.sql.warehouse.dir` when starting the `SparkSession`.
+
+```{r, eval=FALSE}
+spark_warehouse_path <- file.path(path.expand('~'), "spark-warehouse")
+sparkR.session(spark.sql.warehouse.dir = spark_warehouse_path)
+```
+
+
+#### Cluster Mode
+SparkR can connect to remote Spark clusters. [Cluster Mode Overview](https://spark.apache.org/docs/latest/cluster-overview.html) is a good introduction to different Spark cluster modes.
+
+When connecting SparkR to a remote Spark cluster, make sure that the Spark version and Hadoop version on the machine match the corresponding versions on the cluster. Current SparkR package is compatible with
+```{r, echo=FALSE, tidy = TRUE, eval=EVAL_CHUNK}
+paste("Spark", packageVersion("SparkR"))
+```
+It should be used both on the local computer and on the remote cluster.
+
+To connect, pass the URL of the master node to `sparkR.session`. A complete list can be seen in [Spark Master URLs](https://spark.apache.org/docs/latest/submitting-applications.html#master-urls).
+For example, to connect to a local standalone Spark master, we can call
+
+```{r, eval=FALSE}
+sparkR.session(master = "spark://local:7077")
+```
+
+For YARN cluster, SparkR supports the client mode with the master set as "yarn".
+```{r, eval=FALSE}
+sparkR.session(master = "yarn")
+```
+Yarn cluster mode is not supported in the current version.
+
+## Data Import
+
+### Local Data Frame
+The simplest way is to convert a local R data frame into a `SparkDataFrame`. Specifically we can use `as.DataFrame` or `createDataFrame` and pass in the local R data frame to create a `SparkDataFrame`. As an example, the following creates a `SparkDataFrame` based using the `faithful` dataset from R.
+```{r, eval=EVAL_CHUNK}
+df <- as.DataFrame(faithful)
+head(df)
+```
+
+### Data Sources
+SparkR supports operating on a variety of data sources through the `SparkDataFrame` interface. You can check the Spark SQL Programming Guide for more [specific options](https://spark.apache.org/docs/latest/sql-programming-guide.html#manually-specifying-options) that are available for the built-in data sources.
+
+The general method for creating `SparkDataFrame` from data sources is `read.df`. This method takes in the path for the file to load and the type of data source, and the currently active Spark Session will be used automatically. SparkR supports reading CSV, JSON and Parquet files natively and through Spark Packages you can find data source connectors for popular file formats like Avro. These packages can be added with `sparkPackages` parameter when initializing SparkSession using `sparkR.session`.
+
+```{r, eval=FALSE}
+sparkR.session(sparkPackages = "com.databricks:spark-avro_2.12:3.0.0")
+```
+
+We can see how to use data sources using an example CSV input file. For more information please refer to SparkR [read.df](https://spark.apache.org/docs/latest/api/R/reference/read.df.html) API documentation.
+```{r, eval=FALSE}
+df <- read.df(csvPath, "csv", header = "true", inferSchema = "true", na.strings = "NA")
+```
+
+The data sources API natively supports JSON formatted input files. Note that the file that is used here is not a typical JSON file. Each line in the file must contain a separate, self-contained valid JSON object. As a consequence, a regular multi-line JSON file will most often fail.
+
+Let's take a look at the first two lines of the raw JSON file used here.
+
+```{r, eval=EVAL_CHUNK}
+filePath <- paste0(sparkR.conf("spark.home"),
+ "/examples/src/main/resources/people.json")
+readLines(filePath, n = 2L)
+```
+
+We use `read.df` to read that into a `SparkDataFrame`.
+
+```{r, eval=EVAL_CHUNK}
+people <- read.df(filePath, "json")
+count(people)
+head(people)
+```
+
+SparkR automatically infers the schema from the JSON file.
+```{r, eval=EVAL_CHUNK}
+printSchema(people)
+```
+
+If we want to read multiple JSON files, `read.json` can be used.
+```{r, eval=EVAL_CHUNK}
+people <- read.json(paste0(Sys.getenv("SPARK_HOME"),
+ c("/examples/src/main/resources/people.json",
+ "/examples/src/main/resources/people.json")))
+count(people)
+```
+
+The data sources API can also be used to save out `SparkDataFrames` into multiple file formats. For example we can save the `SparkDataFrame` from the previous example to a Parquet file using `write.df`.
+```{r, eval=FALSE}
+write.df(people, path = "people.parquet", source = "parquet", mode = "overwrite")
+```
+
+### Hive Tables
+You can also create SparkDataFrames from Hive tables. To do this we will need to create a SparkSession with Hive support which can access tables in the Hive MetaStore. Note that Spark should have been built with Hive support and more details can be found in the [SQL Programming Guide](https://spark.apache.org/docs/latest/sql-programming-guide.html). In SparkR, by default it will attempt to create a SparkSession with Hive support enabled (`enableHiveSupport = TRUE`).
+
+```{r, eval=FALSE}
+sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
+
+txtPath <- paste0(sparkR.conf("spark.home"), "/examples/src/main/resources/kv1.txt")
+sqlCMD <- sprintf("LOAD DATA LOCAL INPATH '%s' INTO TABLE src", txtPath)
+sql(sqlCMD)
+
+results <- sql("FROM src SELECT key, value")
+
+# results is now a SparkDataFrame
+head(results)
+```
+
+
+## Data Processing
+
+**To dplyr users**: SparkR has similar interface as dplyr in data processing. However, some noticeable differences are worth mentioning in the first place. We use `df` to represent a `SparkDataFrame` and `col` to represent the name of column here.
+
+1. indicate columns. SparkR uses either a character string of the column name or a Column object constructed with `$` to indicate a column. For example, to select `col` in `df`, we can write `select(df, "col")` or `select(df, df$col)`.
+
+2. describe conditions. In SparkR, the Column object representation can be inserted into the condition directly, or we can use a character string to describe the condition, without referring to the `SparkDataFrame` used. For example, to select rows with value > 1, we can write `filter(df, df$col > 1)` or `filter(df, "col > 1")`.
+
+Here are more concrete examples.
+
+dplyr | SparkR
+-------- | ---------
+`select(mtcars, mpg, hp)` | `select(carsDF, "mpg", "hp")`
+`filter(mtcars, mpg > 20, hp > 100)` | `filter(carsDF, carsDF$mpg > 20, carsDF$hp > 100)`
+
+Other differences will be mentioned in the specific methods.
+
+We use the `SparkDataFrame` `carsDF` created above. We can get basic information about the `SparkDataFrame`.
+```{r, eval=EVAL_CHUNK}
+carsDF
+```
+
+Print out the schema in tree format.
+```{r, eval=EVAL_CHUNK}
+printSchema(carsDF)
+```
+
+### SparkDataFrame Operations
+
+#### Selecting rows, columns
+
+SparkDataFrames support a number of functions to do structured data processing. Here we include some basic examples and a complete list can be found in the [API](https://spark.apache.org/docs/latest/api/R/index.html) docs:
+
+You can also pass in column name as strings.
+```{r, eval=EVAL_CHUNK}
+head(select(carsDF, "mpg"))
+```
+
+Filter the SparkDataFrame to only retain rows with mpg less than 20 miles/gallon.
+```{r, eval=EVAL_CHUNK}
+head(filter(carsDF, carsDF$mpg < 20))
+```
+
+#### Grouping, Aggregation
+
+A common flow of grouping and aggregation is
+
+1. Use `groupBy` or `group_by` with respect to some grouping variables to create a `GroupedData` object
+
+2. Feed the `GroupedData` object to `agg` or `summarize` functions, with some provided aggregation functions to compute a number within each group.
+
+A number of widely used functions are supported to aggregate data after grouping, including `avg`, `count_distinct`, `count`, `first`, `kurtosis`, `last`, `max`, `mean`, `min`, `sd`, `skewness`, `stddev_pop`, `stddev_samp`, `sum_distinct`, `sum`, `var_pop`, `var_samp`, `var`. See the [API doc for aggregate functions](https://spark.apache.org/docs/latest/api/R/reference/column_aggregate_functions.html) linked there.
+
+For example we can compute a histogram of the number of cylinders in the `mtcars` dataset as shown below.
+
+```{r, eval=EVAL_CHUNK}
+numCyl <- summarize(groupBy(carsDF, carsDF$cyl), count = n(carsDF$cyl))
+head(numCyl)
+```
+
+Use `cube` or `rollup` to compute subtotals across multiple dimensions.
+
+```{r, eval=EVAL_CHUNK}
+mean(cube(carsDF, "cyl", "gear", "am"), "mpg")
+```
+
+generates groupings for {(`cyl`, `gear`, `am`), (`cyl`, `gear`), (`cyl`), ()}, while
+
+```{r, eval=EVAL_CHUNK}
+mean(rollup(carsDF, "cyl", "gear", "am"), "mpg")
+```
+
+generates groupings for all possible combinations of grouping columns.
+
+
+#### Operating on Columns
+
+SparkR also provides a number of functions that can directly applied to columns for data processing and during aggregation. The example below shows the use of basic arithmetic functions.
+
+```{r, eval=EVAL_CHUNK}
+carsDF_km <- carsDF
+carsDF_km$kmpg <- carsDF_km$mpg * 1.61
+head(select(carsDF_km, "model", "mpg", "kmpg"))
+```
+
+
+### Window Functions
+A window function is a variation of aggregation function. In simple words,
+
+* aggregation function: `n` to `1` mapping - returns a single value for a group of entries. Examples include `sum`, `count`, `max`.
+
+* window function: `n` to `n` mapping - returns one value for each entry in the group, but the value may depend on all the entries of the *group*. Examples include `rank`, `lead`, `lag`.
+
+Formally, the *group* mentioned above is called the *frame*. Every input row can have a unique frame associated with it and the output of the window function on that row is based on the rows confined in that frame.
+
+Window functions are often used in conjunction with the following functions: `windowPartitionBy`, `windowOrderBy`, `partitionBy`, `orderBy`, `over`. To illustrate this we next look at an example.
+
+We still use the `mtcars` dataset. The corresponding `SparkDataFrame` is `carsDF`. Suppose for each number of cylinders, we want to calculate the rank of each car in `mpg` within the group.
+```{r, eval=EVAL_CHUNK}
+carsSubDF <- select(carsDF, "model", "mpg", "cyl")
+ws <- orderBy(windowPartitionBy("cyl"), "mpg")
+carsRank <- withColumn(carsSubDF, "rank", over(rank(), ws))
+head(carsRank, n = 20L)
+```
+
+We explain in detail the above steps.
+
+* `windowPartitionBy` creates a window specification object `WindowSpec` that defines the partition. It controls which rows will be in the same partition as the given row. In this case, rows with the same value in `cyl` will be put in the same partition. `orderBy` further defines the ordering - the position a given row is in the partition. The resulting `WindowSpec` is returned as `ws`.
+
+More window specification methods include `rangeBetween`, which can define boundaries of the frame by value, and `rowsBetween`, which can define the boundaries by row indices.
+
+* `withColumn` appends a Column called `rank` to the `SparkDataFrame`. `over` returns a windowing column. The first argument is usually a Column returned by window function(s) such as `rank()`, `lead(carsDF$wt)`. That calculates the corresponding values according to the partitioned-and-ordered table.
+
+### User-Defined Function
+
+In SparkR, we support several kinds of user-defined functions (UDFs).
+
+#### Apply by Partition
+
+`dapply` can apply a function to each partition of a `SparkDataFrame`. The function to be applied to each partition of the `SparkDataFrame` should have only one parameter, a `data.frame` corresponding to a partition, and the output should be a `data.frame` as well. Schema specifies the row format of the resulting a `SparkDataFrame`. It must match to data types of returned value. See [here](#DataTypes) for mapping between R and Spark.
+
+We convert `mpg` to `kmpg` (kilometers per gallon). `carsSubDF` is a `SparkDataFrame` with a subset of `carsDF` columns.
+
+```{r, eval=EVAL_CHUNK}
+carsSubDF <- select(carsDF, "model", "mpg")
+schema <- "model STRING, mpg DOUBLE, kmpg DOUBLE"
+out <- dapply(carsSubDF, function(x) { x <- cbind(x, x$mpg * 1.61) }, schema)
+head(collect(out))
+```
+
+Like `dapply`, `dapplyCollect` can apply a function to each partition of a `SparkDataFrame` and collect the result back. The output of the function should be a `data.frame`, but no schema is required in this case. Note that `dapplyCollect` can fail if the output of the UDF on all partitions cannot be pulled into the driver's memory.
+
+```{r, eval=EVAL_CHUNK}
+out <- dapplyCollect(
+ carsSubDF,
+ function(x) {
+ x <- cbind(x, "kmpg" = x$mpg * 1.61)
+ })
+head(out, 3)
+```
+
+#### Apply by Group
+`gapply` can apply a function to each group of a `SparkDataFrame`. The function is to be applied to each group of the `SparkDataFrame` and should have only two parameters: grouping key and R `data.frame` corresponding to that key. The groups are chosen from `SparkDataFrames` column(s). The output of function should be a `data.frame`. Schema specifies the row format of the resulting `SparkDataFrame`. It must represent R function’s output schema on the basis of Spark data types. The column names of the returned `data.frame` are set by user. See [here](#DataTypes) for mapping between R and Spark.
+
+```{r, eval=EVAL_CHUNK}
+schema <- structType(structField("cyl", "double"), structField("max_mpg", "double"))
+result <- gapply(
+ carsDF,
+ "cyl",
+ function(key, x) {
+ y <- data.frame(key, max(x$mpg))
+ },
+ schema)
+head(arrange(result, "max_mpg", decreasing = TRUE))
+```
+
+Like `gapply`, `gapplyCollect` can apply a function to each partition of a `SparkDataFrame` and collect the result back to R `data.frame`. The output of the function should be a `data.frame` but no schema is required in this case. Note that `gapplyCollect` can fail if the output of the UDF on all partitions cannot be pulled into the driver's memory.
+
+```{r, eval=EVAL_CHUNK}
+result <- gapplyCollect(
+ carsDF,
+ "cyl",
+ function(key, x) {
+ y <- data.frame(key, max(x$mpg))
+ colnames(y) <- c("cyl", "max_mpg")
+ y
+ })
+head(result[order(result$max_mpg, decreasing = TRUE), ])
+```
+
+#### Distribute Local Functions
+
+Similar to `lapply` in native R, `spark.lapply` runs a function over a list of elements and distributes the computations with Spark. `spark.lapply` works in a manner that is similar to `doParallel` or `lapply` to elements of a list. The results of all the computations should fit in a single machine. If that is not the case you can do something like `df <- createDataFrame(list)` and then use `dapply`.
+
+We use `svm` in package `e1071` as an example. We use all default settings except for varying costs of constraints violation. `spark.lapply` can train those different models in parallel.
+
+```{r, eval=EVAL_CHUNK}
+costs <- exp(seq(from = log(1), to = log(1000), length.out = 5))
+train <- function(cost) {
+ stopifnot(requireNamespace("e1071", quietly = TRUE))
+ model <- e1071::svm(Species ~ ., data = iris, cost = cost)
+ summary(model)
+}
+```
+
+Return a list of model's summaries.
+```{r, eval=EVAL_CHUNK}
+model.summaries <- spark.lapply(costs, train)
+```
+
+```{r, eval=EVAL_CHUNK}
+class(model.summaries)
+```
+
+
+To avoid lengthy display, we only present the partial result of the second fitted model. You are free to inspect other models as well.
+```{r, include=FALSE, eval=EVAL_CHUNK}
+ops <- options()
+options(max.print=40)
+```
+```{r, eval=EVAL_CHUNK}
+print(model.summaries[[2]])
+```
+```{r, include=FALSE, eval=EVAL_CHUNK}
+options(ops)
+```
+
+
+### SQL Queries
+A `SparkDataFrame` can also be registered as a temporary view in Spark SQL so that one can run SQL queries over its data. The sql function enables applications to run SQL queries programmatically and returns the result as a `SparkDataFrame`.
+
+```{r, eval=EVAL_CHUNK}
+people <- read.df(paste0(sparkR.conf("spark.home"),
+ "/examples/src/main/resources/people.json"), "json")
+```
+
+Register this `SparkDataFrame` as a temporary view.
+
+```{r, eval=EVAL_CHUNK}
+createOrReplaceTempView(people, "people")
+```
+
+SQL statements can be run using the sql method.
+```{r, eval=EVAL_CHUNK}
+teenagers <- sql("SELECT name FROM people WHERE age >= 13 AND age <= 19")
+head(teenagers)
+```
+
+
+## Machine Learning
+
+SparkR supports the following machine learning models and algorithms.
+
+#### Classification
+
+* Linear Support Vector Machine (SVM) Classifier
+
+* Logistic Regression
+
+* Multilayer Perceptron (MLP)
+
+* Naive Bayes
+
+* Factorization Machines (FM) Classifier
+
+#### Regression
+
+* Accelerated Failure Time (AFT) Survival Model
+
+* Generalized Linear Model (GLM)
+
+* Isotonic Regression
+
+* Linear Regression
+
+* Factorization Machines (FM) Regressor
+
+#### Tree - Classification and Regression
+
+* Decision Tree
+
+* Gradient-Boosted Trees (GBT)
+
+* Random Forest
+
+#### Clustering
+
+* Bisecting $k$-means
+
+* Gaussian Mixture Model (GMM)
+
+* $k$-means Clustering
+
+* Latent Dirichlet Allocation (LDA)
+
+* Power Iteration Clustering (PIC)
+
+#### Collaborative Filtering
+
+* Alternating Least Squares (ALS)
+
+#### Frequent Pattern Mining
+
+* FP-growth
+* PrefixSpan
+
+#### Statistics
+
+* Kolmogorov-Smirnov Test
+
+### R Formula
+
+For most above, SparkR supports **R formula operators**, including `~`, `.`, `:`, `+` and `-` for model fitting. This makes it a similar experience as using R functions.
+
+### Training and Test Sets
+
+We can easily split `SparkDataFrame` into random training and test sets by the `randomSplit` function. It returns a list of split `SparkDataFrames` with provided `weights`. We use `carsDF` as an example and want to have about $70%$ training data and $30%$ test data.
+```{r, eval=EVAL_CHUNK}
+splitDF_list <- randomSplit(carsDF, c(0.7, 0.3), seed = 0)
+carsDF_train <- splitDF_list[[1]]
+carsDF_test <- splitDF_list[[2]]
+```
+
+```{r, eval=EVAL_CHUNK}
+count(carsDF_train)
+head(carsDF_train)
+```
+
+```{r, eval=EVAL_CHUNK}
+count(carsDF_test)
+head(carsDF_test)
+```
+
+### Models and Algorithms
+
+#### Linear Support Vector Machine (SVM) Classifier
+
+[Linear Support Vector Machine (SVM)](https://en.wikipedia.org/wiki/Support_vector_machine#Linear_SVM) classifier is an SVM classifier with linear kernels.
+This is a binary classifier. We use a simple example to show how to use `spark.svmLinear`
+for binary classification.
+
+```{r, eval=EVAL_CHUNK}
+# load training data and create a DataFrame
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# fit a Linear SVM classifier model
+model <- spark.svmLinear(training, Survived ~ ., regParam = 0.01, maxIter = 10)
+summary(model)
+```
+
+Predict values on training data
+```{r, eval=EVAL_CHUNK}
+prediction <- predict(model, training)
+head(select(prediction, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+```
+
+#### Logistic Regression
+
+[Logistic regression](https://en.wikipedia.org/wiki/Logistic_regression) is a widely-used model when the response is categorical. It can be seen as a special case of the [Generalized Linear Predictive Model](https://en.wikipedia.org/wiki/Generalized_linear_model).
+We provide `spark.logit` on top of `spark.glm` to support logistic regression with advanced hyper-parameters.
+It supports both binary and multiclass classification with elastic-net regularization and feature standardization, similar to `glmnet`.
+
+We use a simple example to demonstrate `spark.logit` usage. In general, there are three steps of using `spark.logit`:
+1). Create a dataframe from a proper data source; 2). Fit a logistic regression model using `spark.logit` with a proper parameter setting;
+and 3). Obtain the coefficient matrix of the fitted model using `summary` and use the model for prediction with `predict`.
+
+Binomial logistic regression
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+model <- spark.logit(training, Survived ~ ., regParam = 0.04741301)
+summary(model)
+```
+
+Predict values on training data
+```{r, eval=EVAL_CHUNK}
+fitted <- predict(model, training)
+head(select(fitted, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+```
+
+Multinomial logistic regression against three classes
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# Note in this case, Spark infers it is multinomial logistic regression, so family = "multinomial" is optional.
+model <- spark.logit(training, Class ~ ., regParam = 0.07815179)
+summary(model)
+```
+
+#### Multilayer Perceptron
+
+Multilayer perceptron classifier (MLPC) is a classifier based on the [feedforward artificial neural network](https://en.wikipedia.org/wiki/Feedforward_neural_network). MLPC consists of multiple layers of nodes. Each layer is fully connected to the next layer in the network. Nodes in the input layer represent the input data. All other nodes map inputs to outputs by a linear combination of the inputs with the node’s weights $w$ and bias $b$ and applying an activation function. This can be written in matrix form for MLPC with $K+1$ layers as follows:
+$$
+y(x)=f_K(\ldots f_2(w_2^T f_1(w_1^T x + b_1) + b_2) \ldots + b_K).
+$$
+
+Nodes in intermediate layers use sigmoid (logistic) function:
+$$
+f(z_i) = \frac{1}{1+e^{-z_i}}.
+$$
+
+Nodes in the output layer use softmax function:
+$$
+f(z_i) = \frac{e^{z_i}}{\sum_{k=1}^N e^{z_k}}.
+$$
+
+The number of nodes $N$ in the output layer corresponds to the number of classes.
+
+MLPC employs backpropagation for learning the model. We use the logistic loss function for optimization and L-BFGS as an optimization routine.
+
+`spark.mlp` requires at least two columns in `data`: one named `"label"` and the other one `"features"`. The `"features"` column should be in libSVM-format.
+
+We use Titanic data set to show how to use `spark.mlp` in classification.
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# fit a Multilayer Perceptron Classification Model
+model <- spark.mlp(training, Survived ~ Age + Sex, blockSize = 128, layers = c(2, 2), solver = "l-bfgs", maxIter = 100, tol = 0.5, stepSize = 1, seed = 1, initialWeights = c( 0, 0, 5, 5, 9, 9))
+```
+
+To avoid lengthy display, we only present partial results of the model summary. You can check the full result from your sparkR shell.
+```{r, include=FALSE, eval=EVAL_CHUNK}
+ops <- options()
+options(max.print=5)
+```
+```{r, eval=EVAL_CHUNK}
+# check the summary of the fitted model
+summary(model)
+```
+```{r, include=FALSE, eval=EVAL_CHUNK}
+options(ops)
+```
+```{r, eval=EVAL_CHUNK}
+# make predictions use the fitted model
+predictions <- predict(model, training)
+head(select(predictions, predictions$prediction))
+```
+
+#### Naive Bayes
+
+Naive Bayes model assumes independence among the features. `spark.naiveBayes` fits a [Bernoulli naive Bayes model](https://en.wikipedia.org/wiki/Naive_Bayes_classifier#Bernoulli_naive_Bayes) against a SparkDataFrame. The data should be all categorical. These models are often used for document classification.
+
+```{r, eval=EVAL_CHUNK}
+titanic <- as.data.frame(Titanic)
+titanicDF <- createDataFrame(titanic[titanic$Freq > 0, -5])
+naiveBayesModel <- spark.naiveBayes(titanicDF, Survived ~ Class + Sex + Age)
+summary(naiveBayesModel)
+naiveBayesPrediction <- predict(naiveBayesModel, titanicDF)
+head(select(naiveBayesPrediction, "Class", "Sex", "Age", "Survived", "prediction"))
+```
+
+#### Factorization Machines Classifier
+
+Factorization Machines for classification problems.
+
+For background and details about the implementation of factorization machines,
+refer to the [Factorization Machines section](https://spark.apache.org/docs/latest/ml-classification-regression.html#factorization-machines).
+
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+
+model <- spark.fmClassifier(training, Survived ~ Age + Sex)
+summary(model)
+
+predictions <- predict(model, training)
+head(select(predictions, predictions$prediction))
+```
+
+#### Accelerated Failure Time Survival Model
+
+Survival analysis studies the expected duration of time until an event happens, and often the relationship with risk factors or treatment taken on the subject. In contrast to standard regression analysis, survival modeling has to deal with special characteristics in the data including non-negative survival time and censoring.
+
+Accelerated Failure Time (AFT) model is a parametric survival model for censored data that assumes the effect of a covariate is to accelerate or decelerate the life course of an event by some constant. For more information, refer to the Wikipedia page [AFT Model](https://en.wikipedia.org/wiki/Accelerated_failure_time_model) and the references there. Different from a [Proportional Hazards Model](https://en.wikipedia.org/wiki/Proportional_hazards_model) designed for the same purpose, the AFT model is easier to parallelize because each instance contributes to the objective function independently.
+
+```{r, warning=FALSE, eval=EVAL_CHUNK}
+library(survival)
+ovarianDF <- createDataFrame(ovarian)
+aftModel <- spark.survreg(ovarianDF, Surv(futime, fustat) ~ ecog_ps + rx)
+summary(aftModel)
+aftPredictions <- predict(aftModel, ovarianDF)
+head(aftPredictions)
+```
+
+#### Generalized Linear Model
+
+The main function is `spark.glm`. The following families and link functions are supported. The default is gaussian.
+
+Family | Link Function
+------ | ---------
+gaussian | identity, log, inverse
+binomial | logit, probit, cloglog (complementary log-log)
+poisson | log, identity, sqrt
+gamma | inverse, identity, log
+tweedie | power link function
+
+There are three ways to specify the `family` argument.
+
+* Family name as a character string, e.g. `family = "gaussian"`.
+
+* Family function, e.g. `family = binomial`.
+
+* Result returned by a family function, e.g. `family = poisson(link = log)`.
+
+* Note that there are two ways to specify the tweedie family:
+ a) Set `family = "tweedie"` and specify the `var.power` and `link.power`
+ b) When package `statmod` is loaded, the tweedie family is specified using the family definition therein, i.e., `tweedie()`.
+
+For more information regarding the families and their link functions, see the Wikipedia page [Generalized Linear Model](https://en.wikipedia.org/wiki/Generalized_linear_model).
+
+We use the `mtcars` dataset as an illustration. The corresponding `SparkDataFrame` is `carsDF`. After fitting the model, we print out a summary and see the fitted values by making predictions on the original dataset. We can also pass into a new `SparkDataFrame` of same schema to predict on new data.
+
+```{r, eval=EVAL_CHUNK}
+gaussianGLM <- spark.glm(carsDF, mpg ~ wt + hp)
+summary(gaussianGLM)
+```
+When doing prediction, a new column called `prediction` will be appended. Let's look at only a subset of columns here.
+```{r, eval=EVAL_CHUNK}
+gaussianFitted <- predict(gaussianGLM, carsDF)
+head(select(gaussianFitted, "model", "prediction", "mpg", "wt", "hp"))
+```
+
+The following is the same fit using the tweedie family:
+```{r, eval=EVAL_CHUNK}
+tweedieGLM1 <- spark.glm(carsDF, mpg ~ wt + hp, family = "tweedie", var.power = 0.0)
+summary(tweedieGLM1)
+```
+We can try other distributions in the tweedie family, for example, a compound Poisson distribution with a log link:
+```{r, eval=EVAL_CHUNK}
+tweedieGLM2 <- spark.glm(carsDF, mpg ~ wt + hp, family = "tweedie",
+ var.power = 1.2, link.power = 0.0)
+summary(tweedieGLM2)
+```
+
+#### Isotonic Regression
+
+`spark.isoreg` fits an [Isotonic Regression](https://en.wikipedia.org/wiki/Isotonic_regression) model against a `SparkDataFrame`. It solves a weighted univariate a regression problem under a complete order constraint. Specifically, given a set of real observed responses $y_1, \ldots, y_n$, corresponding real features $x_1, \ldots, x_n$, and optionally positive weights $w_1, \ldots, w_n$, we want to find a monotone (piecewise linear) function $f$ to minimize
+$$
+\ell(f) = \sum_{i=1}^n w_i (y_i - f(x_i))^2.
+$$
+
+There are a few more arguments that may be useful.
+
+* `weightCol`: a character string specifying the weight column.
+
+* `isotonic`: logical value indicating whether the output sequence should be isotonic/increasing (`TRUE`) or antitonic/decreasing (`FALSE`).
+
+* `featureIndex`: the index of the feature on the right hand side of the formula if it is a vector column (default: 0), no effect otherwise.
+
+We use an artificial example to show the use.
+
+```{r, eval=EVAL_CHUNK}
+y <- c(3.0, 6.0, 8.0, 5.0, 7.0)
+x <- c(1.0, 2.0, 3.5, 3.0, 4.0)
+w <- rep(1.0, 5)
+data <- data.frame(y = y, x = x, w = w)
+df <- createDataFrame(data)
+isoregModel <- spark.isoreg(df, y ~ x, weightCol = "w")
+isoregFitted <- predict(isoregModel, df)
+head(select(isoregFitted, "x", "y", "prediction"))
+```
+
+In the prediction stage, based on the fitted monotone piecewise function, the rules are:
+
+* If the prediction input exactly matches a training feature then associated prediction is returned. In case there are multiple predictions with the same feature then one of them is returned. Which one is undefined.
+
+* If the prediction input is lower or higher than all training features then prediction with lowest or highest feature is returned respectively. In case there are multiple predictions with the same feature then the lowest or highest is returned respectively.
+
+* If the prediction input falls between two training features then prediction is treated as piecewise linear function and interpolated value is calculated from the predictions of the two closest features. In case there are multiple values with the same feature then the same rules as in previous point are used.
+
+For example, when the input is $3.2$, the two closest feature values are $3.0$ and $3.5$, then predicted value would be a linear interpolation between the predicted values at $3.0$ and $3.5$.
+
+```{r, eval=EVAL_CHUNK}
+newDF <- createDataFrame(data.frame(x = c(1.5, 3.2)))
+head(predict(isoregModel, newDF))
+```
+
+#### Linear Regression
+
+Linear regression model.
+
+```{r, eval=EVAL_CHUNK}
+model <- spark.lm(carsDF, mpg ~ wt + hp)
+
+summary(model)
+predictions <- predict(model, carsDF)
+head(select(predictions, predictions$prediction))
+```
+
+#### Factorization Machines Regressor
+
+Factorization Machines for regression problems.
+
+For background and details about the implementation of factorization machines,
+refer to the [Factorization Machines section](https://spark.apache.org/docs/latest/ml-classification-regression.html#factorization-machines).
+
+```{r, eval=EVAL_CHUNK}
+model <- spark.fmRegressor(carsDF, mpg ~ wt + hp)
+summary(model)
+predictions <- predict(model, carsDF)
+head(select(predictions, predictions$prediction))
+```
+
+#### Decision Tree
+
+`spark.decisionTree` fits a [decision tree](https://en.wikipedia.org/wiki/Decision_tree_learning) classification or regression model on a `SparkDataFrame`.
+Users can call `summary` to get a summary of the fitted model, `predict` to make predictions, and `write.ml`/`read.ml` to save/load fitted models.
+
+We use the `Titanic` dataset to train a decision tree and make predictions:
+
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+dtModel <- spark.decisionTree(df, Survived ~ ., type = "classification", maxDepth = 2)
+summary(dtModel)
+predictions <- predict(dtModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+```
+
+#### Gradient-Boosted Trees
+
+`spark.gbt` fits a [gradient-boosted tree](https://en.wikipedia.org/wiki/Gradient_boosting) classification or regression model on a `SparkDataFrame`.
+Users can call `summary` to get a summary of the fitted model, `predict` to make predictions, and `write.ml`/`read.ml` to save/load fitted models.
+
+We use the `Titanic` dataset to train a gradient-boosted tree and make predictions:
+
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+gbtModel <- spark.gbt(df, Survived ~ ., type = "classification", maxDepth = 2, maxIter = 2)
+summary(gbtModel)
+predictions <- predict(gbtModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+```
+
+#### Random Forest
+
+`spark.randomForest` fits a [random forest](https://en.wikipedia.org/wiki/Random_forest) classification or regression model on a `SparkDataFrame`.
+Users can call `summary` to get a summary of the fitted model, `predict` to make predictions, and `write.ml`/`read.ml` to save/load fitted models.
+
+In the following example, we use the `Titanic` dataset to train a random forest and make predictions:
+
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+rfModel <- spark.randomForest(df, Survived ~ ., type = "classification", maxDepth = 2, numTrees = 2)
+summary(rfModel)
+predictions <- predict(rfModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))
+```
+
+#### Bisecting k-Means
+
+`spark.bisectingKmeans` is a kind of [hierarchical clustering](https://en.wikipedia.org/wiki/Hierarchical_clustering) using a divisive (or "top-down") approach: all observations start in one cluster, and splits are performed recursively as one moves down the hierarchy.
+
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+model <- spark.bisectingKmeans(training, Class ~ Survived, k = 4)
+summary(model)
+fitted <- predict(model, training)
+head(select(fitted, "Class", "prediction"))
+```
+
+#### Gaussian Mixture Model
+
+`spark.gaussianMixture` fits multivariate [Gaussian Mixture Model](https://en.wikipedia.org/wiki/Mixture_model#Multivariate_Gaussian_mixture_model) (GMM) against a `SparkDataFrame`. [Expectation-Maximization](https://en.wikipedia.org/wiki/Expectation%E2%80%93maximization_algorithm) (EM) is used to approximate the maximum likelihood estimator (MLE) of the model.
+
+We use a simulated example to demonstrate the usage.
+```{r, eval=EVAL_CHUNK}
+X1 <- data.frame(V1 = rnorm(4), V2 = rnorm(4))
+X2 <- data.frame(V1 = rnorm(6, 3), V2 = rnorm(6, 4))
+data <- rbind(X1, X2)
+df <- createDataFrame(data)
+gmmModel <- spark.gaussianMixture(df, ~ V1 + V2, k = 2)
+summary(gmmModel)
+gmmFitted <- predict(gmmModel, df)
+head(select(gmmFitted, "V1", "V2", "prediction"))
+```
+
+#### k-Means Clustering
+
+`spark.kmeans` fits a $k$-means clustering model against a `SparkDataFrame`. As an unsupervised learning method, we don't need a response variable. Hence, the left hand side of the R formula should be left blank. The clustering is based only on the variables on the right hand side.
+
+```{r, eval=EVAL_CHUNK}
+kmeansModel <- spark.kmeans(carsDF, ~ mpg + hp + wt, k = 3)
+summary(kmeansModel)
+kmeansPredictions <- predict(kmeansModel, carsDF)
+head(select(kmeansPredictions, "model", "mpg", "hp", "wt", "prediction"), n = 20L)
+```
+
+#### Latent Dirichlet Allocation
+
+`spark.lda` fits a [Latent Dirichlet Allocation](https://en.wikipedia.org/wiki/Latent_Dirichlet_allocation) model on a `SparkDataFrame`. It is often used in topic modeling in which topics are inferred from a collection of text documents. LDA can be thought of as a clustering algorithm as follows:
+
+* Topics correspond to cluster centers, and documents correspond to examples (rows) in a dataset.
+
+* Topics and documents both exist in a feature space, where feature vectors are vectors of word counts (bag of words).
+
+* Rather than clustering using a traditional distance, LDA uses a function based on a statistical model of how text documents are generated.
+
+To use LDA, we need to specify a `features` column in `data` where each entry represents a document. There are two options for the column:
+
+* character string: This can be a string of the whole document. It will be parsed automatically. Additional stop words can be added in `customizedStopWords`.
+
+* libSVM: Each entry is a collection of words and will be processed directly.
+
+Two more functions are provided for the fitted model.
+
+* `spark.posterior` returns a `SparkDataFrame` containing a column of posterior probabilities vectors named "topicDistribution".
+
+* `spark.perplexity` returns the log perplexity of given `SparkDataFrame`, or the log perplexity of the training data if missing argument `data`.
+
+For more information, see the help document `?spark.lda`.
+
+Let's look an artificial example.
+```{r, eval=EVAL_CHUNK}
+corpus <- data.frame(features = c(
+ "1 2 6 0 2 3 1 1 0 0 3",
+ "1 3 0 1 3 0 0 2 0 0 1",
+ "1 4 1 0 0 4 9 0 1 2 0",
+ "2 1 0 3 0 0 5 0 2 3 9",
+ "3 1 1 9 3 0 2 0 0 1 3",
+ "4 2 0 3 4 5 1 1 1 4 0",
+ "2 1 0 3 0 0 5 0 2 2 9",
+ "1 1 1 9 2 1 2 0 0 1 3",
+ "4 4 0 3 4 2 1 3 0 0 0",
+ "2 8 2 0 3 0 2 0 2 7 2",
+ "1 1 1 9 0 2 2 0 0 3 3",
+ "4 1 0 0 4 5 1 3 0 1 0"))
+corpusDF <- createDataFrame(corpus)
+model <- spark.lda(data = corpusDF, k = 5, optimizer = "em")
+summary(model)
+```
+
+```{r, eval=EVAL_CHUNK}
+posterior <- spark.posterior(model, corpusDF)
+head(posterior)
+```
+
+```{r, eval=EVAL_CHUNK}
+perplexity <- spark.perplexity(model, corpusDF)
+perplexity
+```
+
+#### Alternating Least Squares
+
+`spark.als` learns latent factors in [collaborative filtering](https://en.wikipedia.org/wiki/Recommender_system#Collaborative_filtering) via [alternating least squares](https://dl.acm.org/doi/10.1109/MC.2009.263).
+
+There are multiple options that can be configured in `spark.als`, including `rank`, `reg`, and `nonnegative`. For a complete list, refer to the help file.
+
+```{r, eval=FALSE}
+ratings <- list(list(0, 0, 4.0), list(0, 1, 2.0), list(1, 1, 3.0), list(1, 2, 4.0),
+ list(2, 1, 1.0), list(2, 2, 5.0))
+df <- createDataFrame(ratings, c("user", "item", "rating"))
+model <- spark.als(df, "rating", "user", "item", rank = 10, reg = 0.1, nonnegative = TRUE)
+```
+
+Extract latent factors.
+```{r, eval=FALSE}
+stats <- summary(model)
+userFactors <- stats$userFactors
+itemFactors <- stats$itemFactors
+head(userFactors)
+head(itemFactors)
+```
+
+Make predictions.
+
+```{r, eval=FALSE}
+predicted <- predict(model, df)
+head(predicted)
+```
+
+#### Power Iteration Clustering
+
+Power Iteration Clustering (PIC) is a scalable graph clustering algorithm. `spark.assignClusters` method runs the PIC algorithm and returns a cluster assignment for each input vertex.
+
+```{r, eval=EVAL_CHUNK}
+df <- createDataFrame(list(list(0L, 1L, 1.0), list(0L, 2L, 1.0),
+ list(1L, 2L, 1.0), list(3L, 4L, 1.0),
+ list(4L, 0L, 0.1)),
+ schema = c("src", "dst", "weight"))
+head(spark.assignClusters(df, initMode = "degree", weightCol = "weight"))
+```
+
+#### FP-growth
+
+`spark.fpGrowth` executes FP-growth algorithm to mine frequent itemsets on a `SparkDataFrame`. `itemsCol` should be an array of values.
+
+```{r, eval=EVAL_CHUNK}
+df <- selectExpr(createDataFrame(data.frame(rawItems = c(
+ "T,R,U", "T,S", "V,R", "R,U,T,V", "R,S", "V,S,U", "U,R", "S,T", "V,R", "V,U,S",
+ "T,V,U", "R,V", "T,S", "T,S", "S,T", "S,U", "T,R", "V,R", "S,V", "T,S,U"
+))), "split(rawItems, ',') AS items")
+
+fpm <- spark.fpGrowth(df, minSupport = 0.2, minConfidence = 0.5)
+```
+
+`spark.freqItemsets` method can be used to retrieve a `SparkDataFrame` with the frequent itemsets.
+
+```{r, eval=EVAL_CHUNK}
+head(spark.freqItemsets(fpm))
+```
+
+`spark.associationRules` returns a `SparkDataFrame` with the association rules.
+
+```{r, eval=EVAL_CHUNK}
+head(spark.associationRules(fpm))
+```
+
+We can make predictions based on the `antecedent`.
+
+```{r, eval=EVAL_CHUNK}
+head(predict(fpm, df))
+```
+
+#### PrefixSpan
+
+`spark.findFrequentSequentialPatterns` method can be used to find the complete set of frequent sequential patterns in the input sequences of itemsets.
+
+```{r, eval=EVAL_CHUNK}
+df <- createDataFrame(list(list(list(list(1L, 2L), list(3L))),
+ list(list(list(1L), list(3L, 2L), list(1L, 2L))),
+ list(list(list(1L, 2L), list(5L))),
+ list(list(list(6L)))),
+ schema = c("sequence"))
+head(spark.findFrequentSequentialPatterns(df, minSupport = 0.5, maxPatternLength = 5L))
+```
+
+#### Kolmogorov-Smirnov Test
+
+`spark.kstest` runs a two-sided, one-sample [Kolmogorov-Smirnov (KS) test](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test).
+Given a `SparkDataFrame`, the test compares continuous data in a given column `testCol` with the theoretical distribution
+specified by parameter `nullHypothesis`.
+Users can call `summary` to get a summary of the test results.
+
+In the following example, we test whether the `Titanic` dataset's `Freq` column
+follows a normal distribution. We set the parameters of the normal distribution using
+the mean and standard deviation of the sample.
+
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+freqStats <- head(select(df, mean(df$Freq), sd(df$Freq)))
+freqMean <- freqStats[1]
+freqStd <- freqStats[2]
+
+test <- spark.kstest(df, "Freq", "norm", c(freqMean, freqStd))
+testSummary <- summary(test)
+testSummary
+```
+
+
+### Model Persistence
+The following example shows how to save/load an ML model in SparkR.
+```{r, eval=EVAL_CHUNK}
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+gaussianGLM <- spark.glm(training, Freq ~ Sex + Age, family = "gaussian")
+
+# Save and then load a fitted MLlib model
+modelPath <- tempfile(pattern = "ml", fileext = ".tmp")
+write.ml(gaussianGLM, modelPath)
+gaussianGLM2 <- read.ml(modelPath)
+
+# Check model summary
+summary(gaussianGLM2)
+
+# Check model prediction
+gaussianPredictions <- predict(gaussianGLM2, training)
+head(gaussianPredictions)
+
+unlink(modelPath)
+```
+
+
+## Structured Streaming
+
+SparkR supports the Structured Streaming API.
+
+You can check the Structured Streaming Programming Guide for [an introduction](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#programming-model) to its programming model and basic concepts.
+
+### Simple Source and Sink
+
+Spark has a few built-in input sources. As an example, to test with a socket source reading text into words and displaying the computed word counts:
+
+```{r, eval=FALSE}
+# Create DataFrame representing the stream of input lines from connection
+lines <- read.stream("socket", host = hostname, port = port)
+
+# Split the lines into words
+words <- selectExpr(lines, "explode(split(value, ' ')) as word")
+
+# Generate running word count
+wordCounts <- count(groupBy(words, "word"))
+
+# Start running the query that prints the running counts to the console
+query <- write.stream(wordCounts, "console", outputMode = "complete")
+```
+
+### Kafka Source
+
+It is simple to read data from Kafka. For more information, see [Input Sources](https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#input-sources) supported by Structured Streaming.
+
+```{r, eval=FALSE}
+topic <- read.stream("kafka",
+ kafka.bootstrap.servers = "host1:port1,host2:port2",
+ subscribe = "topic1")
+keyvalue <- selectExpr(topic, "CAST(key AS STRING)", "CAST(value AS STRING)")
+```
+
+### Operations and Sinks
+
+Most of the common operations on `SparkDataFrame` are supported for streaming, including selection, projection, and aggregation. Once you have defined the final result, to start the streaming computation, you will call the `write.stream` method setting a sink and `outputMode`.
+
+A streaming `SparkDataFrame` can be written for debugging to the console, to a temporary in-memory table, or for further processing in a fault-tolerant manner to a File Sink in different formats.
+
+```{r, eval=FALSE}
+noAggDF <- select(where(deviceDataStreamingDf, "signal > 10"), "device")
+
+# Print new data to console
+write.stream(noAggDF, "console")
+
+# Write new data to Parquet files
+write.stream(noAggDF,
+ "parquet",
+ path = "path/to/destination/dir",
+ checkpointLocation = "path/to/checkpoint/dir")
+
+# Aggregate
+aggDF <- count(groupBy(noAggDF, "device"))
+
+# Print updated aggregations to console
+write.stream(aggDF, "console", outputMode = "complete")
+
+# Have all the aggregates in an in memory table. The query name will be the table name
+write.stream(aggDF, "memory", queryName = "aggregates", outputMode = "complete")
+
+head(sql("select * from aggregates"))
+```
+
+
+## Advanced Topics
+
+### SparkR Object Classes
+
+There are three main object classes in SparkR you may be working with.
+
+* `SparkDataFrame`: the central component of SparkR. It is an S4 class representing distributed collection of data organized into named columns, which is conceptually equivalent to a table in a relational database or a data frame in R. It has two slots `sdf` and `env`.
+ + `sdf` stores a reference to the corresponding Spark Dataset in the Spark JVM backend.
+ + `env` saves the meta-information of the object such as `isCached`.
+
+ It can be created by data import methods or by transforming an existing `SparkDataFrame`. We can manipulate `SparkDataFrame` by numerous data processing functions and feed that into machine learning algorithms.
+
+* `Column`: an S4 class representing a column of `SparkDataFrame`. The slot `jc` saves a reference to the corresponding `Column` object in the Spark JVM backend.
+
+ It can be obtained from a `SparkDataFrame` by `$` operator, e.g., `df$col`. More often, it is used together with other functions, for example, with `select` to select particular columns, with `filter` and constructed conditions to select rows, with aggregation functions to compute aggregate statistics for each group.
+
+* `GroupedData`: an S4 class representing grouped data created by `groupBy` or by transforming other `GroupedData`. Its `sgd` slot saves a reference to a `RelationalGroupedDataset` object in the backend.
+
+ This is often an intermediate object with group information and followed up by aggregation operations.
+
+### Architecture
+
+A complete description of architecture can be seen in the references, in particular the paper *SparkR: Scaling R Programs with Spark*.
+
+Under the hood of SparkR is Spark SQL engine. This avoids the overheads of running interpreted R code, and the optimized SQL execution engine in Spark uses structural information about data and computation flow to perform a bunch of optimizations to speed up the computation.
+
+The main method calls of actual computation happen in the Spark JVM of the driver. We have a socket-based SparkR API that allows us to invoke functions on the JVM from R. We use a SparkR JVM backend that listens on a Netty-based socket server.
+
+Two kinds of RPCs are supported in the SparkR JVM backend: method invocation and creating new objects. Method invocation can be done in two ways.
+
+* `sparkR.callJMethod` takes a reference to an existing Java object and a list of arguments to be passed on to the method.
+
+* `sparkR.callJStatic` takes a class name for static method and a list of arguments to be passed on to the method.
+
+The arguments are serialized using our custom wire format which is then deserialized on the JVM side. We then use Java reflection to invoke the appropriate method.
+
+To create objects, `sparkR.newJObject` is used and then similarly the appropriate constructor is invoked with provided arguments.
+
+Finally, we use a new R class `jobj` that refers to a Java object existing in the backend. These references are tracked on the Java side and are automatically garbage collected when they go out of scope on the R side.
+
+## Appendix
+
+### R and Spark Data Types {#DataTypes}
+
+R | Spark
+----------- | -------------
+byte | byte
+integer | integer
+float | float
+double | double
+numeric | double
+character | string
+string | string
+binary | binary
+raw | binary
+logical | boolean
+POSIXct | timestamp
+POSIXlt | timestamp
+Date | date
+array | array
+list | array
+env | map
+
+## References
+
+* [Spark Cluster Mode Overview](https://spark.apache.org/docs/latest/cluster-overview.html)
+
+* [Submitting Spark Applications](https://spark.apache.org/docs/latest/submitting-applications.html)
+
+* [Machine Learning Library Guide (MLlib)](https://spark.apache.org/docs/latest/ml-guide.html)
+
+* [SparkR: Scaling R Programs with Spark](https://people.csail.mit.edu/matei/papers/2016/sigmod_sparkr.pdf), Shivaram Venkataraman, Zongheng Yang, Davies Liu, Eric Liang, Hossein Falaki, Xiangrui Meng, Reynold Xin, Ali Ghodsi, Michael Franklin, Ion Stoica, and Matei Zaharia. SIGMOD 2016. June 2016.
+
+```{r, echo=FALSE, eval=EVAL_CHUNK}
+sparkR.session.stop()
+```
+
+```{r cleanup, include=FALSE, eval=EVAL_CHUNK}
+SparkR:::uninstallDownloadedSpark()
+```
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.html b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.html
new file mode 100644
index 0000000..2e842ef
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.html
@@ -0,0 +1,2309 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Vignettes and other documentation
+![[R logo]](/doc/html/Rlogo.svg)
+
++
+![[Top]](/doc/html/up.jpg)
+
+
+Vignettes from package 'SparkR'
+| SparkR::sparkr-vignettes | +SparkR - Practical Guide | +HTML | +source | +R code |
SparkR - Practical Guide
+ + +
+
+
+
+-
+
- Overview +
- Getting Started +
- Setup +
- Data Import +
- Data Processing +
- Machine Learning
-
+
- Classification +
- Regression +
- Tree - Classification and Regression +
- Clustering +
- Collaborative Filtering +
- Frequent Pattern Mining +
- Statistics +
- R Formula +
- Training and Test Sets +
- Models and Algorithms
-
+
- Linear Support Vector Machine (SVM) Classifier +
- Logistic Regression +
- Multilayer Perceptron +
- Naive Bayes +
- Factorization Machines Classifier +
- Accelerated Failure Time Survival Model +
- Generalized Linear Model +
- Isotonic Regression +
- Linear Regression +
- Factorization Machines Regressor +
- Decision Tree +
- Gradient-Boosted Trees +
- Random Forest +
- Bisecting k-Means +
- Gaussian Mixture Model +
- k-Means Clustering +
- Latent Dirichlet Allocation +
- Alternating Least Squares +
- Power Iteration Clustering +
- FP-growth +
- PrefixSpan +
- Kolmogorov-Smirnov Test +
+ - Model Persistence +
+ - Structured Streaming +
- Advanced Topics +
- Appendix +
- References +
+
+Overview
+SparkR is an R package that provides a light-weight frontend to use Apache Spark from R. With Spark 3.5.0, SparkR provides a distributed data frame implementation that supports data processing operations like selection, filtering, aggregation etc. and distributed machine learning using MLlib.
+
+
+
+Getting Started
+We begin with an example running on the local machine and provide an overview of the use of SparkR: data ingestion, data processing and machine learning.
+First, let’s load and attach the package.
+ +SparkSession is the entry point into SparkR which connects your R program to a Spark cluster. You can create a SparkSession using sparkR.session and pass in options such as the application name, any Spark packages depended on, etc.
We use default settings in which it runs in local mode. It auto downloads Spark package in the background if no previous installation is found. For more details about setup, see Spark Session.
+ +## Java ref type org.apache.spark.sql.SparkSession id 1The operations in SparkR are centered around an R class called SparkDataFrame. It is a distributed collection of data organized into named columns, which is conceptually equivalent to a table in a relational database or a data frame in R, but with richer optimizations under the hood.
SparkDataFrame can be constructed from a wide array of sources such as: structured data files, tables in Hive, external databases, or existing local R data frames. For example, we create a SparkDataFrame from a local R data frame,
We can view the first few rows of the SparkDataFrame by head or showDF function.
## model mpg cyl disp hp drat wt qsec vs am gear carb
+## 1 Mazda RX4 21.0 6 160 110 3.90 2.620 16.46 0 1 4 4
+## 2 Mazda RX4 Wag 21.0 6 160 110 3.90 2.875 17.02 0 1 4 4
+## 3 Datsun 710 22.8 4 108 93 3.85 2.320 18.61 1 1 4 1
+## 4 Hornet 4 Drive 21.4 6 258 110 3.08 3.215 19.44 1 0 3 1
+## 5 Hornet Sportabout 18.7 8 360 175 3.15 3.440 17.02 0 0 3 2
+## 6 Valiant 18.1 6 225 105 2.76 3.460 20.22 1 0 3 1Common data processing operations such as filter and select are supported on the SparkDataFrame.
carsSubDF <- select(carsDF, "model", "mpg", "hp")
+carsSubDF <- filter(carsSubDF, carsSubDF$hp >= 200)
+head(carsSubDF)## model mpg hp
+## 1 Duster 360 14.3 245
+## 2 Cadillac Fleetwood 10.4 205
+## 3 Lincoln Continental 10.4 215
+## 4 Chrysler Imperial 14.7 230
+## 5 Camaro Z28 13.3 245
+## 6 Ford Pantera L 15.8 264SparkR can use many common aggregation functions after grouping.
+ +## gear count
+## 1 4 12
+## 2 3 15
+## 3 5 5The results carsDF and carsSubDF are SparkDataFrame objects. To convert back to R data.frame, we can use collect. Caution: This can cause your interactive environment to run out of memory, though, because collect() fetches the entire distributed DataFrame to your client, which is acting as a Spark driver.
## [1] "data.frame"SparkR supports a number of commonly used machine learning algorithms. Under the hood, SparkR uses MLlib to train the model. Users can call summary to print a summary of the fitted model, predict to make predictions on new data, and write.ml/read.ml to save/load fitted models.
SparkR supports a subset of R formula operators for model fitting, including ‘~’, ‘.’, ‘:’, ‘+’, and ‘-‘. We use linear regression as an example.
+ +The result matches that returned by R glm function applied to the corresponding data.frame mtcars of carsDF. In fact, for Generalized Linear Model, we specifically expose glm for SparkDataFrame as well so that the above is equivalent to model <- glm(mpg ~ wt + cyl, data = carsDF).
##
+## Deviance Residuals:
+## (Note: These are approximate quantiles with relative error <= 0.01)
+## Min 1Q Median 3Q Max
+## -4.2893 -1.7085 -0.4713 1.5729 6.1004
+##
+## Coefficients:
+## Estimate Std. Error t value Pr(>|t|)
+## (Intercept) 39.6863 1.71498 23.1409 0.00000000
+## wt -3.1910 0.75691 -4.2158 0.00022202
+## cyl -1.5078 0.41469 -3.6360 0.00106428
+##
+## (Dispersion parameter for gaussian family taken to be 6.592137)
+##
+## Null deviance: 1126.05 on 31 degrees of freedom
+## Residual deviance: 191.17 on 29 degrees of freedom
+## AIC: 156
+##
+## Number of Fisher Scoring iterations: 1The model can be saved by write.ml and loaded back using read.ml.
In the end, we can stop Spark Session by running
+ +
+
+Setup
+
+
+Installation
+Different from many other R packages, to use SparkR, you need an additional installation of Apache Spark. The Spark installation will be used to run a backend process that will compile and execute SparkR programs.
+After installing the SparkR package, you can call sparkR.session as explained in the previous section to start and it will check for the Spark installation. If you are working with SparkR from an interactive shell (e.g. R, RStudio) then Spark is downloaded and cached automatically if it is not found. Alternatively, we provide an easy-to-use function install.spark for running this manually. If you don’t have Spark installed on the computer, you may download it from Apache Spark Website.
If you already have Spark installed, you don’t have to install again and can pass the sparkHome argument to sparkR.session to let SparkR know where the existing Spark installation is.
+
+
+
+
+
+Spark Session
+In addition to sparkHome, many other options can be specified in sparkR.session. For a complete list, see Starting up: SparkSession and SparkR API doc.
In particular, the following Spark driver properties can be set in sparkConfig.
| Property Name | +Property group | +spark-submit equivalent | +
|---|---|---|
spark.driver.memory |
+Application Properties | +--driver-memory |
+
spark.driver.extraClassPath |
+Runtime Environment | +--driver-class-path |
+
spark.driver.extraJavaOptions |
+Runtime Environment | +--driver-java-options |
+
spark.driver.extraLibraryPath |
+Runtime Environment | +--driver-library-path |
+
spark.kerberos.keytab |
+Application Properties | +--keytab |
+
spark.kerberos.principal |
+Application Properties | +--principal |
+
For Windows users: Due to different file prefixes across operating systems, to avoid the issue of potential wrong prefix, a current workaround is to specify spark.sql.warehouse.dir when starting the SparkSession.
spark_warehouse_path <- file.path(path.expand('~'), "spark-warehouse")
+sparkR.session(spark.sql.warehouse.dir = spark_warehouse_path)
+
+Cluster Mode
+SparkR can connect to remote Spark clusters. Cluster Mode Overview is a good introduction to different Spark cluster modes.
+When connecting SparkR to a remote Spark cluster, make sure that the Spark version and Hadoop version on the machine match the corresponding versions on the cluster. Current SparkR package is compatible with
+## [1] "Spark 3.5.0"It should be used both on the local computer and on the remote cluster.
+To connect, pass the URL of the master node to sparkR.session. A complete list can be seen in Spark Master URLs. For example, to connect to a local standalone Spark master, we can call
For YARN cluster, SparkR supports the client mode with the master set as “yarn”.
+ +Yarn cluster mode is not supported in the current version.
+
+
+Data Import
+
+
+Local Data Frame
+The simplest way is to convert a local R data frame into a SparkDataFrame. Specifically we can use as.DataFrame or createDataFrame and pass in the local R data frame to create a SparkDataFrame. As an example, the following creates a SparkDataFrame based using the faithful dataset from R.
## eruptions waiting
+## 1 3.600 79
+## 2 1.800 54
+## 3 3.333 74
+## 4 2.283 62
+## 5 4.533 85
+## 6 2.883 55
+
+
+
+Data Sources
+SparkR supports operating on a variety of data sources through the SparkDataFrame interface. You can check the Spark SQL Programming Guide for more specific options that are available for the built-in data sources.
The general method for creating SparkDataFrame from data sources is read.df. This method takes in the path for the file to load and the type of data source, and the currently active Spark Session will be used automatically. SparkR supports reading CSV, JSON and Parquet files natively and through Spark Packages you can find data source connectors for popular file formats like Avro. These packages can be added with sparkPackages parameter when initializing SparkSession using sparkR.session.
We can see how to use data sources using an example CSV input file. For more information please refer to SparkR read.df API documentation.
+ +The data sources API natively supports JSON formatted input files. Note that the file that is used here is not a typical JSON file. Each line in the file must contain a separate, self-contained valid JSON object. As a consequence, a regular multi-line JSON file will most often fail.
+Let’s take a look at the first two lines of the raw JSON file used here.
+filePath <- paste0(sparkR.conf("spark.home"),
+ "/examples/src/main/resources/people.json")
+readLines(filePath, n = 2L)## [1] "{\"name\":\"Michael\"}" "{\"name\":\"Andy\", \"age\":30}"We use read.df to read that into a SparkDataFrame.
## [1] 3## age name
+## 1 NA Michael
+## 2 30 Andy
+## 3 19 JustinSparkR automatically infers the schema from the JSON file.
+ +## root
+## |-- age: long (nullable = true)
+## |-- name: string (nullable = true)If we want to read multiple JSON files, read.json can be used.
people <- read.json(paste0(Sys.getenv("SPARK_HOME"),
+ c("/examples/src/main/resources/people.json",
+ "/examples/src/main/resources/people.json")))
+count(people)## [1] 6The data sources API can also be used to save out SparkDataFrames into multiple file formats. For example we can save the SparkDataFrame from the previous example to a Parquet file using write.df.
+
+
+Hive Tables
+You can also create SparkDataFrames from Hive tables. To do this we will need to create a SparkSession with Hive support which can access tables in the Hive MetaStore. Note that Spark should have been built with Hive support and more details can be found in the SQL Programming Guide. In SparkR, by default it will attempt to create a SparkSession with Hive support enabled (enableHiveSupport = TRUE).
sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
+
+txtPath <- paste0(sparkR.conf("spark.home"), "/examples/src/main/resources/kv1.txt")
+sqlCMD <- sprintf("LOAD DATA LOCAL INPATH '%s' INTO TABLE src", txtPath)
+sql(sqlCMD)
+
+results <- sql("FROM src SELECT key, value")
+
+# results is now a SparkDataFrame
+head(results)
+
+
+
+Data Processing
+To dplyr users: SparkR has similar interface as dplyr in data processing. However, some noticeable differences are worth mentioning in the first place. We use df to represent a SparkDataFrame and col to represent the name of column here.
-
+
indicate columns. SparkR uses either a character string of the column name or a Column object constructed with
$to indicate a column. For example, to selectcolindf, we can writeselect(df, "col")orselect(df, df$col).
+describe conditions. In SparkR, the Column object representation can be inserted into the condition directly, or we can use a character string to describe the condition, without referring to the
SparkDataFrameused. For example, to select rows with value > 1, we can writefilter(df, df$col > 1)orfilter(df, "col > 1").
+
Here are more concrete examples.
+| dplyr | +SparkR | +
|---|---|
select(mtcars, mpg, hp) |
+select(carsDF, "mpg", "hp") |
+
filter(mtcars, mpg > 20, hp > 100) |
+filter(carsDF, carsDF$mpg > 20, carsDF$hp > 100) |
+
Other differences will be mentioned in the specific methods.
+We use the SparkDataFrame carsDF created above. We can get basic information about the SparkDataFrame.
## SparkDataFrame[model:string, mpg:double, cyl:double, disp:double, hp:double, drat:double, wt:double, qsec:double, vs:double, am:double, gear:double, carb:double]Print out the schema in tree format.
+ +## root
+## |-- model: string (nullable = true)
+## |-- mpg: double (nullable = true)
+## |-- cyl: double (nullable = true)
+## |-- disp: double (nullable = true)
+## |-- hp: double (nullable = true)
+## |-- drat: double (nullable = true)
+## |-- wt: double (nullable = true)
+## |-- qsec: double (nullable = true)
+## |-- vs: double (nullable = true)
+## |-- am: double (nullable = true)
+## |-- gear: double (nullable = true)
+## |-- carb: double (nullable = true)
+
+SparkDataFrame Operations
+
+
+Selecting rows, columns
+SparkDataFrames support a number of functions to do structured data processing. Here we include some basic examples and a complete list can be found in the API docs:
+You can also pass in column name as strings.
+ +## mpg
+## 1 21.0
+## 2 21.0
+## 3 22.8
+## 4 21.4
+## 5 18.7
+## 6 18.1Filter the SparkDataFrame to only retain rows with mpg less than 20 miles/gallon.
+ +## model mpg cyl disp hp drat wt qsec vs am gear carb
+## 1 Hornet Sportabout 18.7 8 360.0 175 3.15 3.44 17.02 0 0 3 2
+## 2 Valiant 18.1 6 225.0 105 2.76 3.46 20.22 1 0 3 1
+## 3 Duster 360 14.3 8 360.0 245 3.21 3.57 15.84 0 0 3 4
+## 4 Merc 280 19.2 6 167.6 123 3.92 3.44 18.30 1 0 4 4
+## 5 Merc 280C 17.8 6 167.6 123 3.92 3.44 18.90 1 0 4 4
+## 6 Merc 450SE 16.4 8 275.8 180 3.07 4.07 17.40 0 0 3 3
+
+Grouping, Aggregation
+A common flow of grouping and aggregation is
+-
+
Use
groupByorgroup_bywith respect to some grouping variables to create aGroupedDataobject
+Feed the
GroupedDataobject toaggorsummarizefunctions, with some provided aggregation functions to compute a number within each group.
+
A number of widely used functions are supported to aggregate data after grouping, including avg, count_distinct, count, first, kurtosis, last, max, mean, min, sd, skewness, stddev_pop, stddev_samp, sum_distinct, sum, var_pop, var_samp, var. See the API doc for aggregate functions linked there.
For example we can compute a histogram of the number of cylinders in the mtcars dataset as shown below.
## cyl count
+## 1 8 14
+## 2 4 11
+## 3 6 7Use cube or rollup to compute subtotals across multiple dimensions.
## SparkDataFrame[cyl:double, gear:double, am:double, avg(mpg):double]generates groupings for {(cyl, gear, am), (cyl, gear), (cyl), ()}, while
## SparkDataFrame[cyl:double, gear:double, am:double, avg(mpg):double]generates groupings for all possible combinations of grouping columns.
+
+
+
+Operating on Columns
+SparkR also provides a number of functions that can directly applied to columns for data processing and during aggregation. The example below shows the use of basic arithmetic functions.
+carsDF_km <- carsDF
+carsDF_km$kmpg <- carsDF_km$mpg * 1.61
+head(select(carsDF_km, "model", "mpg", "kmpg"))## model mpg kmpg
+## 1 Mazda RX4 21.0 33.810
+## 2 Mazda RX4 Wag 21.0 33.810
+## 3 Datsun 710 22.8 36.708
+## 4 Hornet 4 Drive 21.4 34.454
+## 5 Hornet Sportabout 18.7 30.107
+## 6 Valiant 18.1 29.141
+
+
+Window Functions
+A window function is a variation of aggregation function. In simple words,
+-
+
aggregation function:
nto1mapping - returns a single value for a group of entries. Examples includesum,count,max.
+window function:
ntonmapping - returns one value for each entry in the group, but the value may depend on all the entries of the group. Examples includerank,lead,lag.
+
Formally, the group mentioned above is called the frame. Every input row can have a unique frame associated with it and the output of the window function on that row is based on the rows confined in that frame.
+Window functions are often used in conjunction with the following functions: windowPartitionBy, windowOrderBy, partitionBy, orderBy, over. To illustrate this we next look at an example.
We still use the mtcars dataset. The corresponding SparkDataFrame is carsDF. Suppose for each number of cylinders, we want to calculate the rank of each car in mpg within the group.
carsSubDF <- select(carsDF, "model", "mpg", "cyl")
+ws <- orderBy(windowPartitionBy("cyl"), "mpg")
+carsRank <- withColumn(carsSubDF, "rank", over(rank(), ws))
+head(carsRank, n = 20L)## model mpg cyl rank
+## 1 Volvo 142E 21.4 4 1
+## 2 Toyota Corona 21.5 4 2
+## 3 Datsun 710 22.8 4 3
+## 4 Merc 230 22.8 4 3
+## 5 Merc 240D 24.4 4 5
+## 6 Porsche 914-2 26.0 4 6
+## 7 Fiat X1-9 27.3 4 7
+## 8 Honda Civic 30.4 4 8
+## 9 Lotus Europa 30.4 4 8
+## 10 Fiat 128 32.4 4 10
+## 11 Toyota Corolla 33.9 4 11
+## 12 Merc 280C 17.8 6 1
+## 13 Valiant 18.1 6 2
+## 14 Merc 280 19.2 6 3
+## 15 Ferrari Dino 19.7 6 4
+## 16 Mazda RX4 21.0 6 5
+## 17 Mazda RX4 Wag 21.0 6 5
+## 18 Hornet 4 Drive 21.4 6 7
+## 19 Cadillac Fleetwood 10.4 8 1
+## 20 Lincoln Continental 10.4 8 1We explain in detail the above steps.
+-
+
windowPartitionBycreates a window specification objectWindowSpecthat defines the partition. It controls which rows will be in the same partition as the given row. In this case, rows with the same value incylwill be put in the same partition.orderByfurther defines the ordering - the position a given row is in the partition. The resultingWindowSpecis returned asws.
+
More window specification methods include rangeBetween, which can define boundaries of the frame by value, and rowsBetween, which can define the boundaries by row indices.
-
+
withColumnappends a Column calledrankto theSparkDataFrame.overreturns a windowing column. The first argument is usually a Column returned by window function(s) such asrank(),lead(carsDF$wt). That calculates the corresponding values according to the partitioned-and-ordered table.
+
+
+User-Defined Function
+In SparkR, we support several kinds of user-defined functions (UDFs).
+
+
+
+
+Apply by Partition
+dapply can apply a function to each partition of a SparkDataFrame. The function to be applied to each partition of the SparkDataFrame should have only one parameter, a data.frame corresponding to a partition, and the output should be a data.frame as well. Schema specifies the row format of the resulting a SparkDataFrame. It must match to data types of returned value. See here for mapping between R and Spark.
We convert mpg to kmpg (kilometers per gallon). carsSubDF is a SparkDataFrame with a subset of carsDF columns.
carsSubDF <- select(carsDF, "model", "mpg")
+schema <- "model STRING, mpg DOUBLE, kmpg DOUBLE"
+out <- dapply(carsSubDF, function(x) { x <- cbind(x, x$mpg * 1.61) }, schema)
+head(collect(out))## model mpg kmpg
+## 1 Mazda RX4 21.0 33.810
+## 2 Mazda RX4 Wag 21.0 33.810
+## 3 Datsun 710 22.8 36.708
+## 4 Hornet 4 Drive 21.4 34.454
+## 5 Hornet Sportabout 18.7 30.107
+## 6 Valiant 18.1 29.141Like dapply, dapplyCollect can apply a function to each partition of a SparkDataFrame and collect the result back. The output of the function should be a data.frame, but no schema is required in this case. Note that dapplyCollect can fail if the output of the UDF on all partitions cannot be pulled into the driver’s memory.
out <- dapplyCollect(
+ carsSubDF,
+ function(x) {
+ x <- cbind(x, "kmpg" = x$mpg * 1.61)
+ })
+head(out, 3)## model mpg kmpg
+## 1 Mazda RX4 21.0 33.810
+## 2 Mazda RX4 Wag 21.0 33.810
+## 3 Datsun 710 22.8 36.708
+
+
+
+Apply by Group
+gapply can apply a function to each group of a SparkDataFrame. The function is to be applied to each group of the SparkDataFrame and should have only two parameters: grouping key and R data.frame corresponding to that key. The groups are chosen from SparkDataFrames column(s). The output of function should be a data.frame. Schema specifies the row format of the resulting SparkDataFrame. It must represent R function’s output schema on the basis of Spark data types. The column names of the returned data.frame are set by user. See here for mapping between R and Spark.
schema <- structType(structField("cyl", "double"), structField("max_mpg", "double"))
+result <- gapply(
+ carsDF,
+ "cyl",
+ function(key, x) {
+ y <- data.frame(key, max(x$mpg))
+ },
+ schema)
+head(arrange(result, "max_mpg", decreasing = TRUE))## cyl max_mpg
+## 1 4 33.9
+## 2 6 21.4
+## 3 8 19.2Like gapply, gapplyCollect can apply a function to each partition of a SparkDataFrame and collect the result back to R data.frame. The output of the function should be a data.frame but no schema is required in this case. Note that gapplyCollect can fail if the output of the UDF on all partitions cannot be pulled into the driver’s memory.
result <- gapplyCollect(
+ carsDF,
+ "cyl",
+ function(key, x) {
+ y <- data.frame(key, max(x$mpg))
+ colnames(y) <- c("cyl", "max_mpg")
+ y
+ })
+head(result[order(result$max_mpg, decreasing = TRUE), ])## cyl max_mpg
+## 1 4 33.9
+## 2 6 21.4
+## 3 8 19.2
+
+
+Distribute Local Functions
+Similar to lapply in native R, spark.lapply runs a function over a list of elements and distributes the computations with Spark. spark.lapply works in a manner that is similar to doParallel or lapply to elements of a list. The results of all the computations should fit in a single machine. If that is not the case you can do something like df <- createDataFrame(list) and then use dapply.
We use svm in package e1071 as an example. We use all default settings except for varying costs of constraints violation. spark.lapply can train those different models in parallel.
costs <- exp(seq(from = log(1), to = log(1000), length.out = 5))
+train <- function(cost) {
+ stopifnot(requireNamespace("e1071", quietly = TRUE))
+ model <- e1071::svm(Species ~ ., data = iris, cost = cost)
+ summary(model)
+}Return a list of model’s summaries.
+ + +## [1] "list"To avoid lengthy display, we only present the partial result of the second fitted model. You are free to inspect other models as well.
+ +## $call
+## svm(formula = Species ~ ., data = iris, cost = cost)
+##
+## $type
+## [1] 0
+##
+## $kernel
+## [1] 2
+##
+## $cost
+## [1] 5.623413
+##
+## $degree
+## [1] 3
+##
+## $gamma
+## [1] 0.25
+##
+## $coef0
+## [1] 0
+##
+## $nu
+## [1] 0.5
+##
+## $epsilon
+## [1] 0.1
+##
+## $sparse
+## [1] FALSE
+##
+## $scaled
+## [1] TRUE TRUE TRUE TRUE
+##
+## $x.scale
+## $x.scale$`scaled:center`
+## Sepal.Length Sepal.Width Petal.Length Petal.Width
+## 5.843333 3.057333 3.758000 1.199333
+##
+## $x.scale$`scaled:scale`
+## Sepal.Length Sepal.Width Petal.Length Petal.Width
+## 0.8280661 0.4358663 1.7652982 0.7622377
+##
+##
+## $y.scale
+## NULL
+##
+## $nclasses
+## [1] 3
+##
+## $levels
+## [1] "setosa" "versicolor" "virginica"
+##
+## $tot.nSV
+## [1] 35
+##
+## $nSV
+## [1] 6 15 14
+##
+## $labels
+## [1] 1 2 3
+##
+## $SV
+## Sepal.Length Sepal.Width Petal.Length Petal.Width
+## 14 -1.86378030 -0.13153881 -1.5056946 -1.4422448
+## 16 -0.17309407 3.08045544 -1.2791040 -1.0486668
+## 21 -0.53538397 0.78617383 -1.1658087 -1.3110521
+## 23 -1.50149039 1.24503015 -1.5623422 -1.3110521
+## 24 -0.89767388 0.55674567 -1.1658087 -0.9174741
+## 42 -1.62225369 -1.73753594 -1.3923993 -1.1798595
+## 51 1.39682886 0.32731751 0.5336209 0.2632600
+## 53 1.27606556 0.09788935 0.6469162 0.3944526
+## 54 -0.41462067 -1.73753594 0.1370873 0.1320673
+## 55 0.79301235 -0.59039513 0.4769732 0.3944526
+## [ reached getOption("max.print") -- omitted 25 rows ]
+##
+## $index
+## [1] 14 16 21 23 24 42 51 53 54 55 58 61 69 71 73 78 79 84 85
+## [20] 86 99 107 111 119 120 124 127 128 130 132 134 135 139 149 150
+##
+## $rho
+## [1] -0.10346530 0.12160294 -0.09540346
+##
+## $compprob
+## [1] FALSE
+##
+## $probA
+## NULL
+##
+## $probB
+## NULL
+##
+## $sigma
+## NULL
+##
+## $coefs
+## [,1] [,2]
+## [1,] 0.00000000 0.06561739
+## [2,] 0.76813720 0.93378721
+## [3,] 0.00000000 0.12123270
+## [4,] 0.00000000 0.31170741
+## [5,] 1.11614066 0.46397392
+## [6,] 1.88141600 1.10392128
+## [7,] -0.55872622 0.00000000
+## [8,] 0.00000000 5.62341325
+## [9,] 0.00000000 0.27711792
+## [10,] 0.00000000 5.28440007
+## [11,] -1.06596713 0.00000000
+## [12,] -0.57076709 1.09019756
+## [13,] -0.03365904 5.62341325
+## [14,] 0.00000000 5.62341325
+## [15,] 0.00000000 5.62341325
+## [16,] 0.00000000 5.62341325
+## [17,] 0.00000000 4.70398738
+## [18,] 0.00000000 5.62341325
+## [19,] 0.00000000 4.97981371
+## [20,] -0.77497987 0.00000000
+## [ reached getOption("max.print") -- omitted 15 rows ]
+##
+## $na.action
+## NULL
+##
+## $fitted
+## 1 2 3 4 5 6 7 8 9 10 11
+## setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa
+## 12 13 14 15 16 17 18 19 20 21 22
+## setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa
+## 23 24 25 26 27 28 29 30 31 32 33
+## setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa setosa
+## 34 35 36 37 38 39 40
+## setosa setosa setosa setosa setosa setosa setosa
+## [ reached getOption("max.print") -- omitted 110 entries ]
+## Levels: setosa versicolor virginica
+##
+## $decision.values
+## setosa/versicolor setosa/virginica versicolor/virginica
+## 1 1.1911739 1.0908424 1.1275805
+## 2 1.1336557 1.0619543 1.3260964
+## 3 1.2085065 1.0698101 1.0511345
+## 4 1.1646153 1.0505915 1.0806874
+## 5 1.1880814 1.0950348 0.9542815
+## 6 1.0990761 1.0984626 0.9326361
+## 7 1.1573474 1.0343287 0.9726843
+## 8 1.1851598 1.0815750 1.2206802
+## 9 1.1673499 1.0406734 0.8837945
+## 10 1.1629911 1.0560925 1.2430067
+## 11 1.1339282 1.0803946 1.0338357
+## 12 1.1724182 1.0641469 1.1190423
+## 13 1.1827355 1.0667956 1.1414844
+## [ reached getOption("max.print") -- omitted 137 rows ]
+##
+## $terms
+## Species ~ Sepal.Length + Sepal.Width + Petal.Length + Petal.Width
+## attr(,"variables")
+## list(Species, Sepal.Length, Sepal.Width, Petal.Length, Petal.Width)
+## attr(,"factors")
+## Sepal.Length Sepal.Width Petal.Length Petal.Width
+## Species 0 0 0 0
+## Sepal.Length 1 0 0 0
+## Sepal.Width 0 1 0 0
+## Petal.Length 0 0 1 0
+## Petal.Width 0 0 0 1
+## attr(,"term.labels")
+## [1] "Sepal.Length" "Sepal.Width" "Petal.Length" "Petal.Width"
+## attr(,"order")
+## [1] 1 1 1 1
+## attr(,"intercept")
+## [1] 0
+## attr(,"response")
+## [1] 1
+## attr(,".Environment")
+## <environment: 0x55f721df1dc0>
+## attr(,"predvars")
+## list(Species, Sepal.Length, Sepal.Width, Petal.Length, Petal.Width)
+## attr(,"dataClasses")
+## Species Sepal.Length Sepal.Width Petal.Length Petal.Width
+## "factor" "numeric" "numeric" "numeric" "numeric"
+##
+## attr(,"class")
+## [1] "summary.svm"
+
+
+SQL Queries
+A SparkDataFrame can also be registered as a temporary view in Spark SQL so that one can run SQL queries over its data. The sql function enables applications to run SQL queries programmatically and returns the result as a SparkDataFrame.
people <- read.df(paste0(sparkR.conf("spark.home"),
+ "/examples/src/main/resources/people.json"), "json")Register this SparkDataFrame as a temporary view.
SQL statements can be run using the sql method.
+ +## name
+## 1 Justin
+
+Machine Learning
+SparkR supports the following machine learning models and algorithms.
+
+
+Classification
+-
+
Linear Support Vector Machine (SVM) Classifier
+Logistic Regression
+Multilayer Perceptron (MLP)
+Naive Bayes
+Factorization Machines (FM) Classifier
+
+
+Regression
+-
+
Accelerated Failure Time (AFT) Survival Model
+Generalized Linear Model (GLM)
+Isotonic Regression
+Linear Regression
+Factorization Machines (FM) Regressor
+
+
+Tree - Classification and Regression
+-
+
Decision Tree
+Gradient-Boosted Trees (GBT)
+Random Forest
+
+
+Clustering
+-
+
Bisecting \(k\)-means
+Gaussian Mixture Model (GMM)
+\(k\)-means Clustering
+Latent Dirichlet Allocation (LDA)
+Power Iteration Clustering (PIC)
+
+
+Collaborative Filtering
+-
+
- Alternating Least Squares (ALS) +
+
+Frequent Pattern Mining
+-
+
- FP-growth +
- PrefixSpan +
+
+Statistics
+-
+
- Kolmogorov-Smirnov Test +
+
+R Formula
+For most above, SparkR supports R formula operators, including ~, ., :, + and - for model fitting. This makes it a similar experience as using R functions.
+
+
+
+Training and Test Sets
+We can easily split SparkDataFrame into random training and test sets by the randomSplit function. It returns a list of split SparkDataFrames with provided weights. We use carsDF as an example and want to have about \(70%\) training data and \(30%\) test data.
splitDF_list <- randomSplit(carsDF, c(0.7, 0.3), seed = 0)
+carsDF_train <- splitDF_list[[1]]
+carsDF_test <- splitDF_list[[2]]## [1] 24## model mpg cyl disp hp drat wt qsec vs am gear carb
+## 1 Cadillac Fleetwood 10.4 8 472 205 2.93 5.250 17.98 0 0 3 4
+## 2 Camaro Z28 13.3 8 350 245 3.73 3.840 15.41 0 0 3 4
+## 3 Chrysler Imperial 14.7 8 440 230 3.23 5.345 17.42 0 0 3 4
+## 4 Dodge Challenger 15.5 8 318 150 2.76 3.520 16.87 0 0 3 2
+## 5 Duster 360 14.3 8 360 245 3.21 3.570 15.84 0 0 3 4
+## 6 Ferrari Dino 19.7 6 145 175 3.62 2.770 15.50 0 1 5 6## [1] 8## model mpg cyl disp hp drat wt qsec vs am gear carb
+## 1 AMC Javelin 15.2 8 304.0 150 3.15 3.435 17.30 0 0 3 2
+## 2 Datsun 710 22.8 4 108.0 93 3.85 2.320 18.61 1 1 4 1
+## 3 Fiat 128 32.4 4 78.7 66 4.08 2.200 19.47 1 1 4 1
+## 4 Merc 240D 24.4 4 146.7 62 3.69 3.190 20.00 1 0 4 2
+## 5 Merc 280 19.2 6 167.6 123 3.92 3.440 18.30 1 0 4 4
+## 6 Toyota Corolla 33.9 4 71.1 65 4.22 1.835 19.90 1 1 4 1
+
+Models and Algorithms
+
+
+
+
+Linear Support Vector Machine (SVM) Classifier
+Linear Support Vector Machine (SVM) classifier is an SVM classifier with linear kernels. This is a binary classifier. We use a simple example to show how to use spark.svmLinear for binary classification.
# load training data and create a DataFrame
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# fit a Linear SVM classifier model
+model <- spark.svmLinear(training, Survived ~ ., regParam = 0.01, maxIter = 10)
+summary(model)## $coefficients
+## Estimate
+## (Intercept) 0.993131388
+## Class_1st -0.386500359
+## Class_2nd -0.622627816
+## Class_3rd -0.204446602
+## Sex_Female -0.589950309
+## Age_Adult 0.741676902
+## Freq -0.006582887
+##
+## $numClasses
+## [1] 2
+##
+## $numFeatures
+## [1] 6Predict values on training data
+prediction <- predict(model, training)
+head(select(prediction, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))## Class Sex Age Freq Survived prediction
+## 1 1st Male Child 0 No Yes
+## 2 2nd Male Child 0 No Yes
+## 3 3rd Male Child 35 No Yes
+## 4 Crew Male Child 0 No Yes
+## 5 1st Female Child 0 No Yes
+## 6 2nd Female Child 0 No No
+
+
+
+
+Logistic Regression
+Logistic regression is a widely-used model when the response is categorical. It can be seen as a special case of the Generalized Linear Predictive Model. We provide spark.logit on top of spark.glm to support logistic regression with advanced hyper-parameters. It supports both binary and multiclass classification with elastic-net regularization and feature standardization, similar to glmnet.
We use a simple example to demonstrate spark.logit usage. In general, there are three steps of using spark.logit: 1). Create a dataframe from a proper data source; 2). Fit a logistic regression model using spark.logit with a proper parameter setting; and 3). Obtain the coefficient matrix of the fitted model using summary and use the model for prediction with predict.
Binomial logistic regression
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+model <- spark.logit(training, Survived ~ ., regParam = 0.04741301)
+summary(model)## $coefficients
+## Estimate
+## (Intercept) 0.2255014282
+## Class_1st -0.1338856652
+## Class_2nd -0.1479826947
+## Class_3rd 0.0005674937
+## Sex_Female -0.2011183871
+## Age_Adult 0.3263186885
+## Freq -0.0033111157Predict values on training data
+fitted <- predict(model, training)
+head(select(fitted, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))## Class Sex Age Freq Survived prediction
+## 1 1st Male Child 0 No Yes
+## 2 2nd Male Child 0 No Yes
+## 3 3rd Male Child 35 No Yes
+## 4 Crew Male Child 0 No Yes
+## 5 1st Female Child 0 No No
+## 6 2nd Female Child 0 No NoMultinomial logistic regression against three classes
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# Note in this case, Spark infers it is multinomial logistic regression, so family = "multinomial" is optional.
+model <- spark.logit(training, Class ~ ., regParam = 0.07815179)
+summary(model)## $coefficients
+## 1st 2nd 3rd Crew
+## (Intercept) 0.051662845 0.062998145 -0.039083689 -0.075577300
+## Sex_Female -0.088030587 -0.102528148 0.059233106 0.131325629
+## Age_Adult 0.141935316 0.169492058 -0.102562719 -0.208864654
+## Survived_No 0.052721020 0.057980057 -0.029408423 -0.081292653
+## Freq -0.001555912 -0.001970377 0.001303836 0.002222453
+
+
+
+Multilayer Perceptron
+Multilayer perceptron classifier (MLPC) is a classifier based on the feedforward artificial neural network. MLPC consists of multiple layers of nodes. Each layer is fully connected to the next layer in the network. Nodes in the input layer represent the input data. All other nodes map inputs to outputs by a linear combination of the inputs with the node’s weights \(w\) and bias \(b\) and applying an activation function. This can be written in matrix form for MLPC with \(K+1\) layers as follows: \[ +y(x)=f_K(\ldots f_2(w_2^T f_1(w_1^T x + b_1) + b_2) \ldots + b_K). +\]
+Nodes in intermediate layers use sigmoid (logistic) function: \[ +f(z_i) = \frac{1}{1+e^{-z_i}}. +\]
+Nodes in the output layer use softmax function: \[ +f(z_i) = \frac{e^{z_i}}{\sum_{k=1}^N e^{z_k}}. +\]
+The number of nodes \(N\) in the output layer corresponds to the number of classes.
+MLPC employs backpropagation for learning the model. We use the logistic loss function for optimization and L-BFGS as an optimization routine.
+spark.mlp requires at least two columns in data: one named "label" and the other one "features". The "features" column should be in libSVM-format.
We use Titanic data set to show how to use spark.mlp in classification.
t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+# fit a Multilayer Perceptron Classification Model
+model <- spark.mlp(training, Survived ~ Age + Sex, blockSize = 128, layers = c(2, 2), solver = "l-bfgs", maxIter = 100, tol = 0.5, stepSize = 1, seed = 1, initialWeights = c( 0, 0, 5, 5, 9, 9))To avoid lengthy display, we only present partial results of the model summary. You can check the full result from your sparkR shell.
+ +## $numOfInputs
+## [1] 2
+##
+## $numOfOutputs
+## [1] 2
+##
+## $layers
+## [1] 2 2
+##
+## $weights
+## $weights[[1]]
+## [1] 0
+##
+## $weights[[2]]
+## [1] 0
+##
+## $weights[[3]]
+## [1] 5
+##
+## $weights[[4]]
+## [1] 5
+##
+## $weights[[5]]
+## [1] 9
+##
+## $weights[[6]]
+## [1] 9# make predictions use the fitted model
+predictions <- predict(model, training)
+head(select(predictions, predictions$prediction))## prediction
+## 1 No
+## 2 No
+## 3 No
+## 4 No
+## 5 No
+## 6 No
+
+
+
+Naive Bayes
+Naive Bayes model assumes independence among the features. spark.naiveBayes fits a Bernoulli naive Bayes model against a SparkDataFrame. The data should be all categorical. These models are often used for document classification.
titanic <- as.data.frame(Titanic)
+titanicDF <- createDataFrame(titanic[titanic$Freq > 0, -5])
+naiveBayesModel <- spark.naiveBayes(titanicDF, Survived ~ Class + Sex + Age)
+summary(naiveBayesModel)## $apriori
+## Yes No
+## [1,] 0.5769231 0.4230769
+##
+## $tables
+## Class_3rd Class_1st Class_2nd Sex_Female Age_Adult
+## Yes 0.3125 0.3125 0.3125 0.5 0.5625
+## No 0.4166667 0.25 0.25 0.5 0.75naiveBayesPrediction <- predict(naiveBayesModel, titanicDF)
+head(select(naiveBayesPrediction, "Class", "Sex", "Age", "Survived", "prediction"))## Class Sex Age Survived prediction
+## 1 3rd Male Child No Yes
+## 2 3rd Female Child No Yes
+## 3 1st Male Adult No Yes
+## 4 2nd Male Adult No Yes
+## 5 3rd Male Adult No No
+## 6 Crew Male Adult No Yes
+
+
+Factorization Machines Classifier
+Factorization Machines for classification problems.
+For background and details about the implementation of factorization machines, refer to the Factorization Machines section.
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+
+model <- spark.fmClassifier(training, Survived ~ Age + Sex)
+summary(model)## $coefficients
+## Estimate
+## (Intercept) 0.0064275991
+## Age_Adult 0.0001294448
+## Sex_Female 0.0001294448
+##
+## $factors
+## [,1] [,2] [,3] [,4] [,5] [,6]
+## [1,] -0.3256224 0.11912568 0.1460235 0.1620567 0.13153516 0.06403695
+## [2,] -0.1382155 -0.03658261 0.1717808 -0.1602241 -0.08446129 -0.19287098
+## [,7] [,8]
+## [1,] -0.03292446 -0.05166818
+## [2,] 0.19252571 0.06237194
+##
+## $numClasses
+## [1] 2
+##
+## $numFeatures
+## [1] 2
+##
+## $factorSize
+## [1] 8## prediction
+## 1 Yes
+## 2 Yes
+## 3 Yes
+## 4 Yes
+## 5 Yes
+## 6 Yes
+
+
+Accelerated Failure Time Survival Model
+Survival analysis studies the expected duration of time until an event happens, and often the relationship with risk factors or treatment taken on the subject. In contrast to standard regression analysis, survival modeling has to deal with special characteristics in the data including non-negative survival time and censoring.
+Accelerated Failure Time (AFT) model is a parametric survival model for censored data that assumes the effect of a covariate is to accelerate or decelerate the life course of an event by some constant. For more information, refer to the Wikipedia page AFT Model and the references there. Different from a Proportional Hazards Model designed for the same purpose, the AFT model is easier to parallelize because each instance contributes to the objective function independently.
+library(survival)
+ovarianDF <- createDataFrame(ovarian)
+aftModel <- spark.survreg(ovarianDF, Surv(futime, fustat) ~ ecog_ps + rx)
+summary(aftModel)## $coefficients
+## Value
+## (Intercept) 6.8966910
+## ecog_ps -0.3850414
+## rx 0.5286455
+## Log(scale) -0.1234429## futime fustat age resid_ds rx ecog_ps label prediction
+## 1 59 1 72.3315 2 1 1 59 1141.724
+## 2 115 1 74.4932 2 1 1 115 1141.724
+## 3 156 1 66.4658 2 1 2 156 776.855
+## 4 421 0 53.3644 2 2 1 421 1937.087
+## 5 431 1 50.3397 2 1 1 431 1141.724
+## 6 448 0 56.4301 1 1 2 448 776.855
+
+
+
+
+
+
+
+Generalized Linear Model
+The main function is spark.glm. The following families and link functions are supported. The default is gaussian.
| Family | +Link Function | +
|---|---|
| gaussian | +identity, log, inverse | +
| binomial | +logit, probit, cloglog (complementary log-log) | +
| poisson | +log, identity, sqrt | +
| gamma | +inverse, identity, log | +
| tweedie | +power link function | +
There are three ways to specify the family argument.
-
+
Family name as a character string, e.g.
family = "gaussian".
+Family function, e.g.
family = binomial.
+Result returned by a family function, e.g.
family = poisson(link = log).
+- Note that there are two ways to specify the tweedie family:
+
-
+
- Set
family = "tweedie"and specify thevar.powerandlink.power
+ - When package
statmodis loaded, the tweedie family is specified using the family definition therein, i.e.,tweedie().
+
+ - Set
For more information regarding the families and their link functions, see the Wikipedia page Generalized Linear Model.
+We use the mtcars dataset as an illustration. The corresponding SparkDataFrame is carsDF. After fitting the model, we print out a summary and see the fitted values by making predictions on the original dataset. We can also pass into a new SparkDataFrame of same schema to predict on new data.
##
+## Deviance Residuals:
+## (Note: These are approximate quantiles with relative error <= 0.01)
+## Min 1Q Median 3Q Max
+## -3.9410 -1.6499 -0.3267 1.0373 5.8538
+##
+## Coefficients:
+## Estimate Std. Error t value Pr(>|t|)
+## (Intercept) 37.227270 1.5987875 23.2847 0.0000e+00
+## wt -3.877831 0.6327335 -6.1287 1.1196e-06
+## hp -0.031773 0.0090297 -3.5187 1.4512e-03
+##
+## (Dispersion parameter for gaussian family taken to be 6.725785)
+##
+## Null deviance: 1126.05 on 31 degrees of freedom
+## Residual deviance: 195.05 on 29 degrees of freedom
+## AIC: 156.7
+##
+## Number of Fisher Scoring iterations: 1When doing prediction, a new column called prediction will be appended. Let’s look at only a subset of columns here.
gaussianFitted <- predict(gaussianGLM, carsDF)
+head(select(gaussianFitted, "model", "prediction", "mpg", "wt", "hp"))## model prediction mpg wt hp
+## 1 Mazda RX4 23.57233 21.0 2.620 110
+## 2 Mazda RX4 Wag 22.58348 21.0 2.875 110
+## 3 Datsun 710 25.27582 22.8 2.320 93
+## 4 Hornet 4 Drive 21.26502 21.4 3.215 110
+## 5 Hornet Sportabout 18.32727 18.7 3.440 175
+## 6 Valiant 20.47382 18.1 3.460 105The following is the same fit using the tweedie family:
+tweedieGLM1 <- spark.glm(carsDF, mpg ~ wt + hp, family = "tweedie", var.power = 0.0)
+summary(tweedieGLM1)##
+## Deviance Residuals:
+## (Note: These are approximate quantiles with relative error <= 0.01)
+## Min 1Q Median 3Q Max
+## -3.9410 -1.6499 -0.3267 1.0373 5.8538
+##
+## Coefficients:
+## Estimate Std. Error t value Pr(>|t|)
+## (Intercept) 37.227270 1.5987875 23.2847 0.0000e+00
+## wt -3.877831 0.6327335 -6.1287 1.1196e-06
+## hp -0.031773 0.0090297 -3.5187 1.4512e-03
+##
+## (Dispersion parameter for tweedie family taken to be 6.725785)
+##
+## Null deviance: 1126.05 on 31 degrees of freedom
+## Residual deviance: 195.05 on 29 degrees of freedom
+## AIC: 156.7
+##
+## Number of Fisher Scoring iterations: 1We can try other distributions in the tweedie family, for example, a compound Poisson distribution with a log link:
+tweedieGLM2 <- spark.glm(carsDF, mpg ~ wt + hp, family = "tweedie",
+ var.power = 1.2, link.power = 0.0)
+summary(tweedieGLM2)##
+## Deviance Residuals:
+## (Note: These are approximate quantiles with relative error <= 0.01)
+## Min 1Q Median 3Q Max
+## -0.58074 -0.25335 -0.09892 0.18608 0.82717
+##
+## Coefficients:
+## Estimate Std. Error t value Pr(>|t|)
+## (Intercept) 3.8500849 0.06698272 57.4788 0.0000e+00
+## wt -0.2018426 0.02897283 -6.9666 1.1691e-07
+## hp -0.0016248 0.00041603 -3.9054 5.1697e-04
+##
+## (Dispersion parameter for tweedie family taken to be 0.1340111)
+##
+## Null deviance: 29.8820 on 31 degrees of freedom
+## Residual deviance: 3.7739 on 29 degrees of freedom
+## AIC: NA
+##
+## Number of Fisher Scoring iterations: 4
+
+
+Isotonic Regression
+spark.isoreg fits an Isotonic Regression model against a SparkDataFrame. It solves a weighted univariate a regression problem under a complete order constraint. Specifically, given a set of real observed responses \(y_1, \ldots, y_n\), corresponding real features \(x_1, \ldots, x_n\), and optionally positive weights \(w_1, \ldots, w_n\), we want to find a monotone (piecewise linear) function \(f\) to minimize \[
+\ell(f) = \sum_{i=1}^n w_i (y_i - f(x_i))^2.
+\]
There are a few more arguments that may be useful.
+-
+
weightCol: a character string specifying the weight column.
+isotonic: logical value indicating whether the output sequence should be isotonic/increasing (TRUE) or antitonic/decreasing (FALSE).
+featureIndex: the index of the feature on the right hand side of the formula if it is a vector column (default: 0), no effect otherwise.
+
We use an artificial example to show the use.
+y <- c(3.0, 6.0, 8.0, 5.0, 7.0)
+x <- c(1.0, 2.0, 3.5, 3.0, 4.0)
+w <- rep(1.0, 5)
+data <- data.frame(y = y, x = x, w = w)
+df <- createDataFrame(data)
+isoregModel <- spark.isoreg(df, y ~ x, weightCol = "w")
+isoregFitted <- predict(isoregModel, df)
+head(select(isoregFitted, "x", "y", "prediction"))## x y prediction
+## 1 1.0 3 3.0
+## 2 2.0 6 5.5
+## 3 3.5 8 7.5
+## 4 3.0 5 5.5
+## 5 4.0 7 7.5In the prediction stage, based on the fitted monotone piecewise function, the rules are:
+-
+
If the prediction input exactly matches a training feature then associated prediction is returned. In case there are multiple predictions with the same feature then one of them is returned. Which one is undefined.
+If the prediction input is lower or higher than all training features then prediction with lowest or highest feature is returned respectively. In case there are multiple predictions with the same feature then the lowest or highest is returned respectively.
+If the prediction input falls between two training features then prediction is treated as piecewise linear function and interpolated value is calculated from the predictions of the two closest features. In case there are multiple values with the same feature then the same rules as in previous point are used.
+
For example, when the input is \(3.2\), the two closest feature values are \(3.0\) and \(3.5\), then predicted value would be a linear interpolation between the predicted values at \(3.0\) and \(3.5\).
+ +## x prediction
+## 1 1.5 4.25
+## 2 3.2 6.30
+
+Linear Regression
+Linear regression model.
+ +## $coefficients
+## Estimate
+## (Intercept) 37.22727012
+## wt -3.87783074
+## hp -0.03177295
+##
+## $numFeatures
+## [1] 2## prediction
+## 1 23.57233
+## 2 22.58348
+## 3 25.27582
+## 4 21.26502
+## 5 18.32727
+## 6 20.47382
+
+Factorization Machines Regressor
+Factorization Machines for regression problems.
+For background and details about the implementation of factorization machines, refer to the Factorization Machines section.
+ +## $coefficients
+## Estimate
+## (Intercept) 0.1518559
+## wt 3.6472555
+## hp 2.8026828
+##
+## $factors
+## [,1] [,2] [,3] [,4] [,5] [,6]
+## [1,] 0.1424420 -0.1178110 -0.3970272 -0.4696695 0.400288 0.3690930
+## [2,] -0.1626185 0.1512138 0.3690435 0.4076975 -0.625752 -0.3715109
+## [,7] [,8]
+## [1,] 0.03472468 -0.1703219
+## [2,] -0.02109148 -0.2006249
+##
+## $numFeatures
+## [1] 2
+##
+## $factorSize
+## [1] 8## prediction
+## 1 106.70996
+## 2 87.07526
+## 3 111.07931
+## 4 60.89565
+## 5 61.81374
+## 6 40.70095
+
+
+
+Decision Tree
+spark.decisionTree fits a decision tree classification or regression model on a SparkDataFrame. Users can call summary to get a summary of the fitted model, predict to make predictions, and write.ml/read.ml to save/load fitted models.
We use the Titanic dataset to train a decision tree and make predictions:
t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+dtModel <- spark.decisionTree(df, Survived ~ ., type = "classification", maxDepth = 2)
+summary(dtModel)## Formula: Survived ~ .
+## Number of features: 6
+## Features: Class_1st Class_2nd Class_3rd Sex_Female Age_Adult Freq
+## Feature importances: (6,[5],[1.0])
+## Max Depth: 2
+## DecisionTreeClassificationModel: uid=dtc_1a036ac3b7f5, depth=2, numNodes=5, numClasses=2, numFeatures=6
+## If (feature 5 <= 4.5)
+## Predict: 0.0
+## Else (feature 5 > 4.5)
+## If (feature 5 <= 84.5)
+## Predict: 1.0
+## Else (feature 5 > 84.5)
+## Predict: 0.0
+## predictions <- predict(dtModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))## Class Sex Age Freq Survived prediction
+## 1 1st Male Child 0 No No
+## 2 2nd Male Child 0 No No
+## 3 3rd Male Child 35 No Yes
+## 4 Crew Male Child 0 No No
+## 5 1st Female Child 0 No No
+## 6 2nd Female Child 0 No No
+
+
+
+Gradient-Boosted Trees
+spark.gbt fits a gradient-boosted tree classification or regression model on a SparkDataFrame. Users can call summary to get a summary of the fitted model, predict to make predictions, and write.ml/read.ml to save/load fitted models.
We use the Titanic dataset to train a gradient-boosted tree and make predictions:
t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+gbtModel <- spark.gbt(df, Survived ~ ., type = "classification", maxDepth = 2, maxIter = 2)
+summary(gbtModel)## Formula: Survived ~ .
+## Number of features: 6
+## Features: Class_1st Class_2nd Class_3rd Sex_Female Age_Adult Freq
+## Feature importances: (6,[1,2,5],[0.03336902858878361,0.16099525743106016,0.8056357139801562])
+## Max Depth: 2
+## Number of trees: 2
+## Tree weights: 1 0.1
+## GBTClassificationModel: uid = gbtc_b9a7ebafe868, numTrees=2, numClasses=2, numFeatures=6
+## Tree 0 (weight 1.0):
+## If (feature 5 <= 4.5)
+## If (feature 1 in {1.0})
+## Predict: -1.0
+## Else (feature 1 not in {1.0})
+## Predict: -0.3333333333333333
+## Else (feature 5 > 4.5)
+## If (feature 5 <= 84.5)
+## Predict: 0.5714285714285714
+## Else (feature 5 > 84.5)
+## Predict: -0.42857142857142855
+## Tree 1 (weight 0.1):
+## If (feature 2 in {1.0})
+## If (feature 5 <= 15.5)
+## Predict: 0.9671846896296403
+## Else (feature 5 > 15.5)
+## Predict: -1.0857923804083338
+## Else (feature 2 not in {1.0})
+## If (feature 5 <= 13.5)
+## Predict: -0.08651035613926407
+## Else (feature 5 > 13.5)
+## Predict: 0.6566673506774614
+## predictions <- predict(gbtModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))## Class Sex Age Freq Survived prediction
+## 1 1st Male Child 0 No No
+## 2 2nd Male Child 0 No No
+## 3 3rd Male Child 35 No Yes
+## 4 Crew Male Child 0 No No
+## 5 1st Female Child 0 No No
+## 6 2nd Female Child 0 No No
+
+
+
+Random Forest
+spark.randomForest fits a random forest classification or regression model on a SparkDataFrame. Users can call summary to get a summary of the fitted model, predict to make predictions, and write.ml/read.ml to save/load fitted models.
In the following example, we use the Titanic dataset to train a random forest and make predictions:
t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+rfModel <- spark.randomForest(df, Survived ~ ., type = "classification", maxDepth = 2, numTrees = 2)
+summary(rfModel)## Formula: Survived ~ .
+## Number of features: 6
+## Features: Class_1st Class_2nd Class_3rd Sex_Female Age_Adult Freq
+## Feature importances: (6,[3,4,5],[0.17058779274099098,0.09676977311565654,0.7326424341433525])
+## Max Depth: 2
+## Number of trees: 2
+## Tree weights: 1 1
+## RandomForestClassificationModel: uid=rfc_a4079206d906, numTrees=2, numClasses=2, numFeatures=6
+## Tree 0 (weight 1.0):
+## If (feature 4 in {0.0})
+## If (feature 3 in {0.0})
+## Predict: 0.0
+## Else (feature 3 not in {0.0})
+## Predict: 1.0
+## Else (feature 4 not in {0.0})
+## If (feature 5 <= 13.5)
+## Predict: 0.0
+## Else (feature 5 > 13.5)
+## Predict: 1.0
+## Tree 1 (weight 1.0):
+## If (feature 5 <= 84.5)
+## If (feature 5 <= 4.5)
+## Predict: 0.0
+## Else (feature 5 > 4.5)
+## Predict: 1.0
+## Else (feature 5 > 84.5)
+## Predict: 0.0
+## predictions <- predict(rfModel, df)
+head(select(predictions, "Class", "Sex", "Age", "Freq", "Survived", "prediction"))## Class Sex Age Freq Survived prediction
+## 1 1st Male Child 0 No No
+## 2 2nd Male Child 0 No No
+## 3 3rd Male Child 35 No Yes
+## 4 Crew Male Child 0 No No
+## 5 1st Female Child 0 No No
+## 6 2nd Female Child 0 No No
+
+
+Bisecting k-Means
+spark.bisectingKmeans is a kind of hierarchical clustering using a divisive (or “top-down”) approach: all observations start in one cluster, and splits are performed recursively as one moves down the hierarchy.
t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+model <- spark.bisectingKmeans(training, Class ~ Survived, k = 4)
+summary(model)## $k
+## [1] 4
+##
+## $coefficients
+## Survived_No
+## 1 0
+## 2 1
+## 3 0
+## 4 1
+##
+## $size
+## $size[[1]]
+## [1] 16
+##
+## $size[[2]]
+## [1] 16
+##
+## $size[[3]]
+## [1] 0
+##
+## $size[[4]]
+## [1] 0
+##
+##
+## $cluster
+## SparkDataFrame[prediction:int]
+##
+## $is.loaded
+## [1] FALSE## Class prediction
+## 1 1st 1
+## 2 2nd 1
+## 3 3rd 1
+## 4 Crew 1
+## 5 1st 1
+## 6 2nd 1
+
+
+Gaussian Mixture Model
+spark.gaussianMixture fits multivariate Gaussian Mixture Model (GMM) against a SparkDataFrame. Expectation-Maximization (EM) is used to approximate the maximum likelihood estimator (MLE) of the model.
We use a simulated example to demonstrate the usage.
+X1 <- data.frame(V1 = rnorm(4), V2 = rnorm(4))
+X2 <- data.frame(V1 = rnorm(6, 3), V2 = rnorm(6, 4))
+data <- rbind(X1, X2)
+df <- createDataFrame(data)
+gmmModel <- spark.gaussianMixture(df, ~ V1 + V2, k = 2)
+summary(gmmModel)## $lambda
+## [1] 0.486168 0.513832
+##
+## $mu
+## $mu[[1]]
+## [1] 1.424505 1.995314
+##
+## $mu[[2]]
+## [1] 1.922353 2.469800
+##
+##
+## $sigma
+## $sigma[[1]]
+## [,1] [,2]
+## [1,] 3.130571 2.557531
+## [2,] 2.557531 3.903907
+##
+## $sigma[[2]]
+## [,1] [,2]
+## [1,] 2.952933 2.382424
+## [2,] 2.382424 3.763825
+##
+##
+## $loglik
+## [1] -37.05907
+##
+## $posterior
+## SparkDataFrame[posterior:array<double>]
+##
+## $is.loaded
+## [1] FALSE## V1 V2 prediction
+## 1 -1.04010580 0.44731013 0
+## 2 1.03343749 -0.80795279 0
+## 3 0.07209263 0.75728370 0
+## 4 -0.81326509 0.03299667 0
+## 5 3.06240539 5.25819335 1
+## 6 3.23457954 2.18585177 1
+
+
+k-Means Clustering
+spark.kmeans fits a \(k\)-means clustering model against a SparkDataFrame. As an unsupervised learning method, we don’t need a response variable. Hence, the left hand side of the R formula should be left blank. The clustering is based only on the variables on the right hand side.
## $k
+## [1] 3
+##
+## $coefficients
+## mpg hp wt
+## 1 24.22353 93.52941 2.599588
+## 2 15.80000 178.50000 3.926400
+## 3 14.62000 263.80000 3.899000
+##
+## $size
+## $size[[1]]
+## [1] 17
+##
+## $size[[2]]
+## [1] 10
+##
+## $size[[3]]
+## [1] 5
+##
+##
+## $cluster
+## SparkDataFrame[prediction:int]
+##
+## $is.loaded
+## [1] FALSE
+##
+## $clusterSize
+## [1] 3kmeansPredictions <- predict(kmeansModel, carsDF)
+head(select(kmeansPredictions, "model", "mpg", "hp", "wt", "prediction"), n = 20L)## model mpg hp wt prediction
+## 1 Mazda RX4 21.0 110 2.620 0
+## 2 Mazda RX4 Wag 21.0 110 2.875 0
+## 3 Datsun 710 22.8 93 2.320 0
+## 4 Hornet 4 Drive 21.4 110 3.215 0
+## 5 Hornet Sportabout 18.7 175 3.440 1
+## 6 Valiant 18.1 105 3.460 0
+## 7 Duster 360 14.3 245 3.570 2
+## 8 Merc 240D 24.4 62 3.190 0
+## 9 Merc 230 22.8 95 3.150 0
+## 10 Merc 280 19.2 123 3.440 0
+## 11 Merc 280C 17.8 123 3.440 0
+## 12 Merc 450SE 16.4 180 4.070 1
+## 13 Merc 450SL 17.3 180 3.730 1
+## 14 Merc 450SLC 15.2 180 3.780 1
+## 15 Cadillac Fleetwood 10.4 205 5.250 1
+## 16 Lincoln Continental 10.4 215 5.424 1
+## 17 Chrysler Imperial 14.7 230 5.345 2
+## 18 Fiat 128 32.4 66 2.200 0
+## 19 Honda Civic 30.4 52 1.615 0
+## 20 Toyota Corolla 33.9 65 1.835 0
+
+
+Latent Dirichlet Allocation
+spark.lda fits a Latent Dirichlet Allocation model on a SparkDataFrame. It is often used in topic modeling in which topics are inferred from a collection of text documents. LDA can be thought of as a clustering algorithm as follows:
-
+
Topics correspond to cluster centers, and documents correspond to examples (rows) in a dataset.
+Topics and documents both exist in a feature space, where feature vectors are vectors of word counts (bag of words).
+Rather than clustering using a traditional distance, LDA uses a function based on a statistical model of how text documents are generated.
+
To use LDA, we need to specify a features column in data where each entry represents a document. There are two options for the column:
-
+
character string: This can be a string of the whole document. It will be parsed automatically. Additional stop words can be added in
customizedStopWords.
+libSVM: Each entry is a collection of words and will be processed directly.
+
Two more functions are provided for the fitted model.
+-
+
spark.posteriorreturns aSparkDataFramecontaining a column of posterior probabilities vectors named “topicDistribution”.
+spark.perplexityreturns the log perplexity of givenSparkDataFrame, or the log perplexity of the training data if missing argumentdata.
+
For more information, see the help document ?spark.lda.
Let’s look an artificial example.
+corpus <- data.frame(features = c(
+ "1 2 6 0 2 3 1 1 0 0 3",
+ "1 3 0 1 3 0 0 2 0 0 1",
+ "1 4 1 0 0 4 9 0 1 2 0",
+ "2 1 0 3 0 0 5 0 2 3 9",
+ "3 1 1 9 3 0 2 0 0 1 3",
+ "4 2 0 3 4 5 1 1 1 4 0",
+ "2 1 0 3 0 0 5 0 2 2 9",
+ "1 1 1 9 2 1 2 0 0 1 3",
+ "4 4 0 3 4 2 1 3 0 0 0",
+ "2 8 2 0 3 0 2 0 2 7 2",
+ "1 1 1 9 0 2 2 0 0 3 3",
+ "4 1 0 0 4 5 1 3 0 1 0"))
+corpusDF <- createDataFrame(corpus)
+model <- spark.lda(data = corpusDF, k = 5, optimizer = "em")
+summary(model)## $docConcentration
+## [1] 11 11 11 11 11
+##
+## $topicConcentration
+## [1] 1.1
+##
+## $logLikelihood
+## [1] -353.2948
+##
+## $logPerplexity
+## [1] 2.676476
+##
+## $isDistributed
+## [1] TRUE
+##
+## $vocabSize
+## [1] 10
+##
+## $topics
+## SparkDataFrame[topic:int, term:array<string>, termWeights:array<double>]
+##
+## $vocabulary
+## [1] "0" "1" "2" "3" "4" "9" "5" "8" "7" "6"
+##
+## $trainingLogLikelihood
+## [1] -239.5629
+##
+## $logPrior
+## [1] -980.2974## features topicDistribution
+## 1 1 2 6 0 2 3 1 1 0 0 3 0.1972168, 0.1986649, 0.2022016, 0.2006567, 0.2012600
+## 2 1 3 0 1 3 0 0 2 0 0 1 0.1989973, 0.1988744, 0.2015969, 0.2006370, 0.1998944
+## 3 1 4 1 0 0 4 9 0 1 2 0 0.2020597, 0.2026093, 0.1968839, 0.1987286, 0.1997187
+## 4 2 1 0 3 0 0 5 0 2 3 9 0.2004066, 0.1981940, 0.2013016, 0.2006305, 0.1994673
+## 5 3 1 1 9 3 0 2 0 0 1 3 0.1971473, 0.1983960, 0.2023580, 0.2011575, 0.2009413
+## 6 4 2 0 3 4 5 1 1 1 4 0 0.2020226, 0.2041832, 0.1955394, 0.1997222, 0.1985325## [1] 2.676476
+
+
+
+Alternating Least Squares
+spark.als learns latent factors in collaborative filtering via alternating least squares.
There are multiple options that can be configured in spark.als, including rank, reg, and nonnegative. For a complete list, refer to the help file.
ratings <- list(list(0, 0, 4.0), list(0, 1, 2.0), list(1, 1, 3.0), list(1, 2, 4.0),
+ list(2, 1, 1.0), list(2, 2, 5.0))
+df <- createDataFrame(ratings, c("user", "item", "rating"))
+model <- spark.als(df, "rating", "user", "item", rank = 10, reg = 0.1, nonnegative = TRUE)Extract latent factors.
+stats <- summary(model)
+userFactors <- stats$userFactors
+itemFactors <- stats$itemFactors
+head(userFactors)
+head(itemFactors)Make predictions.
+ +
+
+
+Power Iteration Clustering
+Power Iteration Clustering (PIC) is a scalable graph clustering algorithm. spark.assignClusters method runs the PIC algorithm and returns a cluster assignment for each input vertex.
df <- createDataFrame(list(list(0L, 1L, 1.0), list(0L, 2L, 1.0),
+ list(1L, 2L, 1.0), list(3L, 4L, 1.0),
+ list(4L, 0L, 0.1)),
+ schema = c("src", "dst", "weight"))
+head(spark.assignClusters(df, initMode = "degree", weightCol = "weight"))## id cluster
+## 1 4 1
+## 2 0 0
+## 3 1 0
+## 4 3 1
+## 5 2 0
+
+
+FP-growth
+spark.fpGrowth executes FP-growth algorithm to mine frequent itemsets on a SparkDataFrame. itemsCol should be an array of values.
df <- selectExpr(createDataFrame(data.frame(rawItems = c(
+ "T,R,U", "T,S", "V,R", "R,U,T,V", "R,S", "V,S,U", "U,R", "S,T", "V,R", "V,U,S",
+ "T,V,U", "R,V", "T,S", "T,S", "S,T", "S,U", "T,R", "V,R", "S,V", "T,S,U"
+))), "split(rawItems, ',') AS items")
+
+fpm <- spark.fpGrowth(df, minSupport = 0.2, minConfidence = 0.5)spark.freqItemsets method can be used to retrieve a SparkDataFrame with the frequent itemsets.
## items freq
+## 1 R 9
+## 2 U 8
+## 3 U, T 4
+## 4 U, V 4
+## 5 U, S 4
+## 6 T 10spark.associationRules returns a SparkDataFrame with the association rules.
## antecedent consequent confidence lift support
+## 1 V R 0.5555556 1.234568 0.25
+## 2 S T 0.5454545 1.090909 0.30
+## 3 T S 0.6000000 1.090909 0.30
+## 4 R V 0.5555556 1.234568 0.25
+## 5 U T 0.5000000 1.000000 0.20
+## 6 U V 0.5000000 1.111111 0.20We can make predictions based on the antecedent.
## items prediction
+## 1 T, R, U S, V
+## 2 T, S NULL
+## 3 V, R NULL
+## 4 R, U, T, V S
+## 5 R, S T, V
+## 6 V, S, U R, T
+
+
+PrefixSpan
+spark.findFrequentSequentialPatterns method can be used to find the complete set of frequent sequential patterns in the input sequences of itemsets.
df <- createDataFrame(list(list(list(list(1L, 2L), list(3L))),
+ list(list(list(1L), list(3L, 2L), list(1L, 2L))),
+ list(list(list(1L, 2L), list(5L))),
+ list(list(list(6L)))),
+ schema = c("sequence"))
+head(spark.findFrequentSequentialPatterns(df, minSupport = 0.5, maxPatternLength = 5L))## sequence freq
+## 1 1 3
+## 2 3 2
+## 3 2 3
+## 4 1, 2 3
+## 5 1, 3 2
+
+
+Kolmogorov-Smirnov Test
+spark.kstest runs a two-sided, one-sample Kolmogorov-Smirnov (KS) test. Given a SparkDataFrame, the test compares continuous data in a given column testCol with the theoretical distribution specified by parameter nullHypothesis. Users can call summary to get a summary of the test results.
In the following example, we test whether the Titanic dataset’s Freq column follows a normal distribution. We set the parameters of the normal distribution using the mean and standard deviation of the sample.
t <- as.data.frame(Titanic)
+df <- createDataFrame(t)
+freqStats <- head(select(df, mean(df$Freq), sd(df$Freq)))
+freqMean <- freqStats[1]
+freqStd <- freqStats[2]
+
+test <- spark.kstest(df, "Freq", "norm", c(freqMean, freqStd))
+testSummary <- summary(test)
+testSummary## Kolmogorov-Smirnov test summary:
+## degrees of freedom = 0
+## statistic = 0.3065126710255011
+## pValue = 0.0036336792155329256
+## Very strong presumption against null hypothesis: Sample follows theoretical distribution.
+
+
+
+Model Persistence
+The following example shows how to save/load an ML model in SparkR.
+t <- as.data.frame(Titanic)
+training <- createDataFrame(t)
+gaussianGLM <- spark.glm(training, Freq ~ Sex + Age, family = "gaussian")
+
+# Save and then load a fitted MLlib model
+modelPath <- tempfile(pattern = "ml", fileext = ".tmp")
+write.ml(gaussianGLM, modelPath)
+gaussianGLM2 <- read.ml(modelPath)
+
+# Check model summary
+summary(gaussianGLM2)##
+## Saved-loaded model does not support output 'Deviance Residuals'.
+##
+## Coefficients:
+## Estimate Std. Error t value Pr(>|t|)
+## (Intercept) 46.219 35.994 1.2841 0.2092846
+## Sex_Female -78.812 41.562 -1.8962 0.0679311
+## Age_Adult 123.938 41.562 2.9820 0.0057522
+##
+## (Dispersion parameter for gaussian family taken to be 13819.52)
+##
+## Null deviance: 573341 on 31 degrees of freedom
+## Residual deviance: 400766 on 29 degrees of freedom
+## AIC: 400.7
+##
+## Number of Fisher Scoring iterations: 1# Check model prediction
+gaussianPredictions <- predict(gaussianGLM2, training)
+head(gaussianPredictions)## Class Sex Age Survived Freq label prediction
+## 1 1st Male Child No 0 0 46.21875
+## 2 2nd Male Child No 0 0 46.21875
+## 3 3rd Male Child No 35 35 46.21875
+## 4 Crew Male Child No 0 0 46.21875
+## 5 1st Female Child No 0 0 -32.59375
+## 6 2nd Female Child No 0 0 -32.59375
+
+Structured Streaming
+SparkR supports the Structured Streaming API.
+You can check the Structured Streaming Programming Guide for an introduction to its programming model and basic concepts.
+
+
+
+Simple Source and Sink
+Spark has a few built-in input sources. As an example, to test with a socket source reading text into words and displaying the computed word counts:
+# Create DataFrame representing the stream of input lines from connection
+lines <- read.stream("socket", host = hostname, port = port)
+
+# Split the lines into words
+words <- selectExpr(lines, "explode(split(value, ' ')) as word")
+
+# Generate running word count
+wordCounts <- count(groupBy(words, "word"))
+
+# Start running the query that prints the running counts to the console
+query <- write.stream(wordCounts, "console", outputMode = "complete")
+
+Kafka Source
+It is simple to read data from Kafka. For more information, see Input Sources supported by Structured Streaming.
+ +
+
+
+Operations and Sinks
+Most of the common operations on SparkDataFrame are supported for streaming, including selection, projection, and aggregation. Once you have defined the final result, to start the streaming computation, you will call the write.stream method setting a sink and outputMode.
A streaming SparkDataFrame can be written for debugging to the console, to a temporary in-memory table, or for further processing in a fault-tolerant manner to a File Sink in different formats.
noAggDF <- select(where(deviceDataStreamingDf, "signal > 10"), "device")
+
+# Print new data to console
+write.stream(noAggDF, "console")
+
+# Write new data to Parquet files
+write.stream(noAggDF,
+ "parquet",
+ path = "path/to/destination/dir",
+ checkpointLocation = "path/to/checkpoint/dir")
+
+# Aggregate
+aggDF <- count(groupBy(noAggDF, "device"))
+
+# Print updated aggregations to console
+write.stream(aggDF, "console", outputMode = "complete")
+
+# Have all the aggregates in an in memory table. The query name will be the table name
+write.stream(aggDF, "memory", queryName = "aggregates", outputMode = "complete")
+
+head(sql("select * from aggregates"))
+
+Advanced Topics
+
+
+SparkR Object Classes
+There are three main object classes in SparkR you may be working with.
+-
+
SparkDataFrame: the central component of SparkR. It is an S4 class representing distributed collection of data organized into named columns, which is conceptually equivalent to a table in a relational database or a data frame in R. It has two slotssdfandenv. +-
+
sdfstores a reference to the corresponding Spark Dataset in the Spark JVM backend.
+envsaves the meta-information of the object such asisCached.
+
It can be created by data import methods or by transforming an existing
SparkDataFrame. We can manipulateSparkDataFrameby numerous data processing functions and feed that into machine learning algorithms.
+
+Column: an S4 class representing a column ofSparkDataFrame. The slotjcsaves a reference to the correspondingColumnobject in the Spark JVM backend.It can be obtained from a
SparkDataFrameby$operator, e.g.,df$col. More often, it is used together with other functions, for example, withselectto select particular columns, withfilterand constructed conditions to select rows, with aggregation functions to compute aggregate statistics for each group.
+
+GroupedData: an S4 class representing grouped data created bygroupByor by transforming otherGroupedData. Itssgdslot saves a reference to aRelationalGroupedDatasetobject in the backend.This is often an intermediate object with group information and followed up by aggregation operations.
+
+
+Architecture
+A complete description of architecture can be seen in the references, in particular the paper SparkR: Scaling R Programs with Spark.
+Under the hood of SparkR is Spark SQL engine. This avoids the overheads of running interpreted R code, and the optimized SQL execution engine in Spark uses structural information about data and computation flow to perform a bunch of optimizations to speed up the computation.
+The main method calls of actual computation happen in the Spark JVM of the driver. We have a socket-based SparkR API that allows us to invoke functions on the JVM from R. We use a SparkR JVM backend that listens on a Netty-based socket server.
+Two kinds of RPCs are supported in the SparkR JVM backend: method invocation and creating new objects. Method invocation can be done in two ways.
+-
+
sparkR.callJMethodtakes a reference to an existing Java object and a list of arguments to be passed on to the method.
+sparkR.callJStatictakes a class name for static method and a list of arguments to be passed on to the method.
+
The arguments are serialized using our custom wire format which is then deserialized on the JVM side. We then use Java reflection to invoke the appropriate method.
+To create objects, sparkR.newJObject is used and then similarly the appropriate constructor is invoked with provided arguments.
Finally, we use a new R class jobj that refers to a Java object existing in the backend. These references are tracked on the Java side and are automatically garbage collected when they go out of scope on the R side.
+
+Appendix
+
+
+
+
+
+R and Spark Data Types
+| R | +Spark | +
|---|---|
| byte | +byte | +
| integer | +integer | +
| float | +float | +
| double | +double | +
| numeric | +double | +
| character | +string | +
| string | +string | +
| binary | +binary | +
| raw | +binary | +
| logical | +boolean | +
| POSIXct | +timestamp | +
| POSIXlt | +timestamp | +
| Date | +date | +
| array | +array | +
| list | +array | +
| env | +map | +
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/AnIndex b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/AnIndex
new file mode 100644
index 0000000..faeb2f6
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/AnIndex
@@ -0,0 +1,1099 @@
+!,Column-method not
+$ select
+$,SparkDataFrame-method select
+$<- select
+$<-,SparkDataFrame-method select
+%<=>% eq_null_safe
+%<=>%,Column-method eq_null_safe
+%in% match
+%in%,Column-method match
+abs column_math_functions
+abs,Column-method column_math_functions
+acos column_math_functions
+acos,Column-method column_math_functions
+acosh column_math_functions
+acosh,Column-method column_math_functions
+addJobTag addJobTAg
+add_months column_datetime_diff_functions
+add_months,Column,numeric-method column_datetime_diff_functions
+AFTSurvivalRegressionModel-class AFTSurvivalRegressionModel-class
+agg summarize
+agg,GroupedData-method summarize
+agg,SparkDataFrame-method summarize
+alias alias
+alias,Column-method alias
+alias,SparkDataFrame-method alias
+ALSModel-class ALSModel-class
+approxCountDistinct column_aggregate_functions
+approxCountDistinct,Column-method column_aggregate_functions
+approxQuantile approxQuantile
+approxQuantile,SparkDataFrame,character,numeric,numeric-method approxQuantile
+approx_count_distinct column_aggregate_functions
+approx_count_distinct,Column-method column_aggregate_functions
+arrange arrange
+arrange,SparkDataFrame,character-method arrange
+arrange,SparkDataFrame,Column-method arrange
+arrays_overlap column_collection_functions
+arrays_overlap,Column,Column-method column_collection_functions
+arrays_overlap,Column-method column_collection_functions
+arrays_zip column_collection_functions
+arrays_zip,Column-method column_collection_functions
+arrays_zip_with column_collection_functions
+arrays_zip_with,characterOrColumn,characterOrColumn,function-method column_collection_functions
+array_aggregate column_collection_functions
+array_aggregate,characterOrColumn,Column,function-method column_collection_functions
+array_contains column_collection_functions
+array_contains,Column-method column_collection_functions
+array_distinct column_collection_functions
+array_distinct,Column-method column_collection_functions
+array_except column_collection_functions
+array_except,Column,Column-method column_collection_functions
+array_except,Column-method column_collection_functions
+array_exists column_collection_functions
+array_exists,characterOrColumn,function-method column_collection_functions
+array_filter column_collection_functions
+array_filter,characterOrColumn,function-method column_collection_functions
+array_forall column_collection_functions
+array_forall,characterOrColumn,function-method column_collection_functions
+array_intersect column_collection_functions
+array_intersect,Column,Column-method column_collection_functions
+array_intersect,Column-method column_collection_functions
+array_join column_collection_functions
+array_join,Column,character-method column_collection_functions
+array_join,Column-method column_collection_functions
+array_max column_collection_functions
+array_max,Column-method column_collection_functions
+array_min column_collection_functions
+array_min,Column-method column_collection_functions
+array_position column_collection_functions
+array_position,Column-method column_collection_functions
+array_remove column_collection_functions
+array_remove,Column-method column_collection_functions
+array_repeat column_collection_functions
+array_repeat,Column,numericOrColumn-method column_collection_functions
+array_sort column_collection_functions
+array_sort,Column-method column_collection_functions
+array_to_vector column_ml_functions
+array_to_vector,Column-method column_ml_functions
+array_transform column_collection_functions
+array_transform,characterOrColumn,characterOrColumn,function-method column_collection_functions
+array_transform,characterOrColumn,function-method column_collection_functions
+array_union column_collection_functions
+array_union,Column,Column-method column_collection_functions
+array_union,Column-method column_collection_functions
+as.data.frame as.data.frame
+as.data.frame,SparkDataFrame-method as.data.frame
+as.DataFrame createDataFrame
+asc columnfunctions
+ascii column_string_functions
+ascii,Column-method column_string_functions
+asc_nulls_first columnfunctions
+asc_nulls_last columnfunctions
+asin column_math_functions
+asin,Column-method column_math_functions
+asinh column_math_functions
+asinh,Column-method column_math_functions
+assert_true column_misc_functions
+assert_true,Column-method column_misc_functions
+associationRules,FPGrowthModel-method spark.fpGrowth
+atan column_math_functions
+atan,Column-method column_math_functions
+atan2 column_math_functions
+atan2,Column-method column_math_functions
+atanh column_math_functions
+atanh,Column-method column_math_functions
+attach attach
+attach,SparkDataFrame-method attach
+avg avg
+avg,Column-method avg
+awaitTermination awaitTermination
+awaitTermination,StreamingQuery-method awaitTermination
+base64 column_string_functions
+base64,Column-method column_string_functions
+between between
+between,Column-method between
+bin column_math_functions
+bin,Column-method column_math_functions
+BisectingKMeansModel-class BisectingKMeansModel-class
+bitwiseNOT column_nonaggregate_functions
+bitwiseNOT,Column-method column_nonaggregate_functions
+bitwise_not column_nonaggregate_functions
+bitwise_not,Column-method column_nonaggregate_functions
+bit_length column_string_functions
+bit_length,Column-method column_string_functions
+broadcast broadcast
+broadcast,SparkDataFrame-method broadcast
+bround column_math_functions
+bround,Column-method column_math_functions
+cache cache
+cache,SparkDataFrame-method cache
+cacheTable cacheTable
+cancelJobGroup cancelJobGroup
+cancelJobsWithTag cancelJobsWithTag
+cast cast
+cast,Column-method cast
+cbrt column_math_functions
+cbrt,Column-method column_math_functions
+ceil column_math_functions
+ceil,Column-method column_math_functions
+ceiling column_math_functions
+ceiling,Column-method column_math_functions
+checkpoint checkpoint
+checkpoint,SparkDataFrame-method checkpoint
+clearCache clearCache
+clearJobGroup clearJobGroup
+clearJobTags clearJobTags
+coalesce coalesce
+coalesce,Column-method column_nonaggregate_functions
+coalesce,SparkDataFrame-method coalesce
+collect collect
+collect,SparkDataFrame-method collect
+collect_list column_aggregate_functions
+collect_list,Column-method column_aggregate_functions
+collect_set column_aggregate_functions
+collect_set,Column-method column_aggregate_functions
+colnames columns
+colnames,SparkDataFrame-method columns
+colnames<- columns
+colnames<-,SparkDataFrame-method columns
+coltypes coltypes
+coltypes,SparkDataFrame-method coltypes
+coltypes<- coltypes
+coltypes<-,SparkDataFrame,character-method coltypes
+column column
+column,character-method column
+column,jobj-method column
+Column-class column
+columnfunctions columnfunctions
+columns columns
+columns,SparkDataFrame-method columns
+column_aggregate_functions column_aggregate_functions
+column_avro_functions column_avro_functions
+column_collection_functions column_collection_functions
+column_datetime_diff_functions column_datetime_diff_functions
+column_datetime_functions column_datetime_functions
+column_math_functions column_math_functions
+column_misc_functions column_misc_functions
+column_ml_functions column_ml_functions
+column_nonaggregate_functions column_nonaggregate_functions
+column_string_functions column_string_functions
+column_window_functions column_window_functions
+concat column_collection_functions
+concat,Column-method column_collection_functions
+concat_ws column_string_functions
+concat_ws,character,Column-method column_string_functions
+contains columnfunctions
+conv column_math_functions
+conv,Column,numeric,numeric-method column_math_functions
+corr corr
+corr,Column-method corr
+corr,SparkDataFrame-method corr
+cos column_math_functions
+cos,Column-method column_math_functions
+cosh column_math_functions
+cosh,Column-method column_math_functions
+cot column_math_functions
+cot,Column-method column_math_functions
+count count
+count,Column-method count
+count,GroupedData-method count
+count,SparkDataFrame-method nrow
+countDistinct column_aggregate_functions
+countDistinct,Column-method column_aggregate_functions
+count_distinct column_aggregate_functions
+count_distinct,Column-method column_aggregate_functions
+cov cov
+cov,characterOrColumn-method cov
+cov,SparkDataFrame-method cov
+covar_pop cov
+covar_pop,characterOrColumn,characterOrColumn-method cov
+covar_samp cov
+covar_samp,characterOrColumn,characterOrColumn-method cov
+crc32 column_misc_functions
+crc32,Column-method column_misc_functions
+createDataFrame createDataFrame
+createExternalTable createExternalTable-deprecated
+createOrReplaceTempView createOrReplaceTempView
+createOrReplaceTempView,SparkDataFrame,character-method createOrReplaceTempView
+createTable createTable
+create_array column_nonaggregate_functions
+create_array,Column-method column_nonaggregate_functions
+create_lambda create_lambda
+create_map column_nonaggregate_functions
+create_map,Column-method column_nonaggregate_functions
+crossJoin crossJoin
+crossJoin,SparkDataFrame,SparkDataFrame-method crossJoin
+crosstab crosstab
+crosstab,SparkDataFrame,character,character-method crosstab
+csc column_math_functions
+csc,Column-method column_math_functions
+cube cube
+cube,SparkDataFrame-method cube
+cume_dist column_window_functions
+cume_dist,missing-method column_window_functions
+currentCatalog currentCatalog
+currentDatabase currentDatabase
+current_date column_datetime_functions
+current_date,missing-method column_datetime_functions
+current_timestamp column_datetime_functions
+current_timestamp,missing-method column_datetime_functions
+dapply dapply
+dapply,SparkDataFrame,function,characterOrstructType-method dapply
+dapplyCollect dapplyCollect
+dapplyCollect,SparkDataFrame,function-method dapplyCollect
+databaseExists databaseExists
+datediff column_datetime_diff_functions
+datediff,Column-method column_datetime_diff_functions
+date_add column_datetime_diff_functions
+date_add,Column,numeric-method column_datetime_diff_functions
+date_format column_datetime_diff_functions
+date_format,Column,character-method column_datetime_diff_functions
+date_sub column_datetime_diff_functions
+date_sub,Column,numeric-method column_datetime_diff_functions
+date_trunc column_datetime_functions
+date_trunc,character,Column-method column_datetime_functions
+dayofmonth column_datetime_functions
+dayofmonth,Column-method column_datetime_functions
+dayofweek column_datetime_functions
+dayofweek,Column-method column_datetime_functions
+dayofyear column_datetime_functions
+dayofyear,Column-method column_datetime_functions
+DecisionTreeClassificationModel-class DecisionTreeClassificationModel-class
+DecisionTreeRegressionModel-class DecisionTreeRegressionModel-class
+decode column_string_functions
+decode,Column,character-method column_string_functions
+degrees column_math_functions
+degrees,Column-method column_math_functions
+dense_rank column_window_functions
+dense_rank,missing-method column_window_functions
+desc columnfunctions
+describe describe
+describe,SparkDataFrame,ANY-method describe
+describe,SparkDataFrame,character-method describe
+describe,SparkDataFrame-method describe
+desc_nulls_first columnfunctions
+desc_nulls_last columnfunctions
+dim dim
+dim,SparkDataFrame-method dim
+distinct distinct
+distinct,SparkDataFrame-method distinct
+drop drop
+drop,ANY,ANY-method drop
+drop,SparkDataFrame,characterOrColumn-method drop
+dropDuplicates dropDuplicates
+dropDuplicates,SparkDataFrame-method dropDuplicates
+dropFields dropFields
+dropFields,Column-method dropFields
+dropna nafunctions
+dropna,SparkDataFrame-method nafunctions
+dropTempTable dropTempTable-deprecated
+dropTempView dropTempView
+dtypes dtypes
+dtypes,SparkDataFrame-method dtypes
+element_at column_collection_functions
+element_at,Column-method column_collection_functions
+encode column_string_functions
+encode,Column,character-method column_string_functions
+endsWith endsWith
+endsWith,Column-method endsWith
+except except
+except,SparkDataFrame,SparkDataFrame-method except
+exceptAll exceptAll
+exceptAll,SparkDataFrame,SparkDataFrame-method exceptAll
+exp column_math_functions
+exp,Column-method column_math_functions
+explain explain
+explain,SparkDataFrame-method explain
+explain,StreamingQuery-method explain
+explode column_collection_functions
+explode,Column-method column_collection_functions
+explode_outer column_collection_functions
+explode_outer,Column-method column_collection_functions
+expm1 column_math_functions
+expm1,Column-method column_math_functions
+expr column_nonaggregate_functions
+expr,character-method column_nonaggregate_functions
+factorial column_math_functions
+factorial,Column-method column_math_functions
+fillna nafunctions
+fillna,SparkDataFrame-method nafunctions
+filter filter
+filter,SparkDataFrame,characterOrColumn-method filter
+findFrequentSequentialPatterns,PrefixSpan,SparkDataFrame-method spark.prefixSpan
+first first
+first,characterOrColumn-method first
+first,SparkDataFrame-method first
+fitted fitted
+fitted,BisectingKMeansModel-method spark.bisectingKmeans
+fitted,KMeansModel-method fitted
+flatten column_collection_functions
+flatten,Column-method column_collection_functions
+floor column_math_functions
+floor,Column-method column_math_functions
+FMClassificationModel-class FMClassificationModel-class
+FMRegressionModel-class FMRegressionModel-class
+format_number column_string_functions
+format_number,Column,numeric-method column_string_functions
+format_string column_string_functions
+format_string,character,Column-method column_string_functions
+FPGrowthModel-class FPGrowthModel-class
+freqItems freqItems
+freqItems,SparkDataFrame,character-method freqItems
+freqItemsets,FPGrowthModel-method spark.fpGrowth
+from_avro column_avro_functions
+from_avro,characterOrColumn-method column_avro_functions
+from_avro,Column-method column_avro_functions
+from_csv column_collection_functions
+from_csv,Column,characterOrstructTypeOrColumn-method column_collection_functions
+from_json column_collection_functions
+from_json,Column,characterOrstructTypeOrColumn-method column_collection_functions
+from_unixtime column_datetime_functions
+from_unixtime,Column-method column_datetime_functions
+from_utc_timestamp column_datetime_diff_functions
+from_utc_timestamp,Column,character-method column_datetime_diff_functions
+functionExists functionExists
+gapply gapply
+gapply,GroupedData-method gapply
+gapply,SparkDataFrame-method gapply
+gapplyCollect gapplyCollect
+gapplyCollect,GroupedData-method gapplyCollect
+gapplyCollect,SparkDataFrame-method gapplyCollect
+GaussianMixtureModel-class GaussianMixtureModel-class
+GBTClassificationModel-class GBTClassificationModel-class
+GBTRegressionModel-class GBTRegressionModel-class
+GeneralizedLinearRegressionModel-class GeneralizedLinearRegressionModel-class
+getDatabase getDatabase
+getField columnfunctions
+getFunc getFunc
+getItem columnfunctions
+getJobTags getJobTags
+getLocalProperty getLocalProperty
+getNumPartitions getNumPartitions
+getNumPartitions,SparkDataFrame-method getNumPartitions
+getTable getTable
+glm glm
+glm,formula,ANY,SparkDataFrame-method glm
+greatest column_nonaggregate_functions
+greatest,Column-method column_nonaggregate_functions
+groupBy groupBy
+groupBy,SparkDataFrame-method groupBy
+groupedData GroupedData
+GroupedData-class GroupedData
+grouping_bit column_aggregate_functions
+grouping_bit,Column-method column_aggregate_functions
+grouping_id column_aggregate_functions
+grouping_id,Column-method column_aggregate_functions
+group_by groupBy
+group_by,SparkDataFrame-method groupBy
+hash column_misc_functions
+hash,Column-method column_misc_functions
+hashCode hashCode
+head head
+head,SparkDataFrame-method head
+hex column_math_functions
+hex,Column-method column_math_functions
+hint hint
+hint,SparkDataFrame,character-method hint
+histogram histogram
+histogram,SparkDataFrame,characterOrColumn-method histogram
+hour column_datetime_functions
+hour,Column-method column_datetime_functions
+hypot column_math_functions
+hypot,Column-method column_math_functions
+ifelse column_nonaggregate_functions
+ifelse,Column-method column_nonaggregate_functions
+ilike columnfunctions
+initcap column_string_functions
+initcap,Column-method column_string_functions
+input_file_name column_nonaggregate_functions
+input_file_name,missing-method column_nonaggregate_functions
+insertInto insertInto
+insertInto,SparkDataFrame,character-method insertInto
+install.spark install.spark
+instr column_string_functions
+instr,Column,character-method column_string_functions
+intersect intersect
+intersect,SparkDataFrame,SparkDataFrame-method intersect
+intersectAll intersectAll
+intersectAll,SparkDataFrame,SparkDataFrame-method intersectAll
+invoke_higher_order_function invoke_higher_order_function
+is.nan column_nonaggregate_functions
+is.nan,Column-method column_nonaggregate_functions
+isActive isActive
+isActive,StreamingQuery-method isActive
+isLocal isLocal
+isLocal,SparkDataFrame-method isLocal
+isNaN columnfunctions
+isnan column_nonaggregate_functions
+isnan,Column-method column_nonaggregate_functions
+isNotNull columnfunctions
+isNull columnfunctions
+IsotonicRegressionModel-class IsotonicRegressionModel-class
+isStreaming isStreaming
+isStreaming,SparkDataFrame-method isStreaming
+join join
+join,SparkDataFrame,SparkDataFrame-method join
+KMeansModel-class KMeansModel-class
+KSTest-class KSTest-class
+kurtosis column_aggregate_functions
+kurtosis,Column-method column_aggregate_functions
+lag column_window_functions
+lag,characterOrColumn-method column_window_functions
+last last
+last,characterOrColumn-method last
+lastProgress lastProgress
+lastProgress,StreamingQuery-method lastProgress
+last_day column_datetime_functions
+last_day,Column-method column_datetime_functions
+LDAModel-class LDAModel-class
+lead column_window_functions
+lead,characterOrColumn,numeric-method column_window_functions
+least column_nonaggregate_functions
+least,Column-method column_nonaggregate_functions
+length column_string_functions
+length,Column-method column_string_functions
+levenshtein column_string_functions
+levenshtein,Column-method column_string_functions
+like columnfunctions
+limit limit
+limit,SparkDataFrame,numeric-method limit
+LinearRegressionModel-class LinearRegressionModel-class
+LinearSVCModel-class LinearSVCModel-class
+listCatalogs listCatalogs
+listColumns listColumns
+listDatabases listDatabases
+listFunctions listFunctions
+listTables listTables
+lit column_nonaggregate_functions
+lit,ANY-method column_nonaggregate_functions
+ln column_math_functions
+ln,Column-method column_math_functions
+loadDF read.df
+localCheckpoint localCheckpoint
+localCheckpoint,SparkDataFrame-method localCheckpoint
+locate column_string_functions
+locate,character,Column-method column_string_functions
+log column_math_functions
+log,Column-method column_math_functions
+log10 column_math_functions
+log10,Column-method column_math_functions
+log1p column_math_functions
+log1p,Column-method column_math_functions
+log2 column_math_functions
+log2,Column-method column_math_functions
+LogisticRegressionModel-class LogisticRegressionModel-class
+lower column_string_functions
+lower,Column-method column_string_functions
+lpad column_string_functions
+lpad,Column,numeric,character-method column_string_functions
+ltrim column_string_functions
+ltrim,Column,character-method column_string_functions
+ltrim,Column,missing-method column_string_functions
+make_date column_datetime_functions
+make_date,Column,Column,Column-method column_datetime_functions
+make_date,Column-method column_datetime_functions
+map_concat column_collection_functions
+map_concat,Column-method column_collection_functions
+map_entries column_collection_functions
+map_entries,Column-method column_collection_functions
+map_filter column_collection_functions
+map_filter,characterOrColumn,function-method column_collection_functions
+map_from_arrays column_collection_functions
+map_from_arrays,Column,Column-method column_collection_functions
+map_from_arrays,Column-method column_collection_functions
+map_from_entries column_collection_functions
+map_from_entries,Column-method column_collection_functions
+map_keys column_collection_functions
+map_keys,Column-method column_collection_functions
+map_values column_collection_functions
+map_values,Column-method column_collection_functions
+map_zip_with column_collection_functions
+map_zip_with,characterOrColumn,characterOrColumn,function-method column_collection_functions
+max column_aggregate_functions
+max,Column-method column_aggregate_functions
+max_by column_aggregate_functions
+max_by,Column,Column-method column_aggregate_functions
+max_by,Column-method column_aggregate_functions
+md5 column_misc_functions
+md5,Column-method column_misc_functions
+mean column_aggregate_functions
+mean,Column-method column_aggregate_functions
+melt unpivot
+melt,SparkDataFrame,ANY,ANY,character,character-method unpivot
+merge merge
+merge,SparkDataFrame,SparkDataFrame-method merge
+min column_aggregate_functions
+min,Column-method column_aggregate_functions
+minute column_datetime_functions
+minute,Column-method column_datetime_functions
+min_by column_aggregate_functions
+min_by,Column,Column-method column_aggregate_functions
+min_by,Column-method column_aggregate_functions
+monotonically_increasing_id column_nonaggregate_functions
+monotonically_increasing_id,missing-method column_nonaggregate_functions
+month column_datetime_functions
+month,Column-method column_datetime_functions
+months_between column_datetime_diff_functions
+months_between,Column-method column_datetime_diff_functions
+MultilayerPerceptronClassificationModel-class MultilayerPerceptronClassificationModel-class
+mutate mutate
+mutate,SparkDataFrame-method mutate
+n count
+n,Column-method count
+na.omit nafunctions
+na.omit,SparkDataFrame-method nafunctions
+NaiveBayesModel-class NaiveBayesModel-class
+names columns
+names,SparkDataFrame-method columns
+names<- columns
+names<-,SparkDataFrame-method columns
+nanvl column_nonaggregate_functions
+nanvl,Column-method column_nonaggregate_functions
+ncol ncol
+ncol,SparkDataFrame-method ncol
+negate column_nonaggregate_functions
+negate,Column-method column_nonaggregate_functions
+negative column_nonaggregate_functions
+negative,Column-method column_nonaggregate_functions
+next_day column_datetime_diff_functions
+next_day,Column,character-method column_datetime_diff_functions
+not not
+not,Column-method not
+nrow nrow
+nrow,SparkDataFrame-method nrow
+nth_value column_window_functions
+nth_value,characterOrColumn,numeric-method column_window_functions
+nth_value,characterOrColumn-method column_window_functions
+ntile column_window_functions
+ntile,numeric-method column_window_functions
+n_distinct column_aggregate_functions
+n_distinct,Column-method column_aggregate_functions
+octet_length column_string_functions
+octet_length,Column-method column_string_functions
+orderBy orderBy
+orderBy,SparkDataFrame,characterOrColumn-method arrange
+orderBy,WindowSpec,character-method orderBy
+orderBy,WindowSpec,Column-method orderBy
+otherwise otherwise
+otherwise,Column-method otherwise
+over over
+over,Column,WindowSpec-method over
+overlay column_string_functions
+overlay,Column,Column,numericOrColumn-method column_string_functions
+overlay,Column-method,numericOrColumn-method column_string_functions
+partitionBy partitionBy
+partitionBy,WindowSpec-method partitionBy
+percentile_approx column_aggregate_functions
+percentile_approx,characterOrColumn,numericOrColumn-method column_aggregate_functions
+percentile_approx,Column-method column_aggregate_functions
+percent_rank column_window_functions
+percent_rank,missing-method column_window_functions
+persist persist
+persist,SparkDataFrame,character-method persist
+pivot pivot
+pivot,GroupedData,character-method pivot
+pmod column_math_functions
+pmod,Column-method column_math_functions
+posexplode column_collection_functions
+posexplode,Column-method column_collection_functions
+posexplode_outer column_collection_functions
+posexplode_outer,Column-method column_collection_functions
+positive column_nonaggregate_functions
+positive,Column-method column_nonaggregate_functions
+PowerIterationClustering-class PowerIterationClustering-class
+predict predict
+predict,AFTSurvivalRegressionModel-method spark.survreg
+predict,ALSModel-method spark.als
+predict,BisectingKMeansModel-method spark.bisectingKmeans
+predict,DecisionTreeClassificationModel-method spark.decisionTree
+predict,DecisionTreeRegressionModel-method spark.decisionTree
+predict,FMClassificationModel,SparkDataFrame-method spark.fmClassifier
+predict,FMClassificationModel-method spark.fmClassifier
+predict,FMRegressionModel,SparkDataFrame-method spark.fmRegressor
+predict,FMRegressionModel-method spark.fmRegressor
+predict,FPGrowthModel-method spark.fpGrowth
+predict,GaussianMixtureModel,SparkDataFrame-method spark.gaussianMixture
+predict,GaussianMixtureModel-method spark.gaussianMixture
+predict,GBTClassificationModel-method spark.gbt
+predict,GBTRegressionModel-method spark.gbt
+predict,GeneralizedLinearRegressionModel-method spark.glm
+predict,IsotonicRegressionModel,SparkDataFrame-method spark.isoreg
+predict,IsotonicRegressionModel-method spark.isoreg
+predict,KMeansModel-method spark.kmeans
+predict,LinearRegressionModel,SparkDataFrame-method spark.lm
+predict,LinearRegressionModel-method spark.lm
+predict,LinearSVCModel,SparkDataFrame-method spark.svmLinear
+predict,LinearSVCModel-method spark.svmLinear
+predict,LogisticRegressionModel,SparkDataFrame-method spark.logit
+predict,LogisticRegressionModel-method spark.logit
+predict,MultilayerPerceptronClassificationModel-method spark.mlp
+predict,NaiveBayesModel-method spark.naiveBayes
+predict,RandomForestClassificationModel-method spark.randomForest
+predict,RandomForestRegressionModel-method spark.randomForest
+PrefixSpan-class PrefixSpan-class
+print.jobj print.jobj
+print.structField print.structField
+print.structType print.structType
+print.summary.DecisionTreeClassificationModel spark.decisionTree
+print.summary.DecisionTreeRegressionModel spark.decisionTree
+print.summary.GBTClassificationModel spark.gbt
+print.summary.GBTRegressionModel spark.gbt
+print.summary.GeneralizedLinearRegressionModel spark.glm
+print.summary.KSTest spark.kstest
+print.summary.RandomForestClassificationModel spark.randomForest
+print.summary.RandomForestRegressionModel spark.randomForest
+printSchema printSchema
+printSchema,SparkDataFrame-method printSchema
+product column_aggregate_functions
+product,Column-method column_aggregate_functions
+quarter column_datetime_functions
+quarter,Column-method column_datetime_functions
+queryName queryName
+queryName,StreamingQuery-method queryName
+radians column_math_functions
+radians,Column-method column_math_functions
+raise_error column_misc_functions
+raise_error,characterOrColumn-method column_misc_functions
+rand column_nonaggregate_functions
+rand,missing-method column_nonaggregate_functions
+rand,numeric-method column_nonaggregate_functions
+randn column_nonaggregate_functions
+randn,missing-method column_nonaggregate_functions
+randn,numeric-method column_nonaggregate_functions
+RandomForestClassificationModel-class RandomForestClassificationModel-class
+RandomForestRegressionModel-class RandomForestRegressionModel-class
+randomSplit randomSplit
+randomSplit,SparkDataFrame,numeric-method randomSplit
+rangeBetween rangeBetween
+rangeBetween,WindowSpec,numeric,numeric-method rangeBetween
+rank column_window_functions
+rank,ANY-method column_window_functions
+rank,missing-method column_window_functions
+rbind rbind
+rbind,SparkDataFrame-method rbind
+read.df read.df
+read.jdbc read.jdbc
+read.json read.json
+read.ml read.ml
+read.orc read.orc
+read.parquet read.parquet
+read.stream read.stream
+read.text read.text
+recoverPartitions recoverPartitions
+refreshByPath refreshByPath
+refreshTable refreshTable
+regexp_extract column_string_functions
+regexp_extract,Column,character,numeric-method column_string_functions
+regexp_replace column_string_functions
+regexp_replace,Column,character,character-method column_string_functions
+registerTempTable registerTempTable-deprecated
+registerTempTable,SparkDataFrame,character-method registerTempTable-deprecated
+removeJobTag removeJobTAg
+rename rename
+rename,SparkDataFrame-method rename
+repartition repartition
+repartition,SparkDataFrame-method repartition
+repartitionByRange repartitionByRange
+repartitionByRange,SparkDataFrame-method repartitionByRange
+repeat_string column_string_functions
+repeat_string,Column,numeric-method column_string_functions
+repeat_string,Column-method column_string_functions
+reverse column_collection_functions
+reverse,Column-method column_collection_functions
+rint column_math_functions
+rint,Column-method column_math_functions
+rlike columnfunctions
+rollup rollup
+rollup,SparkDataFrame-method rollup
+round column_math_functions
+round,Column-method column_math_functions
+rowsBetween rowsBetween
+rowsBetween,WindowSpec,numeric,numeric-method rowsBetween
+row_number column_window_functions
+row_number,missing-method column_window_functions
+rpad column_string_functions
+rpad,Column,numeric,character-method column_string_functions
+rtrim column_string_functions
+rtrim,Column,character-method column_string_functions
+rtrim,Column,missing-method column_string_functions
+sample sample
+sample,SparkDataFrame-method sample
+sampleBy sampleBy
+sampleBy,SparkDataFrame,character,list,numeric-method sampleBy
+sample_frac sample
+sample_frac,SparkDataFrame-method sample
+saveAsTable saveAsTable
+saveAsTable,SparkDataFrame,character-method saveAsTable
+saveDF write.df
+saveDF,SparkDataFrame,character-method write.df
+schema schema
+schema,SparkDataFrame-method schema
+schema_of_csv column_collection_functions
+schema_of_csv,characterOrColumn-method column_collection_functions
+schema_of_json column_collection_functions
+schema_of_json,characterOrColumn-method column_collection_functions
+sd column_aggregate_functions
+sd,Column-method column_aggregate_functions
+sec column_math_functions
+sec,Column-method column_math_functions
+second column_datetime_functions
+second,Column-method column_datetime_functions
+select select
+select,SparkDataFrame,character-method select
+select,SparkDataFrame,Column-method select
+select,SparkDataFrame,list-method select
+selectExpr selectExpr
+selectExpr,SparkDataFrame,character-method selectExpr
+setCheckpointDir setCheckpointDir
+setCurrentCatalog setCurrentCatalog
+setCurrentDatabase setCurrentDatabase
+setInterruptOnCancel setInterruptOnCancel
+setJobDescription setJobDescription
+setJobGroup setJobGroup
+setLocalProperty setLocalProperty
+setLogLevel setLogLevel
+sha1 column_misc_functions
+sha1,Column-method column_misc_functions
+sha2 column_misc_functions
+sha2,Column,numeric-method column_misc_functions
+shiftLeft column_math_functions
+shiftleft column_math_functions
+shiftLeft,Column,numeric-method column_math_functions
+shiftleft,Column,numeric-method column_math_functions
+shiftRight column_math_functions
+shiftright column_math_functions
+shiftRight,Column,numeric-method column_math_functions
+shiftright,Column,numeric-method column_math_functions
+shiftRightUnsigned column_math_functions
+shiftrightunsigned column_math_functions
+shiftRightUnsigned,Column,numeric-method column_math_functions
+shiftrightunsigned,Column,numeric-method column_math_functions
+show show
+show,Column-method show
+show,GroupedData-method show
+show,SparkDataFrame-method show
+show,StreamingQuery-method show
+show,WindowSpec-method show
+showDF showDF
+showDF,SparkDataFrame-method showDF
+shuffle column_collection_functions
+shuffle,Column-method column_collection_functions
+sign column_math_functions
+sign,Column-method column_math_functions
+signum column_math_functions
+signum,Column-method column_math_functions
+sin column_math_functions
+sin,Column-method column_math_functions
+sinh column_math_functions
+sinh,Column-method column_math_functions
+size column_collection_functions
+size,Column-method column_collection_functions
+skewness column_aggregate_functions
+skewness,Column-method column_aggregate_functions
+slice column_collection_functions
+slice,Column-method column_collection_functions
+sort_array column_collection_functions
+sort_array,Column-method column_collection_functions
+soundex column_string_functions
+soundex,Column-method column_string_functions
+spark.addFile spark.addFile
+spark.als spark.als
+spark.als,SparkDataFrame-method spark.als
+spark.assignClusters spark.powerIterationClustering
+spark.assignClusters,SparkDataFrame-method spark.powerIterationClustering
+spark.associationRules spark.fpGrowth
+spark.associationRules,FPGrowthModel-method spark.fpGrowth
+spark.bisectingKmeans spark.bisectingKmeans
+spark.bisectingKmeans,SparkDataFrame,formula-method spark.bisectingKmeans
+spark.decisionTree spark.decisionTree
+spark.decisionTree,SparkDataFrame,formula-method spark.decisionTree
+spark.findFrequentSequentialPatterns spark.prefixSpan
+spark.findFrequentSequentialPatterns,SparkDataFrame-method spark.prefixSpan
+spark.fmClassifier spark.fmClassifier
+spark.fmClassifier,SparkDataFrame,formula-method spark.fmClassifier
+spark.fmRegressor spark.fmRegressor
+spark.fmRegressor,SparkDataFrame,formula-method spark.fmRegressor
+spark.fpGrowth spark.fpGrowth
+spark.fpGrowth,SparkDataFrame-method spark.fpGrowth
+spark.freqItemsets spark.fpGrowth
+spark.freqItemsets,FPGrowthModel-method spark.fpGrowth
+spark.gaussianMixture spark.gaussianMixture
+spark.gaussianMixture,SparkDataFrame,formula-method spark.gaussianMixture
+spark.gbt spark.gbt
+spark.gbt,SparkDataFrame,formula-method spark.gbt
+spark.getSparkFiles spark.getSparkFiles
+spark.getSparkFilesRootDirectory spark.getSparkFilesRootDirectory
+spark.glm spark.glm
+spark.glm,SparkDataFrame,formula-method spark.glm
+spark.isoreg spark.isoreg
+spark.isoreg,SparkDataFrame,formula-method spark.isoreg
+spark.kmeans spark.kmeans
+spark.kmeans,SparkDataFrame,formula-method spark.kmeans
+spark.kstest spark.kstest
+spark.kstest,SparkDataFrame-method spark.kstest
+spark.lapply spark.lapply
+spark.lda spark.lda
+spark.lda,SparkDataFrame-method spark.lda
+spark.lm spark.lm
+spark.lm,SparkDataFrame,formula-method spark.lm
+spark.logit spark.logit
+spark.logit,SparkDataFrame,formula-method spark.logit
+spark.mlp spark.mlp
+spark.mlp,SparkDataFrame,formula-method spark.mlp
+spark.naiveBayes spark.naiveBayes
+spark.naiveBayes,SparkDataFrame,formula-method spark.naiveBayes
+spark.perplexity spark.lda
+spark.perplexity,LDAModel,SparkDataFrame-method spark.lda
+spark.perplexity,LDAModel-method spark.lda
+spark.posterior spark.lda
+spark.posterior,LDAModel,SparkDataFrame-method spark.lda
+spark.randomForest spark.randomForest
+spark.randomForest,SparkDataFrame,formula-method spark.randomForest
+spark.survreg spark.survreg
+spark.survreg,SparkDataFrame,formula-method spark.survreg
+spark.svmLinear spark.svmLinear
+spark.svmLinear,SparkDataFrame,formula-method spark.svmLinear
+SparkDataFrame-class SparkDataFrame
+sparkR.callJMethod sparkR.callJMethod
+sparkR.callJStatic sparkR.callJStatic
+sparkR.conf sparkR.conf
+sparkR.init sparkR.init-deprecated
+sparkR.newJObject sparkR.newJObject
+sparkR.session sparkR.session
+sparkR.session.stop sparkR.session.stop
+sparkR.stop sparkR.session.stop
+sparkR.uiWebUrl sparkR.uiWebUrl
+sparkR.version sparkR.version
+sparkRHive.init sparkRHive.init-deprecated
+sparkRSQL.init sparkRSQL.init-deprecated
+spark_partition_id column_nonaggregate_functions
+spark_partition_id,missing-method column_nonaggregate_functions
+split_string column_string_functions
+split_string,Column,character-method column_string_functions
+split_string,Column-method column_string_functions
+sql sql
+sqrt column_math_functions
+sqrt,Column-method column_math_functions
+startsWith startsWith
+startsWith,Column-method startsWith
+status status
+status,StreamingQuery-method status
+std column_aggregate_functions
+std,Column-method column_aggregate_functions
+stddev column_aggregate_functions
+stddev,Column-method column_aggregate_functions
+stddev_pop column_aggregate_functions
+stddev_pop,Column-method column_aggregate_functions
+stddev_samp column_aggregate_functions
+stddev_samp,Column-method column_aggregate_functions
+stopQuery stopQuery
+stopQuery,StreamingQuery-method stopQuery
+storageLevel storageLevel
+storageLevel,SparkDataFrame-method storageLevel
+str str
+str,SparkDataFrame-method str
+StreamingQuery-class StreamingQuery
+struct column_nonaggregate_functions
+struct,characterOrColumn-method column_nonaggregate_functions
+structField structField
+structField.character structField
+structField.jobj structField
+structType structType
+structType.character structType
+structType.jobj structType
+structType.structField structType
+subset subset
+subset,SparkDataFrame-method subset
+substr substr
+substr,Column-method substr
+substring_index column_string_functions
+substring_index,Column,character,numeric-method column_string_functions
+sum column_aggregate_functions
+sum,Column-method column_aggregate_functions
+sumDistinct column_aggregate_functions
+sumDistinct,Column-method column_aggregate_functions
+summarize summarize
+summarize,GroupedData-method summarize
+summarize,SparkDataFrame-method summarize
+summary summary
+summary,AFTSurvivalRegressionModel-method spark.survreg
+summary,ALSModel-method spark.als
+summary,BisectingKMeansModel-method spark.bisectingKmeans
+summary,DecisionTreeClassificationModel-method spark.decisionTree
+summary,DecisionTreeRegressionModel-method spark.decisionTree
+summary,FMClassificationModel-method spark.fmClassifier
+summary,FMRegressionModel-method spark.fmRegressor
+summary,GaussianMixtureModel-method spark.gaussianMixture
+summary,GBTClassificationModel-method spark.gbt
+summary,GBTRegressionModel-method spark.gbt
+summary,GeneralizedLinearRegressionModel-method spark.glm
+summary,IsotonicRegressionModel-method spark.isoreg
+summary,KMeansModel-method spark.kmeans
+summary,KSTest-method spark.kstest
+summary,LDAModel-method spark.lda
+summary,LinearRegressionModel-method spark.lm
+summary,LinearSVCModel-method spark.svmLinear
+summary,LogisticRegressionModel-method spark.logit
+summary,MultilayerPerceptronClassificationModel-method spark.mlp
+summary,NaiveBayesModel-method spark.naiveBayes
+summary,RandomForestClassificationModel-method spark.randomForest
+summary,RandomForestRegressionModel-method spark.randomForest
+summary,SparkDataFrame-method summary
+sum_distinct column_aggregate_functions
+sum_distinct,Column-method column_aggregate_functions
+tableExists tableExists
+tableNames tableNames
+tables tables
+tableToDF tableToDF
+take take
+take,SparkDataFrame,numeric-method take
+tan column_math_functions
+tan,Column-method column_math_functions
+tanh column_math_functions
+tanh,Column-method column_math_functions
+timestamp_seconds column_datetime_functions
+timestamp_seconds,Column-method column_datetime_functions
+toDegrees column_math_functions
+toDegrees,Column-method column_math_functions
+toJSON toJSON
+toJSON,SparkDataFrame-method toJSON
+toRadians column_math_functions
+toRadians,Column-method column_math_functions
+to_avro column_avro_functions
+to_avro,characterOrColumn-method column_avro_functions
+to_avro,Column-method column_avro_functions
+to_csv column_collection_functions
+to_csv,Column-method column_collection_functions
+to_date column_datetime_functions
+to_date,Column,character-method column_datetime_functions
+to_date,Column,missing-method column_datetime_functions
+to_json column_collection_functions
+to_json,Column-method column_collection_functions
+to_timestamp column_datetime_functions
+to_timestamp,Column,character-method column_datetime_functions
+to_timestamp,Column,missing-method column_datetime_functions
+to_utc_timestamp column_datetime_diff_functions
+to_utc_timestamp,Column,character-method column_datetime_diff_functions
+transform mutate
+transform,SparkDataFrame-method mutate
+transform_keys column_collection_functions
+transform_keys,characterOrColumn,function-method column_collection_functions
+transform_values column_collection_functions
+transform_values,characterOrColumn,function-method column_collection_functions
+translate column_string_functions
+translate,Column,character,character-method column_string_functions
+trim column_string_functions
+trim,Column,character-method column_string_functions
+trim,Column,missing-method column_string_functions
+trunc column_datetime_functions
+trunc,Column-method column_datetime_functions
+unbase64 column_string_functions
+unbase64,Column-method column_string_functions
+uncacheTable uncacheTable
+unhex column_math_functions
+unhex,Column-method column_math_functions
+union union
+union,SparkDataFrame,SparkDataFrame-method union
+unionAll unionAll
+unionAll,SparkDataFrame,SparkDataFrame-method unionAll
+unionByName unionByName
+unionByName,SparkDataFrame,SparkDataFrame-method unionByName
+unique distinct
+unique,SparkDataFrame-method distinct
+unix_timestamp column_datetime_functions
+unix_timestamp,Column,character-method column_datetime_functions
+unix_timestamp,Column,missing-method column_datetime_functions
+unix_timestamp,missing,missing-method column_datetime_functions
+unpersist unpersist
+unpersist,SparkDataFrame-method unpersist
+unpivot unpivot
+unpivot,SparkDataFrame,ANY,ANY,character,character-method unpivot
+unresolved_named_lambda_var unresolved_named_lambda_var
+upper column_string_functions
+upper,Column-method column_string_functions
+var column_aggregate_functions
+var,Column-method column_aggregate_functions
+variance column_aggregate_functions
+variance,Column-method column_aggregate_functions
+var_pop column_aggregate_functions
+var_pop,Column-method column_aggregate_functions
+var_samp column_aggregate_functions
+var_samp,Column-method column_aggregate_functions
+vector_to_array column_ml_functions
+vector_to_array,Column-method column_ml_functions
+weekofyear column_datetime_functions
+weekofyear,Column-method column_datetime_functions
+when column_nonaggregate_functions
+when,Column-method column_nonaggregate_functions
+where filter
+where,SparkDataFrame,characterOrColumn-method filter
+width_bucket column_math_functions
+width_bucket,Column,Column,Column,Column-method column_math_functions
+width_bucket,Column-method column_math_functions
+window column_datetime_functions
+window,Column-method column_datetime_functions
+windowOrderBy windowOrderBy
+windowOrderBy,character-method windowOrderBy
+windowOrderBy,Column-method windowOrderBy
+windowPartitionBy windowPartitionBy
+windowPartitionBy,character-method windowPartitionBy
+windowPartitionBy,Column-method windowPartitionBy
+WindowSpec-class WindowSpec
+with with
+with,SparkDataFrame-method with
+withColumn withColumn
+withColumn,SparkDataFrame,character-method withColumn
+withColumnRenamed rename
+withColumnRenamed,SparkDataFrame,character,character-method rename
+withField withField
+withField,Column,character,Column-method withField
+withField,Column-method withField
+withWatermark withWatermark
+withWatermark,SparkDataFrame,character,character-method withWatermark
+write.df write.df
+write.df,SparkDataFrame-method write.df
+write.jdbc write.jdbc
+write.jdbc,SparkDataFrame,character,character-method write.jdbc
+write.json write.json
+write.json,SparkDataFrame,character-method write.json
+write.ml write.ml
+write.ml,AFTSurvivalRegressionModel,character-method spark.survreg
+write.ml,ALSModel,character-method spark.als
+write.ml,BisectingKMeansModel,character-method spark.bisectingKmeans
+write.ml,DecisionTreeClassificationModel,character-method spark.decisionTree
+write.ml,DecisionTreeRegressionModel,character-method spark.decisionTree
+write.ml,FMClassificationModel,character-method spark.fmClassifier
+write.ml,FMRegressionModel,character-method spark.fmRegressor
+write.ml,FPGrowthModel,character-method spark.fpGrowth
+write.ml,GaussianMixtureModel,character-method spark.gaussianMixture
+write.ml,GBTClassificationModel,character-method spark.gbt
+write.ml,GBTRegressionModel,character-method spark.gbt
+write.ml,GeneralizedLinearRegressionModel,character-method spark.glm
+write.ml,IsotonicRegressionModel,character-method spark.isoreg
+write.ml,KMeansModel,character-method spark.kmeans
+write.ml,LDAModel,character-method spark.lda
+write.ml,LinearRegressionModel,character-method spark.lm
+write.ml,LinearSVCModel,character-method spark.svmLinear
+write.ml,LogisticRegressionModel,character-method spark.logit
+write.ml,MultilayerPerceptronClassificationModel,character-method spark.mlp
+write.ml,NaiveBayesModel,character-method spark.naiveBayes
+write.ml,RandomForestClassificationModel,character-method spark.randomForest
+write.ml,RandomForestRegressionModel,character-method spark.randomForest
+write.orc write.orc
+write.orc,SparkDataFrame,character-method write.orc
+write.parquet write.parquet
+write.parquet,SparkDataFrame,character-method write.parquet
+write.stream write.stream
+write.stream,SparkDataFrame-method write.stream
+write.text write.text
+write.text,SparkDataFrame,character-method write.text
+xxhash64 column_misc_functions
+xxhash64,Column-method column_misc_functions
+year column_datetime_functions
+year,Column-method column_datetime_functions
+[ subset
+[,SparkDataFrame-method subset
+[[ subset
+[[,SparkDataFrame,numericOrcharacter-method subset
+[[<- subset
+[[<-,SparkDataFrame,numericOrcharacter-method subset
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdb b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdb
new file mode 100644
index 0000000..70c15f2
Binary files /dev/null and b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdb differ
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdx b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdx
new file mode 100644
index 0000000..c595e2f
Binary files /dev/null and b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdx differ
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/aliases.rds b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/aliases.rds
new file mode 100644
index 0000000..035b1b5
Binary files /dev/null and b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/aliases.rds differ
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/paths.rds b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/paths.rds
new file mode 100644
index 0000000..4d7658f
Binary files /dev/null and b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/help/paths.rds differ
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/html/00Index.html b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/html/00Index.html
new file mode 100644
index 0000000..9dd836a
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/html/00Index.html
@@ -0,0 +1,2227 @@
+
+
+References
+-
+
- +
- +
- +
SparkR: Scaling R Programs with Spark, Shivaram Venkataraman, Zongheng Yang, Davies Liu, Eric Liang, Hossein Falaki, Xiangrui Meng, Reynold Xin, Ali Ghodsi, Michael Franklin, Ion Stoica, and Matei Zaharia. SIGMOD 2016. June 2016.
+
+ R Front End for 'Apache Spark'
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/html/R.css b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/html/R.css
new file mode 100644
index 0000000..c228909
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/html/R.css
@@ -0,0 +1,130 @@
+@media screen {
+ .container {
+ padding-right: 10px;
+ padding-left: 10px;
+ margin-right: auto;
+ margin-left: auto;
+ max-width: 900px;
+ }
+}
+
+.rimage img { /* from knitr - for examples and demos */
+ width: 96%;
+ margin-left: 2%;
+}
+
+.katex { font-size: 1.1em; }
+
+code {
+ color: inherit;
+ background: inherit;
+}
+
+body {
+ line-height: 1.4;
+ background: white;
+ color: black;
+}
+
+a:link {
+ background: white;
+ color: blue;
+}
+
+a:visited {
+ background: white;
+ color: rgb(50%, 0%, 50%);
+}
+
+h1 {
+ background: white;
+ color: rgb(55%, 55%, 55%);
+ font-family: monospace;
+ font-size: 1.4em; /* x-large; */
+ text-align: center;
+}
+
+h2 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-size: 1.2em; /* large; */
+ text-align: center;
+}
+
+h3 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-size: 1.2em; /* large; */
+}
+
+h4 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-style: italic;
+ font-size: 1.2em; /* large; */
+}
+
+h5 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+}
+
+h6 {
+ background: white;
+ color: rgb(40%, 40%, 40%);
+ font-family: monospace;
+ font-style: italic;
+}
+
+img.toplogo {
+ width: 4em;
+ vertical-align: middle;
+}
+
+img.arrow {
+ width: 30px;
+ height: 30px;
+ border: 0;
+}
+
+span.acronym {
+ font-size: small;
+}
+
+span.env {
+ font-family: monospace;
+}
+
+span.file {
+ font-family: monospace;
+}
+
+span.option{
+ font-family: monospace;
+}
+
+span.pkg {
+ font-weight: bold;
+}
+
+span.samp{
+ font-family: monospace;
+}
+
+div.vignettes a:hover {
+ background: rgb(85%, 85%, 85%);
+}
+
+tr {
+ vertical-align: top;
+}
+
+span.rlang {
+ font-family: Courier New, Courier;
+ color: #666666;
+}
+
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/profile/general.R b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/profile/general.R
new file mode 100644
index 0000000..8c75c19
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/profile/general.R
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.First <- function() {
+ packageDir <- Sys.getenv("SPARKR_PACKAGE_DIR")
+ dirs <- strsplit(packageDir, ",")[[1]]
+ .libPaths(c(dirs, .libPaths()))
+ Sys.setenv(NOAWT = 1)
+}
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/profile/shell.R b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/profile/shell.R
new file mode 100644
index 0000000..ffedb30
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/profile/shell.R
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.First <- function() {
+ home <- Sys.getenv("SPARK_HOME")
+ .libPaths(c(file.path(home, "R", "lib"), .libPaths()))
+ Sys.setenv(NOAWT = 1)
+
+ # Make sure SparkR package is the last loaded one
+ old <- getOption("defaultPackages")
+ options(defaultPackages = c(old, "SparkR"))
+
+ spark <- SparkR::sparkR.session()
+ assign("spark", spark, envir = .GlobalEnv)
+ sc <- SparkR:::callJStatic("org.apache.spark.sql.api.r.SQLUtils", "getJavaSparkContext", spark)
+ assign("sc", sc, envir = .GlobalEnv)
+ sparkVer <- SparkR:::callJMethod(sc, "version")
+ cat("\nWelcome to")
+ cat("\n")
+ cat(" ____ __", "\n")
+ cat(" / __/__ ___ _____/ /__", "\n")
+ cat(" _\\ \\/ _ \\/ _ `/ __/ '_/", "\n")
+ cat(" /___/ .__/\\_,_/_/ /_/\\_\\")
+ if (nchar(sparkVer) == 0) {
+ cat("\n")
+ } else {
+ cat(" version", sparkVer, "\n")
+ }
+ cat(" /_/", "\n")
+ cat("\n")
+
+ cat("\nSparkSession Web UI available at", SparkR::sparkR.uiWebUrl())
+ cat("\nSparkSession available as 'spark'(master = ", unlist(SparkR::sparkR.conf("spark.master")),
+ ", app id = ", unlist(SparkR::sparkR.conf("spark.app.id")), ").", "\n", sep = "")
+}
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/tests/testthat/test_basic.R b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/tests/testthat/test_basic.R
new file mode 100644
index 0000000..80df3d8
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/tests/testthat/test_basic.R
@@ -0,0 +1,100 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+context("basic tests for CRAN")
+
+test_that("create DataFrame from list or data.frame", {
+ tryCatch(checkJavaVersion(),
+ error = function(e) { skip("error on Java check") },
+ warning = function(e) { skip("warning on Java check") })
+
+ sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE,
+ sparkConfig = sparkRTestConfig)
+
+ i <- 4
+ df <- createDataFrame(data.frame(dummy = 1:i))
+ expect_equal(count(df), i)
+
+ l <- list(list(a = 1, b = 2), list(a = 3, b = 4))
+ df <- createDataFrame(l)
+ expect_equal(columns(df), c("a", "b"))
+
+ a <- 1:3
+ b <- c("a", "b", "c")
+ ldf <- data.frame(a, b)
+ df <- createDataFrame(ldf)
+ expect_equal(columns(df), c("a", "b"))
+ expect_equal(dtypes(df), list(c("a", "int"), c("b", "string")))
+ expect_equal(count(df), 3)
+ ldf2 <- collect(df)
+ expect_equal(ldf$a, ldf2$a)
+
+ mtcarsdf <- createDataFrame(mtcars)
+ expect_equivalent(collect(mtcarsdf), mtcars)
+
+ bytes <- as.raw(c(1, 2, 3))
+ df <- createDataFrame(list(list(bytes)))
+ expect_equal(collect(df)[[1]][[1]], bytes)
+
+ sparkR.session.stop()
+})
+
+test_that("spark.glm and predict", {
+ tryCatch(checkJavaVersion(),
+ error = function(e) { skip("error on Java check") },
+ warning = function(e) { skip("warning on Java check") })
+
+ sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE,
+ sparkConfig = sparkRTestConfig)
+
+ training <- suppressWarnings(createDataFrame(iris))
+ # gaussian family
+ model <- spark.glm(training, Sepal_Width ~ Sepal_Length + Species)
+ prediction <- predict(model, training)
+ expect_equal(typeof(take(select(prediction, "prediction"), 1)$prediction), "double")
+ vals <- collect(select(prediction, "prediction"))
+ rVals <- predict(glm(Sepal.Width ~ Sepal.Length + Species, data = iris), iris)
+ expect_true(all(abs(rVals - vals) < 1e-6), rVals - vals)
+
+ # Gamma family
+ x <- runif(100, -1, 1)
+ y <- rgamma(100, rate = 10 / exp(0.5 + 1.2 * x), shape = 10)
+ df <- as.DataFrame(as.data.frame(list(x = x, y = y)))
+ model <- glm(y ~ x, family = Gamma, df)
+ out <- capture.output(print(summary(model)))
+ expect_true(any(grepl("Dispersion parameter for gamma family", out)))
+
+ # tweedie family
+ model <- spark.glm(training, Sepal_Width ~ Sepal_Length + Species,
+ family = "tweedie", var.power = 1.2, link.power = 0.0)
+ prediction <- predict(model, training)
+ expect_equal(typeof(take(select(prediction, "prediction"), 1)$prediction), "double")
+ vals <- collect(select(prediction, "prediction"))
+
+ # manual calculation of the R predicted values to avoid dependence on statmod
+ #' library(statmod)
+ #' rModel <- glm(Sepal.Width ~ Sepal.Length + Species, data = iris,
+ #' family = tweedie(var.power = 1.2, link.power = 0.0))
+ #' print(coef(rModel))
+
+ rCoef <- c(0.6455409, 0.1169143, -0.3224752, -0.3282174)
+ rVals <- exp(as.numeric(model.matrix(Sepal.Width ~ Sepal.Length + Species,
+ data = iris) %*% rCoef))
+ expect_true(all(abs(rVals - vals) < 1e-5), rVals - vals)
+
+ sparkR.session.stop()
+})
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/worker/daemon.R b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/worker/daemon.R
new file mode 100644
index 0000000..4589bb9
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/worker/daemon.R
@@ -0,0 +1,102 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Worker daemon
+
+rLibDir <- Sys.getenv("SPARKR_RLIBDIR")
+connectionTimeout <- as.integer(Sys.getenv("SPARKR_BACKEND_CONNECTION_TIMEOUT", "6000"))
+dirs <- strsplit(rLibDir, ",")[[1]]
+script <- file.path(dirs[[1]], "SparkR", "worker", "worker.R")
+
+# preload SparkR package, speedup worker
+.libPaths(c(dirs, .libPaths()))
+suppressPackageStartupMessages(library(SparkR))
+
+port <- as.integer(Sys.getenv("SPARKR_WORKER_PORT"))
+inputCon <- socketConnection(
+ port = port, open = "wb", blocking = TRUE, timeout = connectionTimeout)
+
+SparkR:::doServerAuth(inputCon, Sys.getenv("SPARKR_WORKER_SECRET"))
+
+# Waits indefinitely for a socket connection by default.
+selectTimeout <- NULL
+
+while (TRUE) {
+ ready <- socketSelect(list(inputCon), timeout = selectTimeout)
+
+ # Note that the children should be terminated in the parent. If each child terminates
+ # itself, it appears that the resource is not released properly, that causes an unexpected
+ # termination of this daemon due to, for example, running out of file descriptors
+ # (see SPARK-21093). Therefore, the current implementation tries to retrieve children
+ # that are exited (but not terminated) and then sends a kill signal to terminate them properly
+ # in the parent.
+ #
+ # There are two paths that it attempts to send a signal to terminate the children in the parent.
+ #
+ # 1. Every second if any socket connection is not available and if there are child workers
+ # running.
+ # 2. Right after a socket connection is available.
+ #
+ # In other words, the parent attempts to send the signal to the children every second if
+ # any worker is running or right before launching other worker children from the following
+ # new socket connection.
+
+ # The process IDs of exited children are returned below.
+ children <- parallel:::selectChildren(timeout = 0)
+
+ if (is.integer(children)) {
+ lapply(children, function(child) {
+ # This should be the PIDs of exited children. Otherwise, this returns raw bytes if any data
+ # was sent from this child. In this case, we discard it.
+ pid <- parallel:::readChild(child)
+ if (is.integer(pid)) {
+ # This checks if the data from this child is the same pid of this selected child.
+ if (child == pid) {
+ # If so, we terminate this child.
+ tools::pskill(child, tools::SIGUSR1)
+ }
+ }
+ })
+ } else if (is.null(children)) {
+ # If it is NULL, there are no children. Waits indefinitely for a socket connection.
+ selectTimeout <- NULL
+ }
+
+ if (ready) {
+ port <- SparkR:::readInt(inputCon)
+ # There is a small chance that it could be interrupted by signal, retry one time
+ if (length(port) == 0) {
+ port <- SparkR:::readInt(inputCon)
+ if (length(port) == 0) {
+ cat("quitting daemon\n")
+ quit(save = "no")
+ }
+ }
+ p <- parallel:::mcfork()
+ if (inherits(p, "masterProcess")) {
+ # Reach here because this is a child process.
+ close(inputCon)
+ Sys.setenv(SPARKR_WORKER_PORT = port)
+ try(source(script))
+ # Note that this mcexit does not fully terminate this child.
+ parallel:::mcexit(0L)
+ } else {
+ # Forking succeeded and we need to check if they finished their jobs every second.
+ selectTimeout <- 1
+ }
+ }
+}
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/SparkR/worker/worker.R b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/worker/worker.R
new file mode 100644
index 0000000..7fc4680
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/R/lib/SparkR/worker/worker.R
@@ -0,0 +1,292 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Worker class
+
+# Get current system time
+currentTimeSecs <- function() {
+ as.numeric(Sys.time())
+}
+
+# Get elapsed time
+elapsedSecs <- function() {
+ proc.time()[3]
+}
+
+compute <- function(mode, partition, serializer, deserializer, key,
+ colNames, computeFunc, inputData) {
+ if (mode > 0) {
+ if (deserializer == "row") {
+ # Transform the list of rows into a data.frame
+ # Note that the optional argument stringsAsFactors for rbind is
+ # available since R 3.2.4. So we set the global option here.
+ oldOpt <- getOption("stringsAsFactors")
+ options(stringsAsFactors = FALSE)
+
+ # Handle binary data types
+ if ("raw" %in% sapply(inputData[[1]], class)) {
+ inputData <- SparkR:::rbindRaws(inputData)
+ } else {
+ inputData <- do.call(rbind.data.frame, inputData)
+ }
+
+ options(stringsAsFactors = oldOpt)
+
+ names(inputData) <- colNames
+ } else {
+ # Check to see if inputData is a valid data.frame
+ stopifnot(deserializer == "byte" || deserializer == "arrow")
+ stopifnot(is.data.frame(inputData))
+ }
+
+ if (mode == 2) {
+ output <- computeFunc(key, inputData)
+ } else {
+ output <- computeFunc(inputData)
+ }
+ if (serializer == "row") {
+ # Transform the result data.frame back to a list of rows
+ output <- split(output, seq(nrow(output)))
+ } else {
+ # Serialize the output to a byte array
+ stopifnot(serializer == "byte" || serializer == "arrow")
+ }
+ } else {
+ output <- computeFunc(partition, inputData)
+ }
+ return(output)
+}
+
+outputResult <- function(serializer, output, outputCon) {
+ if (serializer == "byte") {
+ SparkR:::writeRawSerialize(outputCon, output)
+ } else if (serializer == "row") {
+ SparkR:::writeRowSerialize(outputCon, output)
+ } else if (serializer == "arrow") {
+ SparkR:::writeSerializeInArrow(outputCon, output)
+ } else {
+ # write lines one-by-one with flag
+ lapply(output, function(line) SparkR:::writeString(outputCon, line))
+ }
+}
+
+# Constants
+specialLengths <- list(END_OF_STREAM = 0L, TIMING_DATA = -1L)
+
+# Timing R process boot
+bootTime <- currentTimeSecs()
+bootElap <- elapsedSecs()
+
+rLibDir <- Sys.getenv("SPARKR_RLIBDIR")
+connectionTimeout <- as.integer(Sys.getenv("SPARKR_BACKEND_CONNECTION_TIMEOUT", "6000"))
+dirs <- strsplit(rLibDir, ",")[[1]]
+# Set libPaths to include SparkR package as loadNamespace needs this
+# TODO: Figure out if we can avoid this by not loading any objects that require
+# SparkR namespace
+.libPaths(c(dirs, .libPaths()))
+suppressPackageStartupMessages(library(SparkR))
+
+port <- as.integer(Sys.getenv("SPARKR_WORKER_PORT"))
+inputCon <- socketConnection(
+ port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
+SparkR:::doServerAuth(inputCon, Sys.getenv("SPARKR_WORKER_SECRET"))
+
+outputCon <- socketConnection(
+ port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
+SparkR:::doServerAuth(outputCon, Sys.getenv("SPARKR_WORKER_SECRET"))
+
+# read the index of the current partition inside the RDD
+partition <- SparkR:::readInt(inputCon)
+
+deserializer <- SparkR:::readString(inputCon)
+serializer <- SparkR:::readString(inputCon)
+
+# Include packages as required
+packageNames <- unserialize(SparkR:::readRaw(inputCon))
+for (pkg in packageNames) {
+ suppressPackageStartupMessages(library(as.character(pkg), character.only = TRUE))
+}
+
+# read function dependencies
+funcLen <- SparkR:::readInt(inputCon)
+computeFunc <- unserialize(SparkR:::readRawLen(inputCon, funcLen))
+env <- environment(computeFunc)
+parent.env(env) <- .GlobalEnv # Attach under global environment.
+
+# Timing init envs for computing
+initElap <- elapsedSecs()
+
+# Read and set broadcast variables
+numBroadcastVars <- SparkR:::readInt(inputCon)
+if (numBroadcastVars > 0) {
+ for (bcast in seq(1:numBroadcastVars)) {
+ bcastId <- SparkR:::readInt(inputCon)
+ value <- unserialize(SparkR:::readRaw(inputCon))
+ SparkR:::setBroadcastValue(bcastId, value)
+ }
+}
+
+# Timing broadcast
+broadcastElap <- elapsedSecs()
+# Initial input timing
+inputElap <- broadcastElap
+
+# If -1: read as normal RDD; if >= 0, treat as pairwise RDD and treat the int
+# as number of partitions to create.
+numPartitions <- SparkR:::readInt(inputCon)
+
+# 0 - RDD mode, 1 - dapply mode, 2 - gapply mode
+mode <- SparkR:::readInt(inputCon)
+
+if (mode > 0) {
+ colNames <- SparkR:::readObject(inputCon)
+}
+
+isEmpty <- SparkR:::readInt(inputCon)
+computeInputElapsDiff <- 0
+outputComputeElapsDiff <- 0
+
+if (isEmpty != 0) {
+ if (numPartitions == -1) {
+ if (deserializer == "byte") {
+ # Now read as many characters as described in funcLen
+ data <- SparkR:::readDeserialize(inputCon)
+ } else if (deserializer == "string") {
+ data <- as.list(readLines(inputCon))
+ } else if (deserializer == "row" && mode == 2) {
+ dataWithKeys <- SparkR:::readMultipleObjectsWithKeys(inputCon)
+ keys <- dataWithKeys$keys
+ data <- dataWithKeys$data
+ } else if (deserializer == "row") {
+ data <- SparkR:::readMultipleObjects(inputCon)
+ } else if (deserializer == "arrow" && mode == 2) {
+ dataWithKeys <- SparkR:::readDeserializeWithKeysInArrow(inputCon)
+ keys <- dataWithKeys$keys
+ data <- dataWithKeys$data
+ } else if (deserializer == "arrow" && mode == 1) {
+ data <- SparkR:::readDeserializeInArrow(inputCon)
+ # See https://stat.ethz.ch/pipermail/r-help/2010-September/252046.html
+ # rbind.fill might be an alternative to make it faster if plyr is installed.
+ # Also, note that, 'dapply' applies a function to each partition.
+ data <- do.call("rbind", data)
+ }
+
+ # Timing reading input data for execution
+ inputElap <- elapsedSecs()
+ if (mode > 0) {
+ if (mode == 1) {
+ output <- compute(mode, partition, serializer, deserializer, NULL,
+ colNames, computeFunc, data)
+ } else {
+ # gapply mode
+ outputs <- list()
+ for (i in seq_len(length(data))) {
+ # Timing reading input data for execution
+ computeStart <- elapsedSecs()
+ output <- compute(mode, partition, serializer, deserializer, keys[[i]],
+ colNames, computeFunc, data[[i]])
+ computeElap <- elapsedSecs()
+ if (serializer == "arrow") {
+ outputs[[length(outputs) + 1L]] <- output
+ } else {
+ outputResult(serializer, output, outputCon)
+ outputComputeElapsDiff <- outputComputeElapsDiff + (elapsedSecs() - computeElap)
+ }
+ computeInputElapsDiff <- computeInputElapsDiff + (computeElap - computeStart)
+ }
+
+ if (serializer == "arrow") {
+ # See https://stat.ethz.ch/pipermail/r-help/2010-September/252046.html
+ # rbind.fill might be an alternative to make it faster if plyr is installed.
+ outputStart <- elapsedSecs()
+ combined <- do.call("rbind", outputs)
+ SparkR:::writeSerializeInArrow(outputCon, combined)
+ outputComputeElapsDiff <- elapsedSecs() - outputStart
+ }
+ }
+ } else {
+ output <- compute(mode, partition, serializer, deserializer, NULL,
+ colNames, computeFunc, data)
+ }
+ if (mode != 2) {
+ # Not a gapply mode
+ computeElap <- elapsedSecs()
+ outputResult(serializer, output, outputCon)
+ outputElap <- elapsedSecs()
+ computeInputElapsDiff <- computeElap - inputElap
+ outputComputeElapsDiff <- outputElap - computeElap
+ }
+ } else {
+ if (deserializer == "byte") {
+ # Now read as many characters as described in funcLen
+ data <- SparkR:::readDeserialize(inputCon)
+ } else if (deserializer == "string") {
+ data <- readLines(inputCon)
+ } else if (deserializer == "row") {
+ data <- SparkR:::readMultipleObjects(inputCon)
+ }
+ # Timing reading input data for execution
+ inputElap <- elapsedSecs()
+
+ res <- new.env()
+
+ # Step 1: hash the data to an environment
+ hashTupleToEnvir <- function(tuple) {
+ # NOTE: execFunction is the hash function here
+ hashVal <- computeFunc(tuple[[1]])
+ bucket <- as.character(hashVal %% numPartitions)
+ acc <- res[[bucket]]
+ # Create a new accumulator
+ if (is.null(acc)) {
+ acc <- SparkR:::initAccumulator()
+ }
+ SparkR:::addItemToAccumulator(acc, tuple)
+ res[[bucket]] <- acc
+ }
+ invisible(lapply(data, hashTupleToEnvir))
+ # Timing computing
+ computeElap <- elapsedSecs()
+
+ # Step 2: write out all of the environment as key-value pairs.
+ for (name in ls(res)) {
+ SparkR:::writeInt(outputCon, 2L)
+ SparkR:::writeInt(outputCon, as.integer(name))
+ # Truncate the accumulator list to the number of elements we have
+ length(res[[name]]$data) <- res[[name]]$counter
+ SparkR:::writeRawSerialize(outputCon, res[[name]]$data)
+ }
+ # Timing output
+ outputElap <- elapsedSecs()
+ computeInputElapsDiff <- computeElap - inputElap
+ outputComputeElapsDiff <- outputElap - computeElap
+ }
+}
+
+# Report timing
+SparkR:::writeInt(outputCon, specialLengths$TIMING_DATA)
+SparkR:::writeDouble(outputCon, bootTime)
+SparkR:::writeDouble(outputCon, initElap - bootElap) # init
+SparkR:::writeDouble(outputCon, broadcastElap - initElap) # broadcast
+SparkR:::writeDouble(outputCon, inputElap - broadcastElap) # input
+SparkR:::writeDouble(outputCon, computeInputElapsDiff) # compute
+SparkR:::writeDouble(outputCon, outputComputeElapsDiff) # output
+
+# End of output
+SparkR:::writeInt(outputCon, specialLengths$END_OF_STREAM)
+
+close(outputCon)
+close(inputCon)
diff --git a/spark-3.5.0-bin-hadoop3/R/lib/sparkr.zip b/spark-3.5.0-bin-hadoop3/R/lib/sparkr.zip
new file mode 100644
index 0000000..4bccfc4
Binary files /dev/null and b/spark-3.5.0-bin-hadoop3/R/lib/sparkr.zip differ
diff --git a/spark-3.5.0-bin-hadoop3/README.md b/spark-3.5.0-bin-hadoop3/README.md
new file mode 100644
index 0000000..310df41
--- /dev/null
+++ b/spark-3.5.0-bin-hadoop3/README.md
@@ -0,0 +1,125 @@
+# Apache Spark
+
+Spark is a unified analytics engine for large-scale data processing. It provides
+high-level APIs in Scala, Java, Python, and R, and an optimized engine that
+supports general computation graphs for data analysis. It also supports a
+rich set of higher-level tools including Spark SQL for SQL and DataFrames,
+pandas API on Spark for pandas workloads, MLlib for machine learning, GraphX for graph processing,
+and Structured Streaming for stream processing.
+
+ R Front End for 'Apache Spark'
+![[R logo]](../../../doc/html/Rlogo.svg)
+
++
+![[Up]](../../../doc/html/left.jpg)
+![[Top]](../../../doc/html/up.jpg)
+
+
+Documentation for package ‘SparkR’ version 3.5.0
+ + + +Help Pages
+ + ++A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +misc +
+ + +-- A --
+ +| abs | +Math functions for Column operations |
| abs-method | +Math functions for Column operations |
| acos | +Math functions for Column operations |
| acos-method | +Math functions for Column operations |
| acosh | +Math functions for Column operations |
| acosh-method | +Math functions for Column operations |
| addJobTag | +Add a tag to be assigned to all the jobs started by this thread. |
| add_months | +Date time arithmetic functions for Column operations |
| add_months-method | +Date time arithmetic functions for Column operations |
| AFTSurvivalRegressionModel-class | +S4 class that represents a AFTSurvivalRegressionModel |
| agg | +summarize |
| agg-method | +summarize |
| alias | +alias |
| alias-method | +alias |
| ALSModel-class | +S4 class that represents an ALSModel |
| approxCountDistinct | +Aggregate functions for Column operations |
| approxCountDistinct-method | +Aggregate functions for Column operations |
| approxQuantile | +Calculates the approximate quantiles of numerical columns of a SparkDataFrame |
| approxQuantile-method | +Calculates the approximate quantiles of numerical columns of a SparkDataFrame |
| approx_count_distinct | +Aggregate functions for Column operations |
| approx_count_distinct-method | +Aggregate functions for Column operations |
| arrange | +Arrange Rows by Variables |
| arrange-method | +Arrange Rows by Variables |
| arrays_overlap | +Collection functions for Column operations |
| arrays_overlap-method | +Collection functions for Column operations |
| arrays_zip | +Collection functions for Column operations |
| arrays_zip-method | +Collection functions for Column operations |
| arrays_zip_with | +Collection functions for Column operations |
| arrays_zip_with-method | +Collection functions for Column operations |
| array_aggregate | +Collection functions for Column operations |
| array_aggregate-method | +Collection functions for Column operations |
| array_contains | +Collection functions for Column operations |
| array_contains-method | +Collection functions for Column operations |
| array_distinct | +Collection functions for Column operations |
| array_distinct-method | +Collection functions for Column operations |
| array_except | +Collection functions for Column operations |
| array_except-method | +Collection functions for Column operations |
| array_exists | +Collection functions for Column operations |
| array_exists-method | +Collection functions for Column operations |
| array_filter | +Collection functions for Column operations |
| array_filter-method | +Collection functions for Column operations |
| array_forall | +Collection functions for Column operations |
| array_forall-method | +Collection functions for Column operations |
| array_intersect | +Collection functions for Column operations |
| array_intersect-method | +Collection functions for Column operations |
| array_join | +Collection functions for Column operations |
| array_join-method | +Collection functions for Column operations |
| array_max | +Collection functions for Column operations |
| array_max-method | +Collection functions for Column operations |
| array_min | +Collection functions for Column operations |
| array_min-method | +Collection functions for Column operations |
| array_position | +Collection functions for Column operations |
| array_position-method | +Collection functions for Column operations |
| array_remove | +Collection functions for Column operations |
| array_remove-method | +Collection functions for Column operations |
| array_repeat | +Collection functions for Column operations |
| array_repeat-method | +Collection functions for Column operations |
| array_sort | +Collection functions for Column operations |
| array_sort-method | +Collection functions for Column operations |
| array_to_vector | +ML functions for Column operations |
| array_to_vector-method | +ML functions for Column operations |
| array_transform | +Collection functions for Column operations |
| array_transform-method | +Collection functions for Column operations |
| array_union | +Collection functions for Column operations |
| array_union-method | +Collection functions for Column operations |
| as.data.frame | +Download data from a SparkDataFrame into a R data.frame |
| as.data.frame-method | +Download data from a SparkDataFrame into a R data.frame |
| as.DataFrame | +Create a SparkDataFrame |
| asc | +A set of operations working with SparkDataFrame columns |
| ascii | +String functions for Column operations |
| ascii-method | +String functions for Column operations |
| asc_nulls_first | +A set of operations working with SparkDataFrame columns |
| asc_nulls_last | +A set of operations working with SparkDataFrame columns |
| asin | +Math functions for Column operations |
| asin-method | +Math functions for Column operations |
| asinh | +Math functions for Column operations |
| asinh-method | +Math functions for Column operations |
| assert_true | +Miscellaneous functions for Column operations |
| assert_true-method | +Miscellaneous functions for Column operations |
| associationRules-method | +FP-growth |
| atan | +Math functions for Column operations |
| atan-method | +Math functions for Column operations |
| atan2 | +Math functions for Column operations |
| atan2-method | +Math functions for Column operations |
| atanh | +Math functions for Column operations |
| atanh-method | +Math functions for Column operations |
| attach | +Attach SparkDataFrame to R search path |
| attach-method | +Attach SparkDataFrame to R search path |
| avg | +avg |
| avg-method | +avg |
| awaitTermination | +awaitTermination |
| awaitTermination-method | +awaitTermination |
-- B --
+ +| base64 | +String functions for Column operations |
| base64-method | +String functions for Column operations |
| between | +between |
| between-method | +between |
| bin | +Math functions for Column operations |
| bin-method | +Math functions for Column operations |
| BisectingKMeansModel-class | +S4 class that represents a BisectingKMeansModel |
| bitwiseNOT | +Non-aggregate functions for Column operations |
| bitwiseNOT-method | +Non-aggregate functions for Column operations |
| bitwise_not | +Non-aggregate functions for Column operations |
| bitwise_not-method | +Non-aggregate functions for Column operations |
| bit_length | +String functions for Column operations |
| bit_length-method | +String functions for Column operations |
| broadcast | +broadcast |
| broadcast-method | +broadcast |
| bround | +Math functions for Column operations |
| bround-method | +Math functions for Column operations |
-- C --
+ +| cache | +Cache |
| cache-method | +Cache |
| cacheTable | +Cache Table |
| cancelJobGroup | +Cancel active jobs for the specified group |
| cancelJobsWithTag | +Cancel active jobs that have the specified tag. |
| cast | +Casts the column to a different data type. |
| cast-method | +Casts the column to a different data type. |
| cbrt | +Math functions for Column operations |
| cbrt-method | +Math functions for Column operations |
| ceil | +Math functions for Column operations |
| ceil-method | +Math functions for Column operations |
| ceiling | +Math functions for Column operations |
| ceiling-method | +Math functions for Column operations |
| checkpoint | +checkpoint |
| checkpoint-method | +checkpoint |
| clearCache | +Clear Cache |
| clearJobGroup | +Clear current job group ID and its description |
| clearJobTags | +Clear the current thread's job tags. |
| coalesce | +Coalesce |
| coalesce-method | +Coalesce |
| coalesce-method | +Non-aggregate functions for Column operations |
| collect | +Collects all the elements of a SparkDataFrame and coerces them into an R data.frame. |
| collect-method | +Collects all the elements of a SparkDataFrame and coerces them into an R data.frame. |
| collect_list | +Aggregate functions for Column operations |
| collect_list-method | +Aggregate functions for Column operations |
| collect_set | +Aggregate functions for Column operations |
| collect_set-method | +Aggregate functions for Column operations |
| colnames | +Column Names of SparkDataFrame |
| colnames-method | +Column Names of SparkDataFrame |
| colnames<- | +Column Names of SparkDataFrame |
| colnames<--method | +Column Names of SparkDataFrame |
| coltypes | +coltypes |
| coltypes-method | +coltypes |
| coltypes<- | +coltypes |
| coltypes<--method | +coltypes |
| column | +S4 class that represents a SparkDataFrame column |
| Column-class | +S4 class that represents a SparkDataFrame column |
| column-method | +S4 class that represents a SparkDataFrame column |
| columnfunctions | +A set of operations working with SparkDataFrame columns |
| columns | +Column Names of SparkDataFrame |
| columns-method | +Column Names of SparkDataFrame |
| column_aggregate_functions | +Aggregate functions for Column operations |
| column_avro_functions | +Avro processing functions for Column operations |
| column_collection_functions | +Collection functions for Column operations |
| column_datetime_diff_functions | +Date time arithmetic functions for Column operations |
| column_datetime_functions | +Date time functions for Column operations |
| column_math_functions | +Math functions for Column operations |
| column_misc_functions | +Miscellaneous functions for Column operations |
| column_ml_functions | +ML functions for Column operations |
| column_nonaggregate_functions | +Non-aggregate functions for Column operations |
| column_string_functions | +String functions for Column operations |
| column_window_functions | +Window functions for Column operations |
| concat | +Collection functions for Column operations |
| concat-method | +Collection functions for Column operations |
| concat_ws | +String functions for Column operations |
| concat_ws-method | +String functions for Column operations |
| contains | +A set of operations working with SparkDataFrame columns |
| conv | +Math functions for Column operations |
| conv-method | +Math functions for Column operations |
| corr | +corr |
| corr-method | +corr |
| cos | +Math functions for Column operations |
| cos-method | +Math functions for Column operations |
| cosh | +Math functions for Column operations |
| cosh-method | +Math functions for Column operations |
| cot | +Math functions for Column operations |
| cot-method | +Math functions for Column operations |
| count | +Count |
| count-method | +Count |
| count-method | +Returns the number of rows in a SparkDataFrame |
| countDistinct | +Aggregate functions for Column operations |
| countDistinct-method | +Aggregate functions for Column operations |
| count_distinct | +Aggregate functions for Column operations |
| count_distinct-method | +Aggregate functions for Column operations |
| cov | +cov |
| cov-method | +cov |
| covar_pop | +cov |
| covar_pop-method | +cov |
| covar_samp | +cov |
| covar_samp-method | +cov |
| crc32 | +Miscellaneous functions for Column operations |
| crc32-method | +Miscellaneous functions for Column operations |
| createDataFrame | +Create a SparkDataFrame |
| createExternalTable | +(Deprecated) Create an external table |
| createOrReplaceTempView | +Creates a temporary view using the given name. |
| createOrReplaceTempView-method | +Creates a temporary view using the given name. |
| createTable | +Creates a table based on the dataset in a data source |
| create_array | +Non-aggregate functions for Column operations |
| create_array-method | +Non-aggregate functions for Column operations |
| create_map | +Non-aggregate functions for Column operations |
| create_map-method | +Non-aggregate functions for Column operations |
| crossJoin | +CrossJoin |
| crossJoin-method | +CrossJoin |
| crosstab | +Computes a pair-wise frequency table of the given columns |
| crosstab-method | +Computes a pair-wise frequency table of the given columns |
| csc | +Math functions for Column operations |
| csc-method | +Math functions for Column operations |
| cube | +cube |
| cube-method | +cube |
| cume_dist | +Window functions for Column operations |
| cume_dist-method | +Window functions for Column operations |
| currentCatalog | +Returns the current default catalog |
| currentDatabase | +Returns the current default database |
| current_date | +Date time functions for Column operations |
| current_date-method | +Date time functions for Column operations |
| current_timestamp | +Date time functions for Column operations |
| current_timestamp-method | +Date time functions for Column operations |
-- D --
+ +| dapply | +dapply |
| dapply-method | +dapply |
| dapplyCollect | +dapplyCollect |
| dapplyCollect-method | +dapplyCollect |
| databaseExists | +Checks if the database with the specified name exists. |
| datediff | +Date time arithmetic functions for Column operations |
| datediff-method | +Date time arithmetic functions for Column operations |
| date_add | +Date time arithmetic functions for Column operations |
| date_add-method | +Date time arithmetic functions for Column operations |
| date_format | +Date time arithmetic functions for Column operations |
| date_format-method | +Date time arithmetic functions for Column operations |
| date_sub | +Date time arithmetic functions for Column operations |
| date_sub-method | +Date time arithmetic functions for Column operations |
| date_trunc | +Date time functions for Column operations |
| date_trunc-method | +Date time functions for Column operations |
| dayofmonth | +Date time functions for Column operations |
| dayofmonth-method | +Date time functions for Column operations |
| dayofweek | +Date time functions for Column operations |
| dayofweek-method | +Date time functions for Column operations |
| dayofyear | +Date time functions for Column operations |
| dayofyear-method | +Date time functions for Column operations |
| DecisionTreeClassificationModel-class | +S4 class that represents a DecisionTreeClassificationModel |
| DecisionTreeRegressionModel-class | +S4 class that represents a DecisionTreeRegressionModel |
| decode | +String functions for Column operations |
| decode-method | +String functions for Column operations |
| degrees | +Math functions for Column operations |
| degrees-method | +Math functions for Column operations |
| dense_rank | +Window functions for Column operations |
| dense_rank-method | +Window functions for Column operations |
| desc | +A set of operations working with SparkDataFrame columns |
| describe | +describe |
| describe-method | +describe |
| desc_nulls_first | +A set of operations working with SparkDataFrame columns |
| desc_nulls_last | +A set of operations working with SparkDataFrame columns |
| dim | +Returns the dimensions of SparkDataFrame |
| dim-method | +Returns the dimensions of SparkDataFrame |
| distinct | +Distinct |
| distinct-method | +Distinct |
| drop | +drop |
| drop-method | +drop |
| dropDuplicates | +dropDuplicates |
| dropDuplicates-method | +dropDuplicates |
| dropFields | +dropFields |
| dropFields-method | +dropFields |
| dropna | +A set of SparkDataFrame functions working with NA values |
| dropna-method | +A set of SparkDataFrame functions working with NA values |
| dropTempTable | +(Deprecated) Drop Temporary Table |
| dropTempView | +Drops the temporary view with the given view name in the catalog. |
| dtypes | +DataTypes |
| dtypes-method | +DataTypes |
-- E --
+ +| element_at | +Collection functions for Column operations |
| element_at-method | +Collection functions for Column operations |
| encode | +String functions for Column operations |
| encode-method | +String functions for Column operations |
| endsWith | +endsWith |
| endsWith-method | +endsWith |
| except | +except |
| except-method | +except |
| exceptAll | +exceptAll |
| exceptAll-method | +exceptAll |
| exp | +Math functions for Column operations |
| exp-method | +Math functions for Column operations |
| explain | +Explain |
| explain-method | +Explain |
| explode | +Collection functions for Column operations |
| explode-method | +Collection functions for Column operations |
| explode_outer | +Collection functions for Column operations |
| explode_outer-method | +Collection functions for Column operations |
| expm1 | +Math functions for Column operations |
| expm1-method | +Math functions for Column operations |
| expr | +Non-aggregate functions for Column operations |
| expr-method | +Non-aggregate functions for Column operations |
-- F --
+ +| factorial | +Math functions for Column operations |
| factorial-method | +Math functions for Column operations |
| fillna | +A set of SparkDataFrame functions working with NA values |
| fillna-method | +A set of SparkDataFrame functions working with NA values |
| filter | +Filter |
| filter-method | +Filter |
| findFrequentSequentialPatterns-method | +PrefixSpan |
| first | +Return the first row of a SparkDataFrame |
| first-method | +Return the first row of a SparkDataFrame |
| fitted | +Get fitted result from a k-means model |
| fitted-method | +Get fitted result from a k-means model |
| fitted-method | +Bisecting K-Means Clustering Model |
| flatten | +Collection functions for Column operations |
| flatten-method | +Collection functions for Column operations |
| floor | +Math functions for Column operations |
| floor-method | +Math functions for Column operations |
| FMClassificationModel-class | +S4 class that represents a FMClassificationModel |
| FMRegressionModel-class | +S4 class that represents a FMRegressionModel |
| format_number | +String functions for Column operations |
| format_number-method | +String functions for Column operations |
| format_string | +String functions for Column operations |
| format_string-method | +String functions for Column operations |
| FPGrowthModel-class | +S4 class that represents a FPGrowthModel |
| freqItems | +Finding frequent items for columns, possibly with false positives |
| freqItems-method | +Finding frequent items for columns, possibly with false positives |
| freqItemsets-method | +FP-growth |
| from_avro | +Avro processing functions for Column operations |
| from_avro-method | +Avro processing functions for Column operations |
| from_csv | +Collection functions for Column operations |
| from_csv-method | +Collection functions for Column operations |
| from_json | +Collection functions for Column operations |
| from_json-method | +Collection functions for Column operations |
| from_unixtime | +Date time functions for Column operations |
| from_unixtime-method | +Date time functions for Column operations |
| from_utc_timestamp | +Date time arithmetic functions for Column operations |
| from_utc_timestamp-method | +Date time arithmetic functions for Column operations |
| functionExists | +Checks if the function with the specified name exists. |
-- G --
+ +| gapply | +gapply |
| gapply-method | +gapply |
| gapplyCollect | +gapplyCollect |
| gapplyCollect-method | +gapplyCollect |
| GaussianMixtureModel-class | +S4 class that represents a GaussianMixtureModel |
| GBTClassificationModel-class | +S4 class that represents a GBTClassificationModel |
| GBTRegressionModel-class | +S4 class that represents a GBTRegressionModel |
| GeneralizedLinearRegressionModel-class | +S4 class that represents a generalized linear model |
| getDatabase | +Get the database with the specified name |
| getField | +A set of operations working with SparkDataFrame columns |
| getFunc | +Get the function with the specified name |
| getItem | +A set of operations working with SparkDataFrame columns |
| getJobTags | +Get the tags that are currently set to be assigned to all the jobs started by this thread. |
| getLocalProperty | +Get a local property set in this thread, or 'NULL' if it is missing. See 'setLocalProperty'. |
| getNumPartitions | +getNumPartitions |
| getNumPartitions-method | +getNumPartitions |
| getTable | +Get the table with the specified name |
| glm | +Generalized Linear Models (R-compliant) |
| glm-method | +Generalized Linear Models (R-compliant) |
| greatest | +Non-aggregate functions for Column operations |
| greatest-method | +Non-aggregate functions for Column operations |
| groupBy | +GroupBy |
| groupBy-method | +GroupBy |
| groupedData | +S4 class that represents a GroupedData |
| GroupedData-class | +S4 class that represents a GroupedData |
| grouping_bit | +Aggregate functions for Column operations |
| grouping_bit-method | +Aggregate functions for Column operations |
| grouping_id | +Aggregate functions for Column operations |
| grouping_id-method | +Aggregate functions for Column operations |
| group_by | +GroupBy |
| group_by-method | +GroupBy |
-- H --
+ +| hash | +Miscellaneous functions for Column operations |
| hash-method | +Miscellaneous functions for Column operations |
| head | +Head |
| head-method | +Head |
| hex | +Math functions for Column operations |
| hex-method | +Math functions for Column operations |
| hint | +hint |
| hint-method | +hint |
| histogram | +Compute histogram statistics for given column |
| histogram-method | +Compute histogram statistics for given column |
| hour | +Date time functions for Column operations |
| hour-method | +Date time functions for Column operations |
| hypot | +Math functions for Column operations |
| hypot-method | +Math functions for Column operations |
-- I --
+ +| ifelse | +Non-aggregate functions for Column operations |
| ifelse-method | +Non-aggregate functions for Column operations |
| ilike | +A set of operations working with SparkDataFrame columns |
| initcap | +String functions for Column operations |
| initcap-method | +String functions for Column operations |
| input_file_name | +Non-aggregate functions for Column operations |
| input_file_name-method | +Non-aggregate functions for Column operations |
| insertInto | +insertInto |
| insertInto-method | +insertInto |
| install.spark | +Download and Install Apache Spark to a Local Directory |
| instr | +String functions for Column operations |
| instr-method | +String functions for Column operations |
| intersect | +Intersect |
| intersect-method | +Intersect |
| intersectAll | +intersectAll |
| intersectAll-method | +intersectAll |
| is.nan | +Non-aggregate functions for Column operations |
| is.nan-method | +Non-aggregate functions for Column operations |
| isActive | +isActive |
| isActive-method | +isActive |
| isLocal | +isLocal |
| isLocal-method | +isLocal |
| isNaN | +A set of operations working with SparkDataFrame columns |
| isnan | +Non-aggregate functions for Column operations |
| isnan-method | +Non-aggregate functions for Column operations |
| isNotNull | +A set of operations working with SparkDataFrame columns |
| isNull | +A set of operations working with SparkDataFrame columns |
| IsotonicRegressionModel-class | +S4 class that represents an IsotonicRegressionModel |
| isStreaming | +isStreaming |
| isStreaming-method | +isStreaming |
-- J --
+ +| join | +Join |
| join-method | +Join |
-- K --
+ +| KMeansModel-class | +S4 class that represents a KMeansModel |
| KSTest-class | +S4 class that represents an KSTest |
| kurtosis | +Aggregate functions for Column operations |
| kurtosis-method | +Aggregate functions for Column operations |
-- L --
+ +| lag | +Window functions for Column operations |
| lag-method | +Window functions for Column operations |
| last | +last |
| last-method | +last |
| lastProgress | +lastProgress |
| lastProgress-method | +lastProgress |
| last_day | +Date time functions for Column operations |
| last_day-method | +Date time functions for Column operations |
| LDAModel-class | +S4 class that represents an LDAModel |
| lead | +Window functions for Column operations |
| lead-method | +Window functions for Column operations |
| least | +Non-aggregate functions for Column operations |
| least-method | +Non-aggregate functions for Column operations |
| length | +String functions for Column operations |
| length-method | +String functions for Column operations |
| levenshtein | +String functions for Column operations |
| levenshtein-method | +String functions for Column operations |
| like | +A set of operations working with SparkDataFrame columns |
| limit | +Limit |
| limit-method | +Limit |
| LinearRegressionModel-class | +S4 class that represents a LinearRegressionModel |
| LinearSVCModel-class | +S4 class that represents an LinearSVCModel |
| listCatalogs | +Returns a list of catalog available |
| listColumns | +Returns a list of columns for the given table/view in the specified database |
| listDatabases | +Returns a list of databases available |
| listFunctions | +Returns a list of functions registered in the specified database |
| listTables | +Returns a list of tables or views in the specified database |
| lit | +Non-aggregate functions for Column operations |
| lit-method | +Non-aggregate functions for Column operations |
| ln | +Math functions for Column operations |
| ln-method | +Math functions for Column operations |
| loadDF | +Load a SparkDataFrame |
| localCheckpoint | +localCheckpoint |
| localCheckpoint-method | +localCheckpoint |
| locate | +String functions for Column operations |
| locate-method | +String functions for Column operations |
| log | +Math functions for Column operations |
| log-method | +Math functions for Column operations |
| log10 | +Math functions for Column operations |
| log10-method | +Math functions for Column operations |
| log1p | +Math functions for Column operations |
| log1p-method | +Math functions for Column operations |
| log2 | +Math functions for Column operations |
| log2-method | +Math functions for Column operations |
| LogisticRegressionModel-class | +S4 class that represents an LogisticRegressionModel |
| lower | +String functions for Column operations |
| lower-method | +String functions for Column operations |
| lpad | +String functions for Column operations |
| lpad-method | +String functions for Column operations |
| ltrim | +String functions for Column operations |
| ltrim-method | +String functions for Column operations |
-- M --
+ +| make_date | +Date time functions for Column operations |
| make_date-method | +Date time functions for Column operations |
| map_concat | +Collection functions for Column operations |
| map_concat-method | +Collection functions for Column operations |
| map_entries | +Collection functions for Column operations |
| map_entries-method | +Collection functions for Column operations |
| map_filter | +Collection functions for Column operations |
| map_filter-method | +Collection functions for Column operations |
| map_from_arrays | +Collection functions for Column operations |
| map_from_arrays-method | +Collection functions for Column operations |
| map_from_entries | +Collection functions for Column operations |
| map_from_entries-method | +Collection functions for Column operations |
| map_keys | +Collection functions for Column operations |
| map_keys-method | +Collection functions for Column operations |
| map_values | +Collection functions for Column operations |
| map_values-method | +Collection functions for Column operations |
| map_zip_with | +Collection functions for Column operations |
| map_zip_with-method | +Collection functions for Column operations |
| max | +Aggregate functions for Column operations |
| max-method | +Aggregate functions for Column operations |
| max_by | +Aggregate functions for Column operations |
| max_by-method | +Aggregate functions for Column operations |
| md5 | +Miscellaneous functions for Column operations |
| md5-method | +Miscellaneous functions for Column operations |
| mean | +Aggregate functions for Column operations |
| mean-method | +Aggregate functions for Column operations |
| melt | +Unpivot a DataFrame from wide format to long format. |
| melt-method | +Unpivot a DataFrame from wide format to long format. |
| merge | +Merges two data frames |
| merge-method | +Merges two data frames |
| min | +Aggregate functions for Column operations |
| min-method | +Aggregate functions for Column operations |
| minute | +Date time functions for Column operations |
| minute-method | +Date time functions for Column operations |
| min_by | +Aggregate functions for Column operations |
| min_by-method | +Aggregate functions for Column operations |
| monotonically_increasing_id | +Non-aggregate functions for Column operations |
| monotonically_increasing_id-method | +Non-aggregate functions for Column operations |
| month | +Date time functions for Column operations |
| month-method | +Date time functions for Column operations |
| months_between | +Date time arithmetic functions for Column operations |
| months_between-method | +Date time arithmetic functions for Column operations |
| MultilayerPerceptronClassificationModel-class | +S4 class that represents a MultilayerPerceptronClassificationModel |
| mutate | +Mutate |
| mutate-method | +Mutate |
-- N --
+ +| n | +Count |
| n-method | +Count |
| na.omit | +A set of SparkDataFrame functions working with NA values |
| na.omit-method | +A set of SparkDataFrame functions working with NA values |
| NaiveBayesModel-class | +S4 class that represents a NaiveBayesModel |
| names | +Column Names of SparkDataFrame |
| names-method | +Column Names of SparkDataFrame |
| names<- | +Column Names of SparkDataFrame |
| names<--method | +Column Names of SparkDataFrame |
| nanvl | +Non-aggregate functions for Column operations |
| nanvl-method | +Non-aggregate functions for Column operations |
| ncol | +Returns the number of columns in a SparkDataFrame |
| ncol-method | +Returns the number of columns in a SparkDataFrame |
| negate | +Non-aggregate functions for Column operations |
| negate-method | +Non-aggregate functions for Column operations |
| negative | +Non-aggregate functions for Column operations |
| negative-method | +Non-aggregate functions for Column operations |
| next_day | +Date time arithmetic functions for Column operations |
| next_day-method | +Date time arithmetic functions for Column operations |
| not | +! |
| not-method | +! |
| nrow | +Returns the number of rows in a SparkDataFrame |
| nrow-method | +Returns the number of rows in a SparkDataFrame |
| nth_value | +Window functions for Column operations |
| nth_value-method | +Window functions for Column operations |
| ntile | +Window functions for Column operations |
| ntile-method | +Window functions for Column operations |
| n_distinct | +Aggregate functions for Column operations |
| n_distinct-method | +Aggregate functions for Column operations |
-- O --
+ +| octet_length | +String functions for Column operations |
| octet_length-method | +String functions for Column operations |
| orderBy | +Ordering Columns in a WindowSpec |
| orderBy-method | +Arrange Rows by Variables |
| orderBy-method | +Ordering Columns in a WindowSpec |
| otherwise | +otherwise |
| otherwise-method | +otherwise |
| over | +over |
| over-method | +over |
| overlay | +String functions for Column operations |
| overlay-method | +String functions for Column operations |
-- P --
+ +| partitionBy | +partitionBy |
| partitionBy-method | +partitionBy |
| percentile_approx | +Aggregate functions for Column operations |
| percentile_approx-method | +Aggregate functions for Column operations |
| percent_rank | +Window functions for Column operations |
| percent_rank-method | +Window functions for Column operations |
| persist | +Persist |
| persist-method | +Persist |
| pivot | +Pivot a column of the GroupedData and perform the specified aggregation. |
| pivot-method | +Pivot a column of the GroupedData and perform the specified aggregation. |
| pmod | +Math functions for Column operations |
| pmod-method | +Math functions for Column operations |
| posexplode | +Collection functions for Column operations |
| posexplode-method | +Collection functions for Column operations |
| posexplode_outer | +Collection functions for Column operations |
| posexplode_outer-method | +Collection functions for Column operations |
| positive | +Non-aggregate functions for Column operations |
| positive-method | +Non-aggregate functions for Column operations |
| PowerIterationClustering-class | +S4 class that represents a PowerIterationClustering |
| predict | +Makes predictions from a MLlib model |
| predict-method | +Alternating Least Squares (ALS) for Collaborative Filtering |
| predict-method | +Bisecting K-Means Clustering Model |
| predict-method | +Decision Tree Model for Regression and Classification |
| predict-method | +Factorization Machines Classification Model |
| predict-method | +Factorization Machines Regression Model |
| predict-method | +FP-growth |
| predict-method | +Multivariate Gaussian Mixture Model (GMM) |
| predict-method | +Gradient Boosted Tree Model for Regression and Classification |
| predict-method | +Generalized Linear Models |
| predict-method | +Isotonic Regression Model |
| predict-method | +K-Means Clustering Model |
| predict-method | +Linear Regression Model |
| predict-method | +Logistic Regression Model |
| predict-method | +Multilayer Perceptron Classification Model |
| predict-method | +Naive Bayes Models |
| predict-method | +Random Forest Model for Regression and Classification |
| predict-method | +Accelerated Failure Time (AFT) Survival Regression Model |
| predict-method | +Linear SVM Model |
| PrefixSpan-class | +S4 class that represents a PrefixSpan |
| print.summary.DecisionTreeClassificationModel | +Decision Tree Model for Regression and Classification |
| print.summary.DecisionTreeRegressionModel | +Decision Tree Model for Regression and Classification |
| print.summary.GBTClassificationModel | +Gradient Boosted Tree Model for Regression and Classification |
| print.summary.GBTRegressionModel | +Gradient Boosted Tree Model for Regression and Classification |
| print.summary.GeneralizedLinearRegressionModel | +Generalized Linear Models |
| print.summary.KSTest | +(One-Sample) Kolmogorov-Smirnov Test |
| print.summary.RandomForestClassificationModel | +Random Forest Model for Regression and Classification |
| print.summary.RandomForestRegressionModel | +Random Forest Model for Regression and Classification |
| printSchema | +Print Schema of a SparkDataFrame |
| printSchema-method | +Print Schema of a SparkDataFrame |
| product | +Aggregate functions for Column operations |
| product-method | +Aggregate functions for Column operations |
-- Q --
+ +| quarter | +Date time functions for Column operations |
| quarter-method | +Date time functions for Column operations |
| queryName | +queryName |
| queryName-method | +queryName |
-- R --
+ +| radians | +Math functions for Column operations |
| radians-method | +Math functions for Column operations |
| raise_error | +Miscellaneous functions for Column operations |
| raise_error-method | +Miscellaneous functions for Column operations |
| rand | +Non-aggregate functions for Column operations |
| rand-method | +Non-aggregate functions for Column operations |
| randn | +Non-aggregate functions for Column operations |
| randn-method | +Non-aggregate functions for Column operations |
| RandomForestClassificationModel-class | +S4 class that represents a RandomForestClassificationModel |
| RandomForestRegressionModel-class | +S4 class that represents a RandomForestRegressionModel |
| randomSplit | +randomSplit |
| randomSplit-method | +randomSplit |
| rangeBetween | +rangeBetween |
| rangeBetween-method | +rangeBetween |
| rank | +Window functions for Column operations |
| rank-method | +Window functions for Column operations |
| rbind | +Union two or more SparkDataFrames |
| rbind-method | +Union two or more SparkDataFrames |
| read.df | +Load a SparkDataFrame |
| read.jdbc | +Create a SparkDataFrame representing the database table accessible via JDBC URL |
| read.json | +Create a SparkDataFrame from a JSON file. |
| read.ml | +Load a fitted MLlib model from the input path. |
| read.orc | +Create a SparkDataFrame from an ORC file. |
| read.parquet | +Create a SparkDataFrame from a Parquet file. |
| read.stream | +Load a streaming SparkDataFrame |
| read.text | +Create a SparkDataFrame from a text file. |
| recoverPartitions | +Recovers all the partitions in the directory of a table and update the catalog |
| refreshByPath | +Invalidates and refreshes all the cached data and metadata for SparkDataFrame containing path |
| refreshTable | +Invalidates and refreshes all the cached data and metadata of the given table |
| regexp_extract | +String functions for Column operations |
| regexp_extract-method | +String functions for Column operations |
| regexp_replace | +String functions for Column operations |
| regexp_replace-method | +String functions for Column operations |
| registerTempTable | +(Deprecated) Register Temporary Table |
| registerTempTable-method | +(Deprecated) Register Temporary Table |
| removeJobTag | +Remove a tag previously added to be assigned to all the jobs started by this thread. Noop if such a tag was not added earlier. |
| rename | +rename |
| rename-method | +rename |
| repartition | +Repartition |
| repartition-method | +Repartition |
| repartitionByRange | +Repartition by range |
| repartitionByRange-method | +Repartition by range |
| repeat_string | +String functions for Column operations |
| repeat_string-method | +String functions for Column operations |
| reverse | +Collection functions for Column operations |
| reverse-method | +Collection functions for Column operations |
| rint | +Math functions for Column operations |
| rint-method | +Math functions for Column operations |
| rlike | +A set of operations working with SparkDataFrame columns |
| rollup | +rollup |
| rollup-method | +rollup |
| round | +Math functions for Column operations |
| round-method | +Math functions for Column operations |
| rowsBetween | +rowsBetween |
| rowsBetween-method | +rowsBetween |
| row_number | +Window functions for Column operations |
| row_number-method | +Window functions for Column operations |
| rpad | +String functions for Column operations |
| rpad-method | +String functions for Column operations |
| rtrim | +String functions for Column operations |
| rtrim-method | +String functions for Column operations |
-- S --
+ +| sample | +Sample |
| sample-method | +Sample |
| sampleBy | +Returns a stratified sample without replacement |
| sampleBy-method | +Returns a stratified sample without replacement |
| sample_frac | +Sample |
| sample_frac-method | +Sample |
| saveAsTable | +Save the contents of the SparkDataFrame to a data source as a table |
| saveAsTable-method | +Save the contents of the SparkDataFrame to a data source as a table |
| saveDF | +Save the contents of SparkDataFrame to a data source. |
| saveDF-method | +Save the contents of SparkDataFrame to a data source. |
| schema | +Get schema object |
| schema-method | +Get schema object |
| schema_of_csv | +Collection functions for Column operations |
| schema_of_csv-method | +Collection functions for Column operations |
| schema_of_json | +Collection functions for Column operations |
| schema_of_json-method | +Collection functions for Column operations |
| sd | +Aggregate functions for Column operations |
| sd-method | +Aggregate functions for Column operations |
| sec | +Math functions for Column operations |
| sec-method | +Math functions for Column operations |
| second | +Date time functions for Column operations |
| second-method | +Date time functions for Column operations |
| select | +Select |
| select-method | +Select |
| selectExpr | +SelectExpr |
| selectExpr-method | +SelectExpr |
| setCheckpointDir | +Set checkpoint directory |
| setCurrentCatalog | +Sets the current default catalog |
| setCurrentDatabase | +Sets the current default database |
| setInterruptOnCancel | +Set the behavior of job cancellation from jobs started in this thread. |
| setJobDescription | +Set a human readable description of the current job. |
| setJobGroup | +Assigns a group ID to all the jobs started by this thread until the group ID is set to a different value or cleared. |
| setLocalProperty | +Set a local property that affects jobs submitted from this thread, such as the Spark fair scheduler pool. |
| setLogLevel | +Set new log level |
| sha1 | +Miscellaneous functions for Column operations |
| sha1-method | +Miscellaneous functions for Column operations |
| sha2 | +Miscellaneous functions for Column operations |
| sha2-method | +Miscellaneous functions for Column operations |
| shiftLeft | +Math functions for Column operations |
| shiftleft | +Math functions for Column operations |
| shiftLeft-method | +Math functions for Column operations |
| shiftleft-method | +Math functions for Column operations |
| shiftRight | +Math functions for Column operations |
| shiftright | +Math functions for Column operations |
| shiftRight-method | +Math functions for Column operations |
| shiftright-method | +Math functions for Column operations |
| shiftRightUnsigned | +Math functions for Column operations |
| shiftrightunsigned | +Math functions for Column operations |
| shiftRightUnsigned-method | +Math functions for Column operations |
| shiftrightunsigned-method | +Math functions for Column operations |
| show | +show |
| show-method | +show |
| showDF | +showDF |
| showDF-method | +showDF |
| shuffle | +Collection functions for Column operations |
| shuffle-method | +Collection functions for Column operations |
| sign | +Math functions for Column operations |
| sign-method | +Math functions for Column operations |
| signum | +Math functions for Column operations |
| signum-method | +Math functions for Column operations |
| sin | +Math functions for Column operations |
| sin-method | +Math functions for Column operations |
| sinh | +Math functions for Column operations |
| sinh-method | +Math functions for Column operations |
| size | +Collection functions for Column operations |
| size-method | +Collection functions for Column operations |
| skewness | +Aggregate functions for Column operations |
| skewness-method | +Aggregate functions for Column operations |
| slice | +Collection functions for Column operations |
| slice-method | +Collection functions for Column operations |
| sort_array | +Collection functions for Column operations |
| sort_array-method | +Collection functions for Column operations |
| soundex | +String functions for Column operations |
| soundex-method | +String functions for Column operations |
| spark.addFile | +Add a file or directory to be downloaded with this Spark job on every node. |
| spark.als | +Alternating Least Squares (ALS) for Collaborative Filtering |
| spark.als-method | +Alternating Least Squares (ALS) for Collaborative Filtering |
| spark.assignClusters | +PowerIterationClustering |
| spark.assignClusters-method | +PowerIterationClustering |
| spark.associationRules | +FP-growth |
| spark.associationRules-method | +FP-growth |
| spark.bisectingKmeans | +Bisecting K-Means Clustering Model |
| spark.bisectingKmeans-method | +Bisecting K-Means Clustering Model |
| spark.decisionTree | +Decision Tree Model for Regression and Classification |
| spark.decisionTree-method | +Decision Tree Model for Regression and Classification |
| spark.findFrequentSequentialPatterns | +PrefixSpan |
| spark.findFrequentSequentialPatterns-method | +PrefixSpan |
| spark.fmClassifier | +Factorization Machines Classification Model |
| spark.fmClassifier-method | +Factorization Machines Classification Model |
| spark.fmRegressor | +Factorization Machines Regression Model |
| spark.fmRegressor-method | +Factorization Machines Regression Model |
| spark.fpGrowth | +FP-growth |
| spark.fpGrowth-method | +FP-growth |
| spark.freqItemsets | +FP-growth |
| spark.freqItemsets-method | +FP-growth |
| spark.gaussianMixture | +Multivariate Gaussian Mixture Model (GMM) |
| spark.gaussianMixture-method | +Multivariate Gaussian Mixture Model (GMM) |
| spark.gbt | +Gradient Boosted Tree Model for Regression and Classification |
| spark.gbt-method | +Gradient Boosted Tree Model for Regression and Classification |
| spark.getSparkFiles | +Get the absolute path of a file added through spark.addFile. |
| spark.getSparkFilesRootDirectory | +Get the root directory that contains files added through spark.addFile. |
| spark.glm | +Generalized Linear Models |
| spark.glm-method | +Generalized Linear Models |
| spark.isoreg | +Isotonic Regression Model |
| spark.isoreg-method | +Isotonic Regression Model |
| spark.kmeans | +K-Means Clustering Model |
| spark.kmeans-method | +K-Means Clustering Model |
| spark.kstest | +(One-Sample) Kolmogorov-Smirnov Test |
| spark.kstest-method | +(One-Sample) Kolmogorov-Smirnov Test |
| spark.lapply | +Run a function over a list of elements, distributing the computations with Spark |
| spark.lda | +Latent Dirichlet Allocation |
| spark.lda-method | +Latent Dirichlet Allocation |
| spark.lm | +Linear Regression Model |
| spark.lm-method | +Linear Regression Model |
| spark.logit | +Logistic Regression Model |
| spark.logit-method | +Logistic Regression Model |
| spark.mlp | +Multilayer Perceptron Classification Model |
| spark.mlp-method | +Multilayer Perceptron Classification Model |
| spark.naiveBayes | +Naive Bayes Models |
| spark.naiveBayes-method | +Naive Bayes Models |
| spark.perplexity | +Latent Dirichlet Allocation |
| spark.perplexity-method | +Latent Dirichlet Allocation |
| spark.posterior | +Latent Dirichlet Allocation |
| spark.posterior-method | +Latent Dirichlet Allocation |
| spark.randomForest | +Random Forest Model for Regression and Classification |
| spark.randomForest-method | +Random Forest Model for Regression and Classification |
| spark.survreg | +Accelerated Failure Time (AFT) Survival Regression Model |
| spark.survreg-method | +Accelerated Failure Time (AFT) Survival Regression Model |
| spark.svmLinear | +Linear SVM Model |
| spark.svmLinear-method | +Linear SVM Model |
| SparkDataFrame-class | +S4 class that represents a SparkDataFrame |
| sparkR.callJMethod | +Call Java Methods |
| sparkR.callJStatic | +Call Static Java Methods |
| sparkR.conf | +Get Runtime Config from the current active SparkSession |
| sparkR.init | +(Deprecated) Initialize a new Spark Context |
| sparkR.newJObject | +Create Java Objects |
| sparkR.session | +Get the existing SparkSession or initialize a new SparkSession. |
| sparkR.session.stop | +Stop the Spark Session and Spark Context |
| sparkR.stop | +Stop the Spark Session and Spark Context |
| sparkR.uiWebUrl | +Get the URL of the SparkUI instance for the current active SparkSession |
| sparkR.version | +Get version of Spark on which this application is running |
| sparkRHive.init | +(Deprecated) Initialize a new HiveContext |
| sparkRSQL.init | +(Deprecated) Initialize a new SQLContext |
| spark_partition_id | +Non-aggregate functions for Column operations |
| spark_partition_id-method | +Non-aggregate functions for Column operations |
| split_string | +String functions for Column operations |
| split_string-method | +String functions for Column operations |
| sql | +SQL Query |
| sqrt | +Math functions for Column operations |
| sqrt-method | +Math functions for Column operations |
| startsWith | +startsWith |
| startsWith-method | +startsWith |
| status | +status |
| status-method | +status |
| std | +Aggregate functions for Column operations |
| std-method | +Aggregate functions for Column operations |
| stddev | +Aggregate functions for Column operations |
| stddev-method | +Aggregate functions for Column operations |
| stddev_pop | +Aggregate functions for Column operations |
| stddev_pop-method | +Aggregate functions for Column operations |
| stddev_samp | +Aggregate functions for Column operations |
| stddev_samp-method | +Aggregate functions for Column operations |
| stopQuery | +stopQuery |
| stopQuery-method | +stopQuery |
| storageLevel | +StorageLevel |
| storageLevel-method | +StorageLevel |
| str | +Compactly display the structure of a dataset |
| str-method | +Compactly display the structure of a dataset |
| StreamingQuery-class | +S4 class that represents a StreamingQuery |
| struct | +Non-aggregate functions for Column operations |
| struct-method | +Non-aggregate functions for Column operations |
| structField | +structField |
| structField.character | +structField |
| structField.jobj | +structField |
| structType | +structType |
| structType.character | +structType |
| structType.jobj | +structType |
| structType.structField | +structType |
| subset | +Subset |
| subset-method | +Subset |
| substr | +substr |
| substr-method | +substr |
| substring_index | +String functions for Column operations |
| substring_index-method | +String functions for Column operations |
| sum | +Aggregate functions for Column operations |
| sum-method | +Aggregate functions for Column operations |
| sumDistinct | +Aggregate functions for Column operations |
| sumDistinct-method | +Aggregate functions for Column operations |
| summarize | +summarize |
| summarize-method | +summarize |
| summary | +summary |
| summary-method | +Alternating Least Squares (ALS) for Collaborative Filtering |
| summary-method | +Bisecting K-Means Clustering Model |
| summary-method | +Decision Tree Model for Regression and Classification |
| summary-method | +Factorization Machines Classification Model |
| summary-method | +Factorization Machines Regression Model |
| summary-method | +Multivariate Gaussian Mixture Model (GMM) |
| summary-method | +Gradient Boosted Tree Model for Regression and Classification |
| summary-method | +Generalized Linear Models |
| summary-method | +Isotonic Regression Model |
| summary-method | +K-Means Clustering Model |
| summary-method | +(One-Sample) Kolmogorov-Smirnov Test |
| summary-method | +Latent Dirichlet Allocation |
| summary-method | +Linear Regression Model |
| summary-method | +Logistic Regression Model |
| summary-method | +Multilayer Perceptron Classification Model |
| summary-method | +Naive Bayes Models |
| summary-method | +Random Forest Model for Regression and Classification |
| summary-method | +Accelerated Failure Time (AFT) Survival Regression Model |
| summary-method | +Linear SVM Model |
| summary-method | +summary |
| sum_distinct | +Aggregate functions for Column operations |
| sum_distinct-method | +Aggregate functions for Column operations |
-- T --
+ +| tableExists | +Checks if the table with the specified name exists. |
| tableNames | +Table Names |
| tables | +Tables |
| tableToDF | +Create a SparkDataFrame from a SparkSQL table or view |
| take | +Take the first NUM rows of a SparkDataFrame and return the results as a R data.frame |
| take-method | +Take the first NUM rows of a SparkDataFrame and return the results as a R data.frame |
| tan | +Math functions for Column operations |
| tan-method | +Math functions for Column operations |
| tanh | +Math functions for Column operations |
| tanh-method | +Math functions for Column operations |
| timestamp_seconds | +Date time functions for Column operations |
| timestamp_seconds-method | +Date time functions for Column operations |
| toDegrees | +Math functions for Column operations |
| toDegrees-method | +Math functions for Column operations |
| toJSON | +toJSON |
| toJSON-method | +toJSON |
| toRadians | +Math functions for Column operations |
| toRadians-method | +Math functions for Column operations |
| to_avro | +Avro processing functions for Column operations |
| to_avro-method | +Avro processing functions for Column operations |
| to_csv | +Collection functions for Column operations |
| to_csv-method | +Collection functions for Column operations |
| to_date | +Date time functions for Column operations |
| to_date-method | +Date time functions for Column operations |
| to_json | +Collection functions for Column operations |
| to_json-method | +Collection functions for Column operations |
| to_timestamp | +Date time functions for Column operations |
| to_timestamp-method | +Date time functions for Column operations |
| to_utc_timestamp | +Date time arithmetic functions for Column operations |
| to_utc_timestamp-method | +Date time arithmetic functions for Column operations |
| transform | +Mutate |
| transform-method | +Mutate |
| transform_keys | +Collection functions for Column operations |
| transform_keys-method | +Collection functions for Column operations |
| transform_values | +Collection functions for Column operations |
| transform_values-method | +Collection functions for Column operations |
| translate | +String functions for Column operations |
| translate-method | +String functions for Column operations |
| trim | +String functions for Column operations |
| trim-method | +String functions for Column operations |
| trunc | +Date time functions for Column operations |
| trunc-method | +Date time functions for Column operations |
-- U --
+ +| unbase64 | +String functions for Column operations |
| unbase64-method | +String functions for Column operations |
| uncacheTable | +Uncache Table |
| unhex | +Math functions for Column operations |
| unhex-method | +Math functions for Column operations |
| union | +Return a new SparkDataFrame containing the union of rows |
| union-method | +Return a new SparkDataFrame containing the union of rows |
| unionAll | +Return a new SparkDataFrame containing the union of rows. |
| unionAll-method | +Return a new SparkDataFrame containing the union of rows. |
| unionByName | +Return a new SparkDataFrame containing the union of rows, matched by column names |
| unionByName-method | +Return a new SparkDataFrame containing the union of rows, matched by column names |
| unique | +Distinct |
| unique-method | +Distinct |
| unix_timestamp | +Date time functions for Column operations |
| unix_timestamp-method | +Date time functions for Column operations |
| unpersist | +Unpersist |
| unpersist-method | +Unpersist |
| unpivot | +Unpivot a DataFrame from wide format to long format. |
| unpivot-method | +Unpivot a DataFrame from wide format to long format. |
| upper | +String functions for Column operations |
| upper-method | +String functions for Column operations |
-- V --
+ +| var | +Aggregate functions for Column operations |
| var-method | +Aggregate functions for Column operations |
| variance | +Aggregate functions for Column operations |
| variance-method | +Aggregate functions for Column operations |
| var_pop | +Aggregate functions for Column operations |
| var_pop-method | +Aggregate functions for Column operations |
| var_samp | +Aggregate functions for Column operations |
| var_samp-method | +Aggregate functions for Column operations |
| vector_to_array | +ML functions for Column operations |
| vector_to_array-method | +ML functions for Column operations |
-- W --
+ +| weekofyear | +Date time functions for Column operations |
| weekofyear-method | +Date time functions for Column operations |
| when | +Non-aggregate functions for Column operations |
| when-method | +Non-aggregate functions for Column operations |
| where | +Filter |
| where-method | +Filter |
| width_bucket | +Math functions for Column operations |
| width_bucket-method | +Math functions for Column operations |
| window | +Date time functions for Column operations |
| window-method | +Date time functions for Column operations |
| windowOrderBy | +windowOrderBy |
| windowOrderBy-method | +windowOrderBy |
| windowPartitionBy | +windowPartitionBy |
| windowPartitionBy-method | +windowPartitionBy |
| WindowSpec-class | +S4 class that represents a WindowSpec |
| with | +Evaluate a R expression in an environment constructed from a SparkDataFrame |
| with-method | +Evaluate a R expression in an environment constructed from a SparkDataFrame |
| withColumn | +WithColumn |
| withColumn-method | +WithColumn |
| withColumnRenamed | +rename |
| withColumnRenamed-method | +rename |
| withField | +withField |
| withField-method | +withField |
| withWatermark | +withWatermark |
| withWatermark-method | +withWatermark |
| write.df | +Save the contents of SparkDataFrame to a data source. |
| write.df-method | +Save the contents of SparkDataFrame to a data source. |
| write.jdbc | +Save the content of SparkDataFrame to an external database table via JDBC. |
| write.jdbc-method | +Save the content of SparkDataFrame to an external database table via JDBC. |
| write.json | +Save the contents of SparkDataFrame as a JSON file |
| write.json-method | +Save the contents of SparkDataFrame as a JSON file |
| write.ml | +Saves the MLlib model to the input path |
| write.ml-method | +Alternating Least Squares (ALS) for Collaborative Filtering |
| write.ml-method | +Bisecting K-Means Clustering Model |
| write.ml-method | +Decision Tree Model for Regression and Classification |
| write.ml-method | +Factorization Machines Classification Model |
| write.ml-method | +Factorization Machines Regression Model |
| write.ml-method | +FP-growth |
| write.ml-method | +Multivariate Gaussian Mixture Model (GMM) |
| write.ml-method | +Gradient Boosted Tree Model for Regression and Classification |
| write.ml-method | +Generalized Linear Models |
| write.ml-method | +Isotonic Regression Model |
| write.ml-method | +K-Means Clustering Model |
| write.ml-method | +Latent Dirichlet Allocation |
| write.ml-method | +Linear Regression Model |
| write.ml-method | +Logistic Regression Model |
| write.ml-method | +Multilayer Perceptron Classification Model |
| write.ml-method | +Naive Bayes Models |
| write.ml-method | +Random Forest Model for Regression and Classification |
| write.ml-method | +Accelerated Failure Time (AFT) Survival Regression Model |
| write.ml-method | +Linear SVM Model |
| write.orc | +Save the contents of SparkDataFrame as an ORC file, preserving the schema. |
| write.orc-method | +Save the contents of SparkDataFrame as an ORC file, preserving the schema. |
| write.parquet | +Save the contents of SparkDataFrame as a Parquet file, preserving the schema. |
| write.parquet-method | +Save the contents of SparkDataFrame as a Parquet file, preserving the schema. |
| write.stream | +Write the streaming SparkDataFrame to a data source. |
| write.stream-method | +Write the streaming SparkDataFrame to a data source. |
| write.text | +Save the content of SparkDataFrame in a text file at the specified path. |
| write.text-method | +Save the content of SparkDataFrame in a text file at the specified path. |
-- X --
+ +| xxhash64 | +Miscellaneous functions for Column operations |
| xxhash64-method | +Miscellaneous functions for Column operations |
-- Y --
+ +| year | +Date time functions for Column operations |
| year-method | +Date time functions for Column operations |
-- misc --
+ +| !-method | +! |
| $ | +Select |
| $-method | +Select |
| $<- | +Select |
| $<--method | +Select |
| %<=>% | +%<=>% |
| %<=>%-method | +%<=>% |
| %in% | +Match a column with given values. |
| %in%-method | +Match a column with given values. |
| [ | +Subset |
| [-method | +Subset |
| [[ | +Subset |
| [[-method | +Subset |
| [[<- | +Subset |
| [[<--method | +Subset |
+ * bin/run-example JavaPageRank data/mllib/pagerank_data.txt 10 + *+ */ +public final class JavaPageRank { + private static final Pattern SPACES = Pattern.compile("\\s+"); + + static void showWarning() { + String warning = "WARN: This is a naive implementation of PageRank " + + "and is given as an example! \n" + + "Please use the PageRank implementation found in " + + "org.apache.spark.graphx.lib.PageRank for more conventional use."; + System.err.println(warning); + } + + private static class Sum implements Function2
+ * bin/run-example ml.JavaAFTSurvivalRegressionExample + *+ */ +public class JavaAFTSurvivalRegressionExample { + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaAFTSurvivalRegressionExample") + .getOrCreate(); + + // $example on$ + List
+ * bin/run-example ml.JavaBisectingKMeansExample + *+ */ +public class JavaBisectingKMeansExample { + + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaBisectingKMeansExample") + .getOrCreate(); + + // $example on$ + // Loads data. + Dataset
+ * bin/run-example ml.JavaBucketizerExample + *+ */ +public class JavaBucketizerExample { + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaBucketizerExample") + .getOrCreate(); + + // $example on$ + double[] splits = {Double.NEGATIVE_INFINITY, -0.5, 0.0, 0.5, Double.POSITIVE_INFINITY}; + + List
+ * bin/run-example ml.JavaChiSquareTestExample + *+ */ +public class JavaChiSquareTestExample { + + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaChiSquareTestExample") + .getOrCreate(); + + // $example on$ + List
+ * bin/run-example ml.JavaCorrelationExample + *+ */ +public class JavaCorrelationExample { + + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaCorrelationExample") + .getOrCreate(); + + // $example on$ + List
+ * bin/run-example ml.JavaFPGrowthExample + *+ */ +public class JavaFPGrowthExample { + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaFPGrowthExample") + .getOrCreate(); + + // $example on$ + List
+ * bin/run-example ml.JavaGaussianMixtureExample + *+ */ +public class JavaGaussianMixtureExample { + + public static void main(String[] args) { + + // Creates a SparkSession + SparkSession spark = SparkSession + .builder() + .appName("JavaGaussianMixtureExample") + .getOrCreate(); + + // $example on$ + // Loads data + Dataset
+ * bin/run-example ml.JavaGeneralizedLinearRegressionExample + *+ */ + +public class JavaGeneralizedLinearRegressionExample { + + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaGeneralizedLinearRegressionExample") + .getOrCreate(); + + // $example on$ + // Load training data + Dataset
+ * bin/run-example ml.JavaIsotonicRegressionExample + *+ */ +public class JavaIsotonicRegressionExample { + + public static void main(String[] args) { + // Create a SparkSession. + SparkSession spark = SparkSession + .builder() + .appName("JavaIsotonicRegressionExample") + .getOrCreate(); + + // $example on$ + // Loads data. + Dataset
+ * bin/run-example ml.JavaKMeansExample + *+ */ +public class JavaKMeansExample { + + public static void main(String[] args) { + // Create a SparkSession. + SparkSession spark = SparkSession + .builder() + .appName("JavaKMeansExample") + .getOrCreate(); + + // $example on$ + // Loads data. + Dataset
+ * bin/run-example ml.JavaLDAExample + *+ */ +public class JavaLDAExample { + + public static void main(String[] args) { + // Creates a SparkSession + SparkSession spark = SparkSession + .builder() + .appName("JavaLDAExample") + .getOrCreate(); + + // $example on$ + // Loads data. + Dataset
+ * bin/run-example ml.JavaOneVsRestExample + *+ */ +public class JavaOneVsRestExample { + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaOneVsRestExample") + .getOrCreate(); + + // $example on$ + // load data file. + Dataset
+ * bin/run-example ml.JavaPrefixSpanExample + *+ */ +public class JavaPrefixSpanExample { + public static void main(String[] args) { + SparkSession spark = SparkSession + .builder() + .appName("JavaPrefixSpanExample") + .getOrCreate(); + + // $example on$ + List