org.apache.spark.mllib.random

Class RandomRDDs

Object

org.apache.spark.mllib.random.RandomRDDs

public class RandomRDDs
extends Object

:: Experimental :: Generator methods for creating RDDs comprised of i.i.d. samples from some distribution.

Constructor Summary

Constructors

Constructor and Description
RandomRDDs()

Method Summary

Methods

Modifier and Type	Method and Description
static JavaDoubleRDD	normalJavaRDD(JavaSparkContext jsc, long size) normalJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default number of partitions and the default seed.
static JavaDoubleRDD	normalJavaRDD(JavaSparkContext jsc, long size, int numPartitions) normalJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default seed.
static JavaDoubleRDD	normalJavaRDD(JavaSparkContext jsc, long size, int numPartitions, long seed) Java-friendly version of normalRDD(org.apache.spark.SparkContext, long, int, long).
static JavaRDD<Vector>	normalJavaVectorRDD(JavaSparkContext jsc, long numRows, int numCols) normalJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default number of partitions and the default seed.
static JavaRDD<Vector>	normalJavaVectorRDD(JavaSparkContext jsc, long numRows, int numCols, int numPartitions) normalJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default seed.
static JavaRDD<Vector>	normalJavaVectorRDD(JavaSparkContext jsc, long numRows, int numCols, int numPartitions, long seed) Java-friendly version of normalVectorRDD(org.apache.spark.SparkContext, long, int, int, long).
static RDD<Object>	normalRDD(SparkContext sc, long size, int numPartitions, long seed) Generates an RDD comprised of i.i.d.
static RDD<Vector>	normalVectorRDD(SparkContext sc, long numRows, int numCols, int numPartitions, long seed) Generates an RDD[Vector] with vectors containing i.i.d.
static JavaDoubleRDD	poissonJavaRDD(JavaSparkContext jsc, double mean, long size) poissonJavaRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, long) with the default number of partitions and the default seed.
static JavaDoubleRDD	poissonJavaRDD(JavaSparkContext jsc, double mean, long size, int numPartitions) poissonJavaRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, long) with the default seed.
static JavaDoubleRDD	poissonJavaRDD(JavaSparkContext jsc, double mean, long size, int numPartitions, long seed) Java-friendly version of poissonRDD(org.apache.spark.SparkContext, double, long, int, long).
static JavaRDD<Vector>	poissonJavaVectorRDD(JavaSparkContext jsc, double mean, long numRows, int numCols) poissonJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, int, long) with the default number of partitions and the default seed.
static JavaRDD<Vector>	poissonJavaVectorRDD(JavaSparkContext jsc, double mean, long numRows, int numCols, int numPartitions) poissonJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, int, long) with the default seed.
static JavaRDD<Vector>	poissonJavaVectorRDD(JavaSparkContext jsc, double mean, long numRows, int numCols, int numPartitions, long seed) Java-friendly version of poissonVectorRDD(org.apache.spark.SparkContext, double, long, int, int, long).
static RDD<Object>	poissonRDD(SparkContext sc, double mean, long size, int numPartitions, long seed) Generates an RDD comprised of i.i.d.
static RDD<Vector>	poissonVectorRDD(SparkContext sc, double mean, long numRows, int numCols, int numPartitions, long seed) Generates an RDD[Vector] with vectors containing i.i.d.
static <T> RDD<T>	randomRDD(SparkContext sc, RandomDataGenerator<T> generator, long size, int numPartitions, long seed, scala.reflect.ClassTag<T> evidence$1) :: DeveloperApi :: Generates an RDD comprised of i.i.d.
static RDD<Vector>	randomVectorRDD(SparkContext sc, RandomDataGenerator<Object> generator, long numRows, int numCols, int numPartitions, long seed) :: DeveloperApi :: Generates an RDD[Vector] with vectors containing i.i.d.
static JavaDoubleRDD	uniformJavaRDD(JavaSparkContext jsc, long size) uniformJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default number of partitions and the default seed.
static JavaDoubleRDD	uniformJavaRDD(JavaSparkContext jsc, long size, int numPartitions) uniformJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default seed.
static JavaDoubleRDD	uniformJavaRDD(JavaSparkContext jsc, long size, int numPartitions, long seed) Java-friendly version of uniformRDD(org.apache.spark.SparkContext, long, int, long).
static JavaRDD<Vector>	uniformJavaVectorRDD(JavaSparkContext jsc, long numRows, int numCols) uniformJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default number of partitions and the default seed.
static JavaRDD<Vector>	uniformJavaVectorRDD(JavaSparkContext jsc, long numRows, int numCols, int numPartitions) uniformJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default seed.
static JavaRDD<Vector>	uniformJavaVectorRDD(JavaSparkContext jsc, long numRows, int numCols, int numPartitions, long seed) Java-friendly version of uniformVectorRDD(org.apache.spark.SparkContext, long, int, int, long).
static RDD<Object>	uniformRDD(SparkContext sc, long size, int numPartitions, long seed) Generates an RDD comprised of i.i.d.
static RDD<Vector>	uniformVectorRDD(SparkContext sc, long numRows, int numCols, int numPartitions, long seed) Generates an RDD[Vector] with vectors containing i.i.d.

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

RandomRDDs

public RandomRDDs()

Method Detail

uniformRDD

public static RDD<Object> uniformRDD(SparkContext sc,
                     long size,
                     int numPartitions,
                     long seed)

Generates an RDD comprised of i.i.d. samples from the uniform distribution U(0.0, 1.0).

To transform the distribution in the generated RDD from U(0.0, 1.0) to U(a, b), use RandomRDDs.uniformRDD(sc, n, p, seed).map(v => a + (b - a) * v).

Parameters:

sc - SparkContext used to create the RDD.

size - Size of the RDD.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism).

seed - Random seed (default: a random long integer).

Returns:

RDD[Double] comprised of i.i.d. samples ~ U(0.0, 1.0).

uniformJavaRDD

public static JavaDoubleRDD uniformJavaRDD(JavaSparkContext jsc,
                           long size,
                           int numPartitions,
                           long seed)

Java-friendly version of uniformRDD(org.apache.spark.SparkContext, long, int, long).

uniformJavaRDD

public static JavaDoubleRDD uniformJavaRDD(JavaSparkContext jsc,
                           long size,
                           int numPartitions)

uniformJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default seed.

uniformJavaRDD

public static JavaDoubleRDD uniformJavaRDD(JavaSparkContext jsc,
                           long size)

uniformJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default number of partitions and the default seed.

normalRDD

public static RDD<Object> normalRDD(SparkContext sc,
                    long size,
                    int numPartitions,
                    long seed)

Generates an RDD comprised of i.i.d. samples from the standard normal distribution.

To transform the distribution in the generated RDD from standard normal to some other normal N(mean, sigma^2^), use RandomRDDs.normalRDD(sc, n, p, seed).map(v => mean + sigma * v).

Parameters:

sc - SparkContext used to create the RDD.

size - Size of the RDD.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism).

seed - Random seed (default: a random long integer).

Returns:

RDD[Double] comprised of i.i.d. samples ~ N(0.0, 1.0).

normalJavaRDD

public static JavaDoubleRDD normalJavaRDD(JavaSparkContext jsc,
                          long size,
                          int numPartitions,
                          long seed)

Java-friendly version of normalRDD(org.apache.spark.SparkContext, long, int, long).

normalJavaRDD

public static JavaDoubleRDD normalJavaRDD(JavaSparkContext jsc,
                          long size,
                          int numPartitions)

normalJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default seed.

normalJavaRDD

public static JavaDoubleRDD normalJavaRDD(JavaSparkContext jsc,
                          long size)

normalJavaRDD(org.apache.spark.api.java.JavaSparkContext, long, int, long) with the default number of partitions and the default seed.

poissonRDD

public static RDD<Object> poissonRDD(SparkContext sc,
                     double mean,
                     long size,
                     int numPartitions,
                     long seed)

Generates an RDD comprised of i.i.d. samples from the Poisson distribution with the input mean.

Parameters:

sc - SparkContext used to create the RDD.

mean - Mean, or lambda, for the Poisson distribution.

size - Size of the RDD.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism).

seed - Random seed (default: a random long integer).

Returns:

RDD[Double] comprised of i.i.d. samples ~ Pois(mean).

poissonJavaRDD

public static JavaDoubleRDD poissonJavaRDD(JavaSparkContext jsc,
                           double mean,
                           long size,
                           int numPartitions,
                           long seed)

Java-friendly version of poissonRDD(org.apache.spark.SparkContext, double, long, int, long).

poissonJavaRDD

public static JavaDoubleRDD poissonJavaRDD(JavaSparkContext jsc,
                           double mean,
                           long size,
                           int numPartitions)

poissonJavaRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, long) with the default seed.

poissonJavaRDD

public static JavaDoubleRDD poissonJavaRDD(JavaSparkContext jsc,
                           double mean,
                           long size)

poissonJavaRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, long) with the default number of partitions and the default seed.

randomRDD

public static <T> RDD<T> randomRDD(SparkContext sc,
                   RandomDataGenerator<T> generator,
                   long size,
                   int numPartitions,
                   long seed,
                   scala.reflect.ClassTag<T> evidence$1)

:: DeveloperApi :: Generates an RDD comprised of i.i.d. samples produced by the input RandomDataGenerator.

Parameters:

sc - SparkContext used to create the RDD.

generator - RandomDataGenerator used to populate the RDD.

size - Size of the RDD.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism).

seed - Random seed (default: a random long integer).

Returns:

RDD[Double] comprised of i.i.d. samples produced by generator.

uniformVectorRDD

public static RDD<Vector> uniformVectorRDD(SparkContext sc,
                           long numRows,
                           int numCols,
                           int numPartitions,
                           long seed)

Generates an RDD[Vector] with vectors containing i.i.d. samples drawn from the uniform distribution on U(0.0, 1.0).

Parameters:

sc - SparkContext used to create the RDD.

numRows - Number of Vectors in the RDD.

numCols - Number of elements in each Vector.

numPartitions - Number of partitions in the RDD.

seed - Seed for the RNG that generates the seed for the generator in each partition.

Returns:

RDD[Vector] with vectors containing i.i.d samples ~ U(0.0, 1.0).

uniformJavaVectorRDD

public static JavaRDD<Vector> uniformJavaVectorRDD(JavaSparkContext jsc,
                                   long numRows,
                                   int numCols,
                                   int numPartitions,
                                   long seed)

Java-friendly version of uniformVectorRDD(org.apache.spark.SparkContext, long, int, int, long).

uniformJavaVectorRDD

public static JavaRDD<Vector> uniformJavaVectorRDD(JavaSparkContext jsc,
                                   long numRows,
                                   int numCols,
                                   int numPartitions)

uniformJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default seed.

uniformJavaVectorRDD

public static JavaRDD<Vector> uniformJavaVectorRDD(JavaSparkContext jsc,
                                   long numRows,
                                   int numCols)

uniformJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default number of partitions and the default seed.

normalVectorRDD

public static RDD<Vector> normalVectorRDD(SparkContext sc,
                          long numRows,
                          int numCols,
                          int numPartitions,
                          long seed)

Generates an RDD[Vector] with vectors containing i.i.d. samples drawn from the standard normal distribution.

Parameters:

sc - SparkContext used to create the RDD.

numRows - Number of Vectors in the RDD.

numCols - Number of elements in each Vector.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism).

seed - Random seed (default: a random long integer).

Returns:

RDD[Vector] with vectors containing i.i.d. samples ~ N(0.0, 1.0).

normalJavaVectorRDD

public static JavaRDD<Vector> normalJavaVectorRDD(JavaSparkContext jsc,
                                  long numRows,
                                  int numCols,
                                  int numPartitions,
                                  long seed)

Java-friendly version of normalVectorRDD(org.apache.spark.SparkContext, long, int, int, long).

normalJavaVectorRDD

public static JavaRDD<Vector> normalJavaVectorRDD(JavaSparkContext jsc,
                                  long numRows,
                                  int numCols,
                                  int numPartitions)

normalJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default seed.

normalJavaVectorRDD

public static JavaRDD<Vector> normalJavaVectorRDD(JavaSparkContext jsc,
                                  long numRows,
                                  int numCols)

normalJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, long, int, int, long) with the default number of partitions and the default seed.

poissonVectorRDD

public static RDD<Vector> poissonVectorRDD(SparkContext sc,
                           double mean,
                           long numRows,
                           int numCols,
                           int numPartitions,
                           long seed)

Generates an RDD[Vector] with vectors containing i.i.d. samples drawn from the Poisson distribution with the input mean.

Parameters:

sc - SparkContext used to create the RDD.

mean - Mean, or lambda, for the Poisson distribution.

numRows - Number of Vectors in the RDD.

numCols - Number of elements in each Vector.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism)

seed - Random seed (default: a random long integer).

Returns:

RDD[Vector] with vectors containing i.i.d. samples ~ Pois(mean).

poissonJavaVectorRDD

public static JavaRDD<Vector> poissonJavaVectorRDD(JavaSparkContext jsc,
                                   double mean,
                                   long numRows,
                                   int numCols,
                                   int numPartitions,
                                   long seed)

Java-friendly version of poissonVectorRDD(org.apache.spark.SparkContext, double, long, int, int, long).

poissonJavaVectorRDD

public static JavaRDD<Vector> poissonJavaVectorRDD(JavaSparkContext jsc,
                                   double mean,
                                   long numRows,
                                   int numCols,
                                   int numPartitions)

poissonJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, int, long) with the default seed.

poissonJavaVectorRDD

public static JavaRDD<Vector> poissonJavaVectorRDD(JavaSparkContext jsc,
                                   double mean,
                                   long numRows,
                                   int numCols)

poissonJavaVectorRDD(org.apache.spark.api.java.JavaSparkContext, double, long, int, int, long) with the default number of partitions and the default seed.

randomVectorRDD

public static RDD<Vector> randomVectorRDD(SparkContext sc,
                          RandomDataGenerator<Object> generator,
                          long numRows,
                          int numCols,
                          int numPartitions,
                          long seed)

:: DeveloperApi :: Generates an RDD[Vector] with vectors containing i.i.d. samples produced by the input RandomDataGenerator.

Parameters:

sc - SparkContext used to create the RDD.

generator - RandomDataGenerator used to populate the RDD.

numRows - Number of Vectors in the RDD.

numCols - Number of elements in each Vector.

numPartitions - Number of partitions in the RDD (default: sc.defaultParallelism).

seed - Random seed (default: a random long integer).

Returns:

RDD[Vector] with vectors containing i.i.d. samples produced by generator.