TestSQLContext

Type Members

implicit class DslAttribute extends AnyRef

Definition Classes
ExpressionConversions
implicit class DslExpression extends ImplicitOperators

Definition Classes
ExpressionConversions
implicit class DslString extends ImplicitOperators

Definition Classes
ExpressionConversions
implicit class DslSymbol extends ImplicitAttribute

Definition Classes
ExpressionConversions
abstract class ImplicitAttribute extends ImplicitOperators

Definition Classes
ExpressionConversions
abstract class QueryExecution extends AnyRef

The primary workflow for executing relational queries using Spark.
class SparkPlanner extends SparkStrategies

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
lazy val analyzer: Analyzer

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
final def asInstanceOf[T0]: T0

Definition Classes
Any
def avg(e: Expression): Average

Definition Classes
ExpressionConversions
implicit def binaryToLiteral(a: Array[Byte]): Literal

Definition Classes
ExpressionConversions
implicit def booleanToLiteral(b: Boolean): Literal

Definition Classes
ExpressionConversions
implicit def byteToLiteral(b: Byte): Literal

Definition Classes
ExpressionConversions
def cacheTable(tableName: String): Unit

Caches the specified table in-memory.
Caches the specified table in-memory.

Definition Classes
SQLContext
lazy val catalog: Catalog

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def contains(key: String): Boolean

Definition Classes
SQLConf
def count(e: Expression): Count

Definition Classes
ExpressionConversions
def countDistinct(e: Expression*): CountDistinct

Definition Classes
ExpressionConversions
def createParquetFile[A <: Product](path: String, allowExisting: Boolean = true, conf: Configuration = new Configuration())(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

:: Experimental :: Creates an empty parquet file with the schema of class A, which can be registered as a table.
:: Experimental :: Creates an empty parquet file with the schema of class A, which can be registered as a table. This registered table can be used as the target of future insertInto operations.
```
val sqlContext = new SQLContext(...)
import sqlContext._

case class Person(name: String, age: Int)
createParquetFile[Person]("path/to/file.parquet").registerAsTable("people")
sql("INSERT INTO people SELECT 'michael', 29")
```
A
A case class type that describes the desired schema of the parquet file to be created.
path
The path where the directory containing parquet metadata should be created. Data inserted into this table will also be stored at this location.
allowExisting
When false, an exception will be thrown if this directory already exists.
conf
A Hadoop configuration object that can be used to specify options to the parquet output format.

Definition Classes
SQLContext
Annotations
@Experimental()
implicit def createSchemaRDD[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

Creates a SchemaRDD from an RDD of case classes.
Creates a SchemaRDD from an RDD of case classes.

Definition Classes
SQLContext
implicit def decimalToLiteral(d: BigDecimal): Literal

Definition Classes
ExpressionConversions
implicit def doubleToLiteral(d: Double): Literal

Definition Classes
ExpressionConversions
lazy val emptyResult: RDD[Row]

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def executePlan(plan: LogicalPlan): QueryExecution

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def executeSql(sql: String): QueryExecution

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def first(e: Expression): First

Definition Classes
ExpressionConversions
implicit def floatToLiteral(f: Float): Literal

Definition Classes
ExpressionConversions
def get(key: String, defaultValue: String): String

Definition Classes
SQLConf
def get(key: String): String

Definition Classes
SQLConf
def getAll: Array[(String, String)]

Definition Classes
SQLConf
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getOption(key: String): Option[String]

Definition Classes
SQLConf
def hashCode(): Int

Definition Classes
AnyRef → Any
implicit def intToLiteral(i: Int): Literal

Definition Classes
ExpressionConversions
def isCached(tableName: String): Boolean

Returns true if the table is currently cached in-memory.
Returns true if the table is currently cached in-memory.

Definition Classes
SQLContext
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def jsonFile(path: String, samplingRatio: Double): SchemaRDD

:: Experimental ::
:: Experimental ::

Definition Classes
SQLContext
Annotations
@Experimental()
def jsonFile(path: String): SchemaRDD

Loads a JSON file (one object per line), returning the result as a SchemaRDD.
Loads a JSON file (one object per line), returning the result as a SchemaRDD. It goes through the entire dataset once to determine the schema.

Definition Classes
SQLContext
def jsonRDD(json: RDD[String], samplingRatio: Double): SchemaRDD

:: Experimental ::
:: Experimental ::

Definition Classes
SQLContext
Annotations
@Experimental()
def jsonRDD(json: RDD[String]): SchemaRDD

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a SchemaRDD.
Loads an RDD[String] storing JSON objects (one object per record), returning the result as a SchemaRDD. It goes through the entire dataset once to determine the schema.

Definition Classes
SQLContext
lazy val logger: Logger

Attributes
protected
Definition Classes
Logging
implicit def logicalPlanToSparkQuery(plan: LogicalPlan): SchemaRDD

:: DeveloperApi :: Allows catalyst LogicalPlans to be executed as a SchemaRDD.
:: DeveloperApi :: Allows catalyst LogicalPlans to be executed as a SchemaRDD. Note that the LogicalPlan interface is considered internal, and thus not guaranteed to be stable. As a result, using them directly is not recommended.

Definition Classes
SQLContext
Annotations
@DeveloperApi()
implicit def longToLiteral(l: Long): Literal

Definition Classes
ExpressionConversions
def lower(e: Expression): Lower

Definition Classes
ExpressionConversions
def max(e: Expression): Max

Definition Classes
ExpressionConversions
def min(e: Expression): Min

Definition Classes
ExpressionConversions
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val optimizer: Optimizer.type

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def parquetFile(path: String): SchemaRDD

Loads a Parquet file, returning the result as a SchemaRDD.
Loads a Parquet file, returning the result as a SchemaRDD.

Definition Classes
SQLContext
def parseSql(sql: String): LogicalPlan

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
val parser: SqlParser

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
val planner: SparkPlanner

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
val prepareForExecution: RuleExecutor[SparkPlan] { val batches: List[this.Batch] }

Prepares a planned SparkPlan for execution by binding references to specific ordinals, and inserting shuffle operations as needed.
Prepares a planned SparkPlan for execution by binding references to specific ordinals, and inserting shuffle operations as needed.

Attributes
protected[org.apache.spark.sql]
Definition Classes
SQLContext
def registerRDDAsTable(rdd: SchemaRDD, tableName: String): Unit

Registers the given RDD as a temporary table in the catalog.
Registers the given RDD as a temporary table in the catalog. Temporary tables exist only during the lifetime of this instance of SQLContext.

Definition Classes
SQLContext
def set(key: String, value: String): Unit

Definition Classes
SQLConf
def set(props: Properties): Unit

Definition Classes
SQLConf
implicit def shortToLiteral(s: Short): Literal

Definition Classes
ExpressionConversions
val sparkContext: SparkContext

Definition Classes
SQLContext
def sql(sqlText: String): SchemaRDD

Executes a SQL query using Spark, returning the result as a SchemaRDD.
Executes a SQL query using Spark, returning the result as a SchemaRDD.

Definition Classes
SQLContext
implicit def stringToLiteral(s: String): Literal

Definition Classes
ExpressionConversions
def sum(e: Expression): Sum

Definition Classes
ExpressionConversions
def sumDistinct(e: Expression): SumDistinct

Definition Classes
ExpressionConversions
implicit def symbolToUnresolvedAttribute(s: Symbol): UnresolvedAttribute

Definition Classes
ExpressionConversions
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def table(tableName: String): SchemaRDD

Returns the specified table as a SchemaRDD
Returns the specified table as a SchemaRDD

Definition Classes
SQLContext
implicit def timestampToLiteral(t: Timestamp): Literal

Definition Classes
ExpressionConversions
def toDebugString: String

Definition Classes
SQLConf
def toString(): String

Definition Classes
AnyRef → Any
def uncacheTable(tableName: String): Unit

Removes the specified table from the in-memory cache.
Removes the specified table from the in-memory cache.

Definition Classes
SQLContext
def upper(e: Expression): Upper

Definition Classes
ExpressionConversions
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

object TestSQLContext extends SQLContext

Type Members

implicit class DslAttribute extends AnyRef

implicit class DslExpression extends ImplicitOperators

implicit class DslString extends ImplicitOperators

implicit class DslSymbol extends ImplicitAttribute

abstract class ImplicitAttribute extends ImplicitOperators

abstract class QueryExecution extends AnyRef

class SparkPlanner extends SparkStrategies

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

lazy val analyzer: Analyzer

final def asInstanceOf[T0]: T0

def avg(e: Expression): Average

implicit def binaryToLiteral(a: Array[Byte]): Literal

implicit def booleanToLiteral(b: Boolean): Literal

implicit def byteToLiteral(b: Byte): Literal

def cacheTable(tableName: String): Unit

lazy val catalog: Catalog

def clone(): AnyRef

def contains(key: String): Boolean

def count(e: Expression): Count

def countDistinct(e: Expression*): CountDistinct

def createParquetFile[A <: Product](path: String, allowExisting: Boolean = true, conf: Configuration = new Configuration())(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

implicit def createSchemaRDD[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaRDD

implicit def decimalToLiteral(d: BigDecimal): Literal

implicit def doubleToLiteral(d: Double): Literal

lazy val emptyResult: RDD[Row]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executePlan(plan: LogicalPlan): QueryExecution

def executeSql(sql: String): QueryExecution

def finalize(): Unit

def first(e: Expression): First

implicit def floatToLiteral(f: Float): Literal

def get(key: String, defaultValue: String): String

def get(key: String): String

def getAll: Array[(String, String)]

final def getClass(): Class[_]

def getOption(key: String): Option[String]

def hashCode(): Int

implicit def intToLiteral(i: Int): Literal

def isCached(tableName: String): Boolean

final def isInstanceOf[T0]: Boolean

def jsonFile(path: String, samplingRatio: Double): SchemaRDD

def jsonFile(path: String): SchemaRDD

def jsonRDD(json: RDD[String], samplingRatio: Double): SchemaRDD

def jsonRDD(json: RDD[String]): SchemaRDD

lazy val logger: Logger

implicit def logicalPlanToSparkQuery(plan: LogicalPlan): SchemaRDD

implicit def longToLiteral(l: Long): Literal

def lower(e: Expression): Lower

def max(e: Expression): Max

def min(e: Expression): Min

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val optimizer: Optimizer.type

def parquetFile(path: String): SchemaRDD

def parseSql(sql: String): LogicalPlan

val parser: SqlParser

val planner: SparkPlanner

val prepareForExecution: RuleExecutor[SparkPlan] { val batches: List[this.Batch] }

def registerRDDAsTable(rdd: SchemaRDD, tableName: String): Unit

def set(key: String, value: String): Unit

def set(props: Properties): Unit

implicit def shortToLiteral(s: Short): Literal

val sparkContext: SparkContext

def sql(sqlText: String): SchemaRDD

implicit def stringToLiteral(s: String): Literal

def sum(e: Expression): Sum

def sumDistinct(e: Expression): SumDistinct

implicit def symbolToUnresolvedAttribute(s: Symbol): UnresolvedAttribute

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): SchemaRDD