SQLContext

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

Convert a BaseRelation created for external data sources into a DataFrame.

def cacheTable(tableName: String): Unit

Caches the specified table in-memory.

def clearCache(): Unit

Removes all cached tables from the in-memory cache.

def clone(): AnyRef

def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame

Applies a schema to a List of Java Beans.

def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rows: List[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from a java.util.List containing Rows using the given schema.

def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from a JavaRDD containing Rows using the given schema.

def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an RDD containing Rows using the given schema.

import org.apache.spark.sql._
import org.apache.spark.sql.types._
val sqlContext = new org.apache.spark.sql.SQLContext(sc)

val schema =
  StructType(
    StructField("name", StringType, false) ::
    StructField("age", IntegerType, true) :: Nil)

val people =
  sc.textFile("examples/src/main/resources/people.txt").map(
    _.split(",")).map(p => Row(p(0), p(1).trim.toInt))
val dataFrame = sqlContext.createDataFrame(people, schema)
dataFrame.printSchema
// root
// |-- name: string (nullable = false)
// |-- age: integer (nullable = true)

dataFrame.createOrReplaceTempView("people")
sqlContext.sql("select name from people").collect.foreach(println)

def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from a local Seq of Product.

def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from an RDD of Product (e.g.

def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from a java.util.List of a given type.

List<String> data = Arrays.asList("hello", "world");
Dataset<String> ds = spark.createDataset(data, Encoders.STRING());

def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from an RDD of a given type.

def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]

:: Experimental :: Creates a Dataset from a local Seq of data of a given type.

import spark.implicits._
case class Person(name: String, age: Long)
val data = Seq(Person("Michael", 29), Person("Andy", 30), Person("Justin", 19))
val ds = spark.createDataset(data)

ds.show()
// +-------+---+
// |   name|age|
// +-------+---+
// |Michael| 29|
// |   Andy| 30|
// | Justin| 19|
// +-------+---+

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

(Scala-specific) Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

(Scala-specific) Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, path: String, source: String): DataFrame

Creates an external table from the given path based on a data source and returns the corresponding DataFrame.

def createExternalTable(tableName: String, path: String): DataFrame

Creates an external table from the given path and returns the corresponding DataFrame.

def dropTempTable(tableName: String): Unit

Drops the temporary table with the given table name in the catalog.

def emptyDataFrame: DataFrame

Returns a DataFrame with no rows or columns.

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def experimental: ExperimentalMethods

:: Experimental :: A collection of methods that are considered experimental, but can be used to hook into the query planner for advanced functionality.

def finalize(): Unit

def getAllConfs: Map[String, String]

Return all the configuration properties that have been set (i.e.

final def getClass(): Class[_]

def getConf(key: String, defaultValue: String): String

Return the value of Spark SQL configuration property for the given key.

def getConf(key: String): String

Return the value of Spark SQL configuration property for the given key.

def hashCode(): Int

object implicits extends SQLImplicits with Serializable

:: Experimental :: (Scala-specific) Implicit methods available in Scala for converting common Scala objects into DataFrames.

val sqlContext = new SQLContext(sc)
import sqlContext.implicits._

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

def isCached(tableName: String): Boolean

Returns true if the table is currently cached in-memory.

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def listenerManager: ExecutionListenerManager

An interface to register custom org.apache.spark.sql.util.QueryExecutionListeners that listen for execution metrics.

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

def newSession(): SQLContext

Returns a SQLContext as new session, with separated SQL configurations, temporary tables, registered functions, but sharing the same SparkContext, cached data and other things.

final def notify(): Unit

final def notifyAll(): Unit

def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from start to end (exclusive) with an step value, with partition number specified.

def range(start: Long, end: Long, step: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in a range from start to end (exclusive) with a step value.

def range(start: Long, end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in a range from start to end (exclusive) with step value 1.

def range(end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in a range from 0 to end (exclusive) with step value 1.

def read: DataFrameReader

Returns a DataFrameReader that can be used to read non-streaming data in as a DataFrame.

sqlContext.read.parquet("/path/to/file.parquet")
sqlContext.read.schema(schema).json("/path/to/file.json")

def readStream: DataStreamReader

:: Experimental :: Returns a DataStreamReader that can be used to read streaming data in as a DataFrame.

sparkSession.readStream.parquet("/path/to/directory/of/parquet/files")
sparkSession.readStream.schema(schema).json("/path/to/directory/of/json/files")

def setConf(key: String, value: String): Unit

Set the given Spark SQL configuration property.

def setConf(props: Properties): Unit

Set Spark SQL configuration properties.

def sparkContext: SparkContext

val sparkSession: SparkSession

def sql(sqlText: String): DataFrame

Executes a SQL query using Spark, returning the result as a DataFrame.

def streams: StreamingQueryManager

Returns a StreamingQueryManager that allows managing all the StreamingQueries active on this context.

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): DataFrame

Returns the specified table as a DataFrame.

def tableNames(databaseName: String): Array[String]

Returns the names of tables in the given database as an array.

def tableNames(): Array[String]

Returns the names of tables in the current database as an array.

def tables(databaseName: String): DataFrame

Returns a DataFrame containing names of existing tables in the given database.

def tables(): DataFrame

Returns a DataFrame containing names of existing tables in the current database.

def toString(): String

def udf: UDFRegistration

A collection of methods for registering user-defined functions (UDF).

sqlContext.udf.register("myUDF", (arg1: Int, arg2: String) => arg2 + arg1)

sqlContext.udf().register("myUDF",
    new UDF2<Integer, String, String>() {
        @Override
        public String call(Integer arg1, String arg2) {
            return arg2 + arg1;
        }
   }, DataTypes.StringType);

sqlContext.udf().register("myUDF",
    (Integer arg1, String arg2) -> arg2 + arg1,
    DataTypes.StringType);

def uncacheTable(tableName: String): Unit

Removes the specified table from the in-memory cache.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame

def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame

def jdbc(url: String, table: String, theParts: Array[String]): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jsonFile(path: String, samplingRatio: Double): DataFrame

def jsonFile(path: String, schema: StructType): DataFrame

Loads a JSON file (one object per line) and applies the given schema, returning the result as a DataFrame.

def jsonFile(path: String): DataFrame

Loads a JSON file (one object per line), returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame

Loads a JavaRDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame

Loads an JavaRDD<String> storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], schema: StructType): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def jsonRDD(json: RDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(path: String, source: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the given data source.

def load(path: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the default data source configured by spark.sql.sources.default.

def parquetFile(paths: String*): DataFrame

Loads a Parquet file, returning the result as a DataFrame.