HadoopWriteConfigUtil

Interface for create output format/committer/writer used during saving an RDD using a Hadoop OutputFormat (both from the old mapred API and the new mapreduce API)

Notes: 1. Implementations should throw IllegalArgumentException when wrong hadoop API is referenced; 2. Implementations must be serializable, as the instance instantiated on the driver will be used for tasks on executors; 3. Implementations should have a constructor with exactly one argument: (conf: SerializableConfiguration) or (conf: SerializableJobConf).

Source: HadoopWriteConfigUtil.scala

Linear Supertypes

Serializable, Serializable, AnyRef, Any

Instance Constructors

new HadoopWriteConfigUtil()(implicit arg0: ClassTag[V])

Abstract Value Members

abstract def assertConf(jobContext: JobContext, conf: SparkConf): Unit
abstract def closeWriter(taskContext: TaskAttemptContext): Unit
abstract def createCommitter(jobId: Int): HadoopMapReduceCommitProtocol
abstract def createJobContext(jobTrackerId: String, jobId: Int): JobContext
abstract def createTaskAttemptContext(jobTrackerId: String, jobId: Int, splitId: Int, taskAttemptId: Int): TaskAttemptContext
abstract def initOutputFormat(jobContext: JobContext): Unit
abstract def initWriter(taskContext: TaskAttemptContext, splitId: Int): Unit
abstract def write(pair: (K, V)): Unit

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package io

abstract class HadoopWriteConfigUtil[K, V] extends Serializable

Instance Constructors

new HadoopWriteConfigUtil()(implicit arg0: ClassTag[V])

Abstract Value Members

abstract def assertConf(jobContext: JobContext, conf: SparkConf): Unit

abstract def closeWriter(taskContext: TaskAttemptContext): Unit

abstract def createCommitter(jobId: Int): HadoopMapReduceCommitProtocol

abstract def createJobContext(jobTrackerId: String, jobId: Int): JobContext

abstract def createTaskAttemptContext(jobTrackerId: String, jobId: Int, splitId: Int, taskAttemptId: Int): TaskAttemptContext

abstract def initOutputFormat(jobContext: JobContext): Unit

abstract def initWriter(taskContext: TaskAttemptContext, splitId: Int): Unit

abstract def write(pair: (K, V)): Unit

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped