public class DirectKafkaInputDStream<K,V,U extends kafka.serializer.Decoder<K>,T extends kafka.serializer.Decoder<V>,R> extends InputDStream<R> implements Logging
KafkaRDD where
each given Kafka topic/partition corresponds to an RDD partition.
The spark configuration spark.streaming.kafka.maxRatePerPartition gives the maximum number
of messages
per second that each '''partition''' will accept.
Starting offsets are specified in advance,
and this DStream is not responsible for committing offsets,
so that you can control exactly-once semantics.
For an easy interface to Kafka-managed offsets,
see KafkaCluster| Modifier and Type | Class and Description |
|---|---|
class |
DirectKafkaInputDStream.DirectKafkaInputDStreamCheckpointData |
| Constructor and Description |
|---|
DirectKafkaInputDStream(StreamingContext ssc_,
scala.collection.immutable.Map<String,String> kafkaParams,
scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> fromOffsets,
scala.Function1<kafka.message.MessageAndMetadata<K,V>,R> messageHandler,
scala.reflect.ClassTag<K> evidence$1,
scala.reflect.ClassTag<V> evidence$2,
scala.reflect.ClassTag<U> evidence$3,
scala.reflect.ClassTag<T> evidence$4,
scala.reflect.ClassTag<R> evidence$5) |
| Modifier and Type | Method and Description |
|---|---|
scala.Option<KafkaRDD<K,V,U,T,R>> |
compute(Time validTime) |
scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> |
fromOffsets() |
scala.collection.immutable.Map<String,String> |
kafkaParams() |
int |
maxRetries() |
void |
start()
Method called to start receiving data.
|
void |
stop()
Method called to stop receiving data.
|
dependencies, isTimeValid, lastValidTime, slideDurationcache, checkpoint, checkpointDuration, clearCheckpointData, clearMetadata, context, count, countByValue, countByValueAndWindow, countByWindow, creationSite, filter, flatMap, foreach, foreach, foreachRDD, foreachRDD, generatedRDDs, generateJob, getCreationSite, getOrCompute, glom, graph, initialize, isInitialized, map, mapPartitions, mustCheckpoint, parentRememberDuration, persist, persist, print, print, reduce, reduceByWindow, reduceByWindow, register, remember, rememberDuration, repartition, restoreCheckpointData, saveAsObjectFiles, saveAsTextFiles, setContext, setGraph, slice, slice, ssc, storageLevel, toPairDStreamFunctions, transform, transform, transformWith, transformWith, union, updateCheckpointData, validate, window, window, zeroTimeequals, getClass, hashCode, notify, notifyAll, toString, wait, wait, waitinitializeIfNecessary, initializeLogging, isTraceEnabled, log_, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarningpublic DirectKafkaInputDStream(StreamingContext ssc_, scala.collection.immutable.Map<String,String> kafkaParams, scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> fromOffsets, scala.Function1<kafka.message.MessageAndMetadata<K,V>,R> messageHandler, scala.reflect.ClassTag<K> evidence$1, scala.reflect.ClassTag<V> evidence$2, scala.reflect.ClassTag<U> evidence$3, scala.reflect.ClassTag<T> evidence$4, scala.reflect.ClassTag<R> evidence$5)
public scala.collection.immutable.Map<String,String> kafkaParams()
public scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> fromOffsets()
public int maxRetries()
public void start()
InputDStreamstart in class InputDStream<R>public void stop()
InputDStreamstop in class InputDStream<R>