org.apache.spark.streaming.kafka

Class KafkaCluster

Object

org.apache.spark.streaming.kafka.KafkaCluster

All Implemented Interfaces:

java.io.Serializable

public class KafkaCluster
extends Object
implements scala.Serializable

:: DeveloperApi :: Convenience methods for interacting with a Kafka cluster. See A Guide To The Kafka Protocol for more details on individual api calls. param: kafkaParams Kafka configuration parameters. Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), NOT zookeeper servers, specified in host1:port1,host2:port2 form

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	KafkaCluster.LeaderOffset
static class	KafkaCluster.LeaderOffset$
static class	KafkaCluster.SimpleConsumerConfig High-level kafka consumers connect to ZK.
static class	KafkaCluster.SimpleConsumerConfig$

Constructor Summary

Constructors

Constructor and Description
KafkaCluster(scala.collection.immutable.Map<String,String> kafkaParams)

Method Summary

Modifier and Type	Method and Description
static <T> T	checkErrors(scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,T> result) If the result is right, return it, otherwise throw SparkException
KafkaCluster.SimpleConsumerConfig	config()
kafka.consumer.SimpleConsumer	connect(String host, int port)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,kafka.consumer.SimpleConsumer>	connectLeader(String topic, int partition)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.Tuple2<String,Object>>	findLeader(String topic, int partition)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.Tuple2<String,Object>>>	findLeaders(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>>	getConsumerOffsetMetadata(String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions) Requires Kafka 0.8.1.1 or later.
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>>	getConsumerOffsetMetadata(String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, short consumerApiVersion)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>>	getConsumerOffsets(String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions) Requires Kafka 0.8.1.1 or later.
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>>	getConsumerOffsets(String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, short consumerApiVersion)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>>	getEarliestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>>	getLatestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>>	getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, long before)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.collection.Seq<KafkaCluster.LeaderOffset>>>	getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, long before, int maxNumOffsets)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Set<kafka.api.TopicMetadata>>	getPartitionMetadata(scala.collection.immutable.Set<String> topics)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Set<kafka.common.TopicAndPartition>>	getPartitions(scala.collection.immutable.Set<String> topics)
scala.collection.immutable.Map<String,String>	kafkaParams()
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>>	setConsumerOffsetMetadata(String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata) Requires Kafka 0.8.1.1 or later.
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>>	setConsumerOffsetMetadata(String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata, short consumerApiVersion)
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>>	setConsumerOffsets(String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> offsets) Requires Kafka 0.8.1.1 or later.
scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>>	setConsumerOffsets(String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> offsets, short consumerApiVersion)

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

KafkaCluster

public KafkaCluster(scala.collection.immutable.Map<String,String> kafkaParams)

Method Detail

checkErrors

public static <T> T checkErrors(scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,T> result)

If the result is right, return it, otherwise throw SparkException

kafkaParams

public scala.collection.immutable.Map<String,String> kafkaParams()

config

public KafkaCluster.SimpleConsumerConfig config()

connect

public kafka.consumer.SimpleConsumer connect(String host,
                                             int port)

connectLeader

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,kafka.consumer.SimpleConsumer> connectLeader(String topic,
                                                                                                                      int partition)

findLeader

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.Tuple2<String,Object>> findLeader(String topic,
                                                                                                                 int partition)

findLeaders

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.Tuple2<String,Object>>> findLeaders(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

getPartitions

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Set<kafka.common.TopicAndPartition>> getPartitions(scala.collection.immutable.Set<String> topics)

getPartitionMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Set<kafka.api.TopicMetadata>> getPartitionMetadata(scala.collection.immutable.Set<String> topics)

getLatestLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>> getLatestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

getEarliestLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>> getEarliestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

getLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>> getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                                    long before)

getLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.collection.Seq<KafkaCluster.LeaderOffset>>> getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                                                          long before,
                                                                                                                                                                                                          int maxNumOffsets)

getConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>> getConsumerOffsets(String groupId,
                                                                                                                                                                   scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

Requires Kafka 0.8.1.1 or later. Defaults to the original ZooKeeper backed API version.

Parameters:

groupId - (undocumented)

topicAndPartitions - (undocumented)

Returns:

(undocumented)

getConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>> getConsumerOffsets(String groupId,
                                                                                                                                                                   scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                   short consumerApiVersion)

getConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>> getConsumerOffsetMetadata(String groupId,
                                                                                                                                                                                                       scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

Requires Kafka 0.8.1.1 or later. Defaults to the original ZooKeeper backed API version.

Parameters:

groupId - (undocumented)

topicAndPartitions - (undocumented)

Returns:

(undocumented)

getConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>> getConsumerOffsetMetadata(String groupId,
                                                                                                                                                                                                       scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                                                       short consumerApiVersion)

setConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>> setConsumerOffsets(String groupId,
                                                                                                                                                                   scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> offsets)

Requires Kafka 0.8.1.1 or later. Defaults to the original ZooKeeper backed API version.

Parameters:

groupId - (undocumented)

offsets - (undocumented)

Returns:

(undocumented)

setConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>> setConsumerOffsets(String groupId,
                                                                                                                                                                   scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object> offsets,
                                                                                                                                                                   short consumerApiVersion)

setConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>> setConsumerOffsetMetadata(String groupId,
                                                                                                                                                                          scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata)

Requires Kafka 0.8.1.1 or later. Defaults to the original ZooKeeper backed API version.

Parameters:

groupId - (undocumented)

metadata - (undocumented)

Returns:

(undocumented)

setConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,Object>> setConsumerOffsetMetadata(String groupId,
                                                                                                                                                                          scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata,
                                                                                                                                                                          short consumerApiVersion)