org.apache.spark.streaming.kafka

Class KafkaCluster

java.lang.Object

org.apache.spark.streaming.kafka.KafkaCluster

All Implemented Interfaces:

java.io.Serializable

public class KafkaCluster
extends java.lang.Object
implements scala.Serializable

:: DeveloperApi :: Convenience methods for interacting with a Kafka cluster. See A Guide To The Kafka Protocol for more details on individual api calls. param: kafkaParams Kafka configuration parameters. Requires "metadata.broker.list" or "bootstrap.servers" to be set with Kafka broker(s), NOT zookeeper servers, specified in host1:port1,host2:port2 form

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	KafkaCluster.LeaderOffset
static class	KafkaCluster.LeaderOffset$
static class	KafkaCluster.SimpleConsumerConfig High-level kafka consumers connect to ZK.
static class	KafkaCluster.SimpleConsumerConfig$

Constructor Summary

Constructors

Constructor and Description
KafkaCluster(scala.collection.immutable.Map<java.lang.String,java.lang.String> kafkaParams)

Method Summary

Methods

Modifier and Type	Method and Description
static <T> T	checkErrors(scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,T> result) If the result is right, return it, otherwise throw SparkException
KafkaCluster.SimpleConsumerConfig	config()
kafka.consumer.SimpleConsumer	connect(java.lang.String host, int port)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,kafka.consumer.SimpleConsumer>	connectLeader(java.lang.String topic, int partition)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.Tuple2<java.lang.String,java.lang.Object>>	findLeader(java.lang.String topic, int partition)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.Tuple2<java.lang.String,java.lang.Object>>>	findLeaders(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>>	getConsumerOffsetMetadata(java.lang.String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions) Requires Kafka >= 0.8.1.1.
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>>	getConsumerOffsetMetadata(java.lang.String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, short consumerApiVersion)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>>	getConsumerOffsets(java.lang.String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions) Requires Kafka >= 0.8.1.1.
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>>	getConsumerOffsets(java.lang.String groupId, scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, short consumerApiVersion)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>>	getEarliestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>>	getLatestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>>	getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, long before)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.collection.Seq<KafkaCluster.LeaderOffset>>>	getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions, long before, int maxNumOffsets)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Set<kafka.api.TopicMetadata>>	getPartitionMetadata(scala.collection.immutable.Set<java.lang.String> topics)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Set<kafka.common.TopicAndPartition>>	getPartitions(scala.collection.immutable.Set<java.lang.String> topics)
scala.collection.immutable.Map<java.lang.String,java.lang.String>	kafkaParams()
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>>	setConsumerOffsetMetadata(java.lang.String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata) Requires Kafka >= 0.8.1.1.
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>>	setConsumerOffsetMetadata(java.lang.String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata, short consumerApiVersion)
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>>	setConsumerOffsets(java.lang.String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object> offsets) Requires Kafka >= 0.8.1.1.
scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>>	setConsumerOffsets(java.lang.String groupId, scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object> offsets, short consumerApiVersion)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

KafkaCluster

public KafkaCluster(scala.collection.immutable.Map<java.lang.String,java.lang.String> kafkaParams)

Method Detail

checkErrors

public static <T> T checkErrors(scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,T> result)

If the result is right, return it, otherwise throw SparkException

kafkaParams

public scala.collection.immutable.Map<java.lang.String,java.lang.String> kafkaParams()

config

public KafkaCluster.SimpleConsumerConfig config()

connect

public kafka.consumer.SimpleConsumer connect(java.lang.String host,
                                    int port)

connectLeader

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,kafka.consumer.SimpleConsumer> connectLeader(java.lang.String topic,
                                                                                                                       int partition)

findLeader

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.Tuple2<java.lang.String,java.lang.Object>> findLeader(java.lang.String topic,
                                                                                                                                      int partition)

findLeaders

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.Tuple2<java.lang.String,java.lang.Object>>> findLeaders(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

getPartitions

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Set<kafka.common.TopicAndPartition>> getPartitions(scala.collection.immutable.Set<java.lang.String> topics)

getPartitionMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Set<kafka.api.TopicMetadata>> getPartitionMetadata(scala.collection.immutable.Set<java.lang.String> topics)

getLatestLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>> getLatestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

getEarliestLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>> getEarliestLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

getLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,KafkaCluster.LeaderOffset>> getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                                     long before)

getLeaderOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,scala.collection.Seq<KafkaCluster.LeaderOffset>>> getLeaderOffsets(scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                                                           long before,
                                                                                                                                                                                                           int maxNumOffsets)

getConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>> getConsumerOffsets(java.lang.String groupId,
                                                                                                                                                                              scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version.

getConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>> getConsumerOffsets(java.lang.String groupId,
                                                                                                                                                                              scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                              short consumerApiVersion)

getConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>> getConsumerOffsetMetadata(java.lang.String groupId,
                                                                                                                                                                                                        scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions)

Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version.

getConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetMetadataAndError>> getConsumerOffsetMetadata(java.lang.String groupId,
                                                                                                                                                                                                        scala.collection.immutable.Set<kafka.common.TopicAndPartition> topicAndPartitions,
                                                                                                                                                                                                        short consumerApiVersion)

setConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>> setConsumerOffsets(java.lang.String groupId,
                                                                                                                                                                              scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object> offsets)

Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version.

setConsumerOffsets

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>> setConsumerOffsets(java.lang.String groupId,
                                                                                                                                                                              scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object> offsets,
                                                                                                                                                                              short consumerApiVersion)

setConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>> setConsumerOffsetMetadata(java.lang.String groupId,
                                                                                                                                                                                     scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata)

Requires Kafka >= 0.8.1.1. Defaults to the original ZooKeeper backed api version.

setConsumerOffsetMetadata

public scala.util.Either<scala.collection.mutable.ArrayBuffer<java.lang.Throwable>,scala.collection.immutable.Map<kafka.common.TopicAndPartition,java.lang.Object>> setConsumerOffsetMetadata(java.lang.String groupId,
                                                                                                                                                                                     scala.collection.immutable.Map<kafka.common.TopicAndPartition,kafka.common.OffsetAndMetadata> metadata,
                                                                                                                                                                                     short consumerApiVersion)