Create Custom Partitioner Class

Pragya Keshap answered on February 11, 2023 Popularity 1/10 Helpfulness 1/10

answer Create Custom Partitioner Class

Create Custom Partitioner Class

Comment

import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.common.PartitionInfo;
import io.github.fernanda.maia.kafka.avro.Track;
import org.apache.kafka.clients.producer.Partitioner;
import java.util.List;
import java.util.Map;
public class TrackPartitioner implements Partitioner {
@Override
public int partition(String topic, Object key, byte[] keyBytes, Object
value, byte[] valueBytes, Cluster cluster) {
Track record = (Track) value;
int partitionNumber = 5;
if(record.getLongitude() != "115.793W" || record.getLatitude() !=
"37.2431N") {
List<PartitionInfo> infoList =
cluster.availablePartitionsForTopic("TrackPartitionsTopic");
partitionNumber = Math.abs(Utils.murmur2(keyBytes) % infoList.size() -
1);
}
return partitionNumber;
}
@Override
public void close() {
}
@Override
public void configure(Map<String, ?> map) {
}
}
import io.confluent.kafka.serializers.KafkaAvroSerializer;
import io.github.fernanda.maia.kafka.avro.Track;
import io.github.fernanda.maia.partitioner.TrackPartitioner;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.util.Properties;
public class TrackProducer {
public static void main(String[] args) {
Properties props = new Properties();
props.setProperty("bootstrap.servers", "http://localhost:9092");
props.setProperty("key.serializer", KafkaAvroSerializer.class.getName());
props.setProperty("value.serializer",
KafkaAvroSerializer.class.getName());
props.setProperty("schema.registry.url", "http://localhost:8081");
props.setProperty("partitioner.class", TrackPartitioner.class.getName());
try(KafkaProducer<Long, Track> producer = new
KafkaProducer<>(props)) {
Track track = Track.newBuilder()
.setId(1L)
.setLatitude("37.2431N")
.setLongitude("115.793W")
.build();
ProducerRecord<Long, Track> record = new
ProducerRecord<>("TrackPartitionsTopic", track.getId(), track);
producer.send(record);
} catch(Exception e) {
e.printStackTrace();
}
}
}
import io.confluent.kafka.serializers.KafkaAvroDeserializer;
import io.github.fernanda.maia.kafka.avro.Track;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
public class TrackConsumer {
public static void main(String[] args) {
Properties props = new Properties();
props.setProperty("bootstrap.servers", "http://localhost:9092");
props.setProperty("key.deserializer",
KafkaAvroDeserializer.class.getName());
props.setProperty("value.deserializer",
KafkaAvroDeserializer.class.getName());
props.setProperty("schema.registry.url", "http://localhost:8081");
props.setProperty("group.id", "TrackGroup");
props.setProperty("specific.avro.reader", "true");
try(KafkaConsumer<Long, Track> consumer = new
KafkaConsumer<>(props)) {
consumer.subscribe(Arrays.asList("TrackAvroTopic",
"TrackPartitionsTopic"));
while(true) {
ConsumerRecords<Long, Track> records =
consumer.poll(Duration.ofSeconds(20));
records.forEach(c -> {
Track coordinates = c.value();
System.out.println("\nID: " + c.key());
System.out.println("LATITUDE: " + coordinates.getLatitude());
System.out.println("LONGITUDE: " + coordinates.getLongitude());
System.out.println("PARTITION: " + c.partition());
});
}
} catch(Exception e) {
e.printStackTrace();
}
}
}

https://www.clairvoyant.ai/blog/writing-custom-partitioner-for-apache-kafka

Popularity 1/10 Helpfulness 1/10 Language java

Source: Grepper

Tags: class java partitioner

Link to this answer
Share Copy Link

Contributed on Feb 11 2023

Pragya Keshap

0 Answers Avg Quality 2/10

Create Custom Partitioner Class

Contents

More Related Answers

Create Custom Partitioner Class

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.