Kafka offset reset

Zealous Zebra answered on December 18, 2021 Popularity 2/10 Helpfulness 1/10

answer Kafka offset reset

related reset kafka offset

related kafka go reset offset

related get pointer to last element in a vector

Kafka offset reset

Comment

1 Listing Topics and Consumer Groups:
kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group nl.logius.bbo.DigikoppelingVerwerkingsverslagAdapter --describe
2 Identifying Earliest Offsets for a Topic:
kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic digikoppeling-events --time -2
Result:
`<topic>:<partition>:<earliest_offset>`
3 Resetting Offsets for a whole Topic:
- Apply the `--reset-offsets` command to shift offsets. 
`reset-offsets` requires one of the following scenarios: `by-duration, to-offset, from-file, to-datetime, to-earliest, to-current, shift-by, to-latest`
kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group nl.logius.bbo.DigikoppelingVerwerkingsverslagAdapter --reset-offsets --shift-by -1 --topic digikoppeling-events --dry-run
4 Partition-Specific Offset Reset:
kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group nl.logius.bbo.DigikoppelingVerwerkingsverslagAdapter --reset-offsets --shift-by -1 --topic digikoppeling-events:8 --dry-run

kafka-consumer-groups.sh --bootstrap-server localhost:9092 --group nl.logius.bbo.DigikoppelingVerwerkingsverslagAdapter --reset-offsets --shift-by -1 --topic digikoppeling-events:8 --execute

Popularity 2/10 Helpfulness 1/10 Language whatever

Source: Grepper

Tags: offset reset whatever

Link to this answer
Share Copy Link

Contributed on Jan 14 2025

Zealous Zebra

0 Answers Avg Quality 2/10

Closely Related Answers

reset kafka offset

Comment

# List the topics to which the group is subscribed
kafka-consumer-groups.sh --bootstrap-server <kafkahost:port> --group <group_id> --describe

# Reset the consumer offset for a topic (dry-run)
kafka-consumer-groups.sh --bootstrap-server <kafkahost:port> --group <group_id> --topic <topic_name> --reset-offsets --to-earliest

# Reset the consumer offset for a topic (execute)
kafka-consumer-groups.sh --bootstrap-server <kafkahost:port> --group <group_id> --topic <topic_name> --reset-offsets --to-earliest --execute

Popularity 9/10 Helpfulness 5/10 Language whatever

Source: gist.github.com

Tags: offset of

Link to this answer
Share Copy Link

Contributed on Dec 18 2021

scott2e

0 Answers Avg Quality 2/10

kafka go reset offset

Comment

package packages

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	kafkaBroker "github.com/segmentio/kafka-go"
)

type (
	Ikafka interface {
		Publisher(key string, topic string, body interface{}) error
		Consumer(topic, groupId string, handler func(message kafkaBroker.Message) error) (*kafkaBroker.Reader, error)
	}

	kafka struct {
		ctx     context.Context
		brokers []string
	}
)

func NewKafka(ctx context.Context, brokers []string) Ikafka {
	return &kafka{ctx: ctx, brokers: brokers}
}

func (h *kafka) Publisher(key string, topic string, body interface{}) error {
	broker := kafkaBroker.Writer{
		Addr:                   kafkaBroker.TCP(h.brokers...),
		Topic:                  topic,
		Compression:            kafkaBroker.Snappy,
		RequiredAcks:           kafkaBroker.RequireAll,
		AllowAutoTopicCreation: true,
		BatchBytes:             1e+9,
		BatchSize:              50,
		MaxAttempts:            10,
		ErrorLogger: kafkaBroker.LoggerFunc(func(s string, i ...interface{}) {
			Logrus("error", s)
		}),
	}

	bodyByte, err := json.Marshal(body)
	if err != nil {
		return err
	}

	msg := kafkaBroker.Message{Key: []byte(key), Value: bodyByte}

	if err := broker.WriteMessages(h.ctx, msg); err != nil {
		return err
	}

	return nil
}

func (h *kafka) Consumer(topic, groupId string, handler func(message kafkaBroker.Message) error) (*kafkaBroker.Reader, error) {
	reader := kafkaBroker.NewReader(kafkaBroker.ReaderConfig{
		Brokers:                h.brokers,
		Topic:                  topic,
		GroupID:                groupId,
		WatchPartitionChanges:  true,
		OffsetOutOfRangeError:  true,
		MaxBytes:               1e+9,
		MaxAttempts:            10,
		PartitionWatchInterval: 3,
		JoinGroupBackoff:       3,
		ErrorLogger: kafkaBroker.LoggerFunc(func(s string, i ...interface{}) {
			Logrus("error", s)
		}),
	})

	message, err := reader.FetchMessage(h.ctx)
	if err != nil {
		return nil, err
	}

	if err := handler(message); err != nil {
		return nil, err
	}

	if err := reader.CommitMessages(h.ctx, message); err != nil {
		return nil, err
	}

	// if err := h.ResetOffset(topic, groupId); err != nil {
	// 	return nil, err
	// }

	return reader, nil
}

func (h *kafka) ResetOffset(topic, groupId string) error {
	broker, err := kafkaBroker.DialContext(h.ctx, "tcp", h.brokers[0])
	defer broker.Close()
	if err != nil {
		return err
	}

	partitions, err := broker.ReadPartitions(topic)
	if err != nil {
		return err
	}

	offsets := make(map[int]int64, len(partitions))

	for _, partition := range partitions {
		host := fmt.Sprintf("%s:%d", partition.Leader.Host, partition.Leader.Port)

		connection, err := kafkaBroker.DialLeader(h.ctx, "tcp", host, topic, partition.ID)
		if err != nil {
			return err
		}

		defer connection.Close()
		offset, err := connection.ReadOffset(time.Unix(int64(time.Now().Second()), 0))
		if err != nil {
			return err
		}

		offsets[partition.ID] = offset
	}

	group, err := kafkaBroker.NewConsumerGroup(kafkaBroker.ConsumerGroupConfig{
		Brokers: h.brokers,
		Topics:  []string{topic},
		ID:      groupId,
	})

	if err != nil {
		return err
	}

	defer group.Close()
	groupNext, err := group.Next(h.ctx)
	if err != nil {
		return err
	}

	err = groupNext.CommitOffsets(map[string]map[int]int64{topic: offsets})
	if err != nil {
		return err
	}

	return nil
}

Popularity 6/10 Helpfulness 1/10 Language go

Source: Grepper

Tags: go offset reset

Link to this answer
Share Copy Link

Contributed on May 27 2024

Restu Wahyu Saputra

0 Answers Avg Quality 2/10

get pointer to last element in a vector

Comment

For dynamic arrays, we retrieve the last element of the vector and return its memory address.For static arrays, we simply retrieve the memory address of its last element.

#include <vector> // If you're using dynamic arrays
#include <memory>

// Dynamic arrays
std::vector<int> s = {1, 2, 3, 4, 5};
int *ptr = &s[s.size() - 1]; // Pointer to the last element

// Static arrays
int *s = new int[...];
int *ptr = &(*s)[last_element_number];

More DetailsFor dynamic arrays, we evaluate std::vector::size(), substact one, get the `result`th element of the vector and then set the pointer to its memory address. (&vector_name[vector_name.size() - 1])

For static arrays, we get the last value of the array and set the pointer to its memory address. (&(*array_name)[last_element_number])

Popularity 1/10 Helpfulness 1/10 Language javascript

Source: Grepper

Tags: c++ element get vector

Link to this answer
Share Copy Link

Contributed on Jan 12 2025

Programmer

0 Answers Avg Quality 2/10

Kafka offset reset

Contents

More Related Answers

Kafka offset reset

Closely Related Answers

reset kafka offset

kafka go reset offset

get pointer to last element in a vector

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.