Kafka消费者批次控制：基于字节大小优化poll()行为

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.Properties;
import java.util.Collections;

public class ByteBasedKafkaConsumer {

    public static void main(String[] args) {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "my_byte_based_group");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

        // 设置最大拉取字节数为 1MB (1 * 1024 * 1024 字节)
        props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, 1 * 1024 * 1024); // 1MB

        // 将 MAX_POLL_RECORDS_CONFIG 设置为一个非常大的值，使其不成为主要限制
        // 例如，Integer.MAX_VALUE 或一个远超预期单次处理记录数的上限
        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, Integer.MAX_VALUE); 

        // 也可以设置 FETCH_MIN_BYTES_CONFIG 来控制最小拉取量，避免小批量频繁拉取
        // props.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, 1024); // 例如，至少拉取 1KB

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Collections.singletonList("my_topic"));

        try {
            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
                if (!records.isEmpty()) {
                    System.out.println("Pulled " + records.count() + " records.");
                    long totalBytes = 0;
                    for (ConsumerRecord<String, String> record : records) {
                        // 注意: record.serializedValueSize() 和 record.serializedKeySize() 
                        // 估算的是序列化后的字节大小，实际网络传输可能包含额外开销
                        totalBytes += record.serializedValueSize() + record.serializedKeySize();
                        // 处理 record...
                    }
                    System.out.println("Total bytes in this poll: " + totalBytes + " bytes.");
                    consumer.commitSync(); // 提交偏移量
                }
            }
        } finally {
            consumer.close();
        }
    }
}