kafka 集群迁移：topic原数据备份

• 使用场景： kafka集群迁移，或备份；涉及topic元数据批量备份
  

  文章目录

  • 1, 调试过程
  • 2, 备份脚本

1, 调试过程

./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --list > /tmp/kaf-top.txt
#test1
#test2

./bin/kafka-topics.sh --bootstrap-server kafka01:9092  --describe --topic test1 |grep 'PartitionCount' 
#Topic: test1  TopicId: akSpXlQ PartitionCount: 1       ReplicationFactor: 2    Configs: segment.bytes=1073741824,max.message.bytes=1048576

./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --describe --topic test1 |grep 'PartitionCount'  |awk '{print $2,$6,$8,$10}'
#test1 1 2 segment.bytes=1073741824,max.message.bytes=1048576

./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --describe --topic test1  |grep PartitionCount |awk '{print $2,$6,$8,","$10}' |sed 's@,@ --config @g '
#test1 1 2  --config segment.bytes=1073741824 --config max.message.bytes=1048576

./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --describe --topic test1 |grep PartitionCount  | awk '{print "./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --create --topic "$2 " --partitions "$6" --replication-factor "$8 " ,"$10}'  |sed 's@,@ --config @g '
#./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --create --topic test1 --partitions 1 --replication-factor 2  --config segment.bytes=1073741824 --config max.message.bytes=1048576

2, 备份脚本

./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --list > /tmp/kaf-top.txt
for topic in `cat /tmp/kaf-top.txt`
do
 echo $topic
 #./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --describe --topic $topic |grep PartitionCount  | awk '{print "./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --create --topic "$2 " --partitions "$6" --replication-factor "$8 }'   >> /tmp/kaf-top-create.sh
 #输出：./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --create --topic test1 --partitions 1 --replication-factor 2 
 ./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --describe --topic $topic |grep PartitionCount  | awk '{print "./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --create --topic "$2 " --partitions "$6" --replication-factor "$8 " ,"$10}'  |sed 's@,@ --config @g ' >> /tmp/kaf-top-create2.sh
 #输出：./bin/kafka-topics.sh --bootstrap-server kafka01:9092 --create --topic test1 --partitions 1 --replication-factor 2  --config segment.bytes=1073741824 --config retention.ms=86400000 --config max.message.bytes=10485760
done