升级kafka4.0.0，无ZK版本

发布于：2025-05-17 ⋅ 阅读:(13) ⋅ 点赞:(0)

设备规划：

172.20.192.47 kafka-0

172.20.192.48 kafka-1

172.20.192.49 kafka-2

单机块7TB Nvme磁盘一共9块

# 格式化成GPT分区
sudo parted /dev/nvme0n1 --script mklabel gpt
sudo parted /dev/nvme1n1 --script mklabel gpt
sudo parted /dev/nvme2n1 --script mklabel gpt
# 磁盘格式xfs
sudo mkfs.xfs -f /dev/nvme0n1
sudo mkfs.xfs -f /dev/nvme1n1
sudo mkfs.xfs -f /dev/nvme2n1

mkdir -p /data/kafka0
mkdir -p /data/kafka1
mkdir -p /data/kafka2

# 修改/etc/fstab
/dev/nvme0n1 /data/kafka0 xfs defaults 0 0
/dev/nvme1n1 /data/kafka1 xfs defaults 0 0
/dev/nvme2n1 /data/kafka2 xfs defaults 0 0

# 测试mount
sudo mount -a

# 检测
df -hT

环境设置 

-e KAFKA_CFG_DEFAULT_REPLICATION_FACTOR=3 \  # 3副本
-e KAFKA_CFG_NUM_PARTITIONS=2 \  # 2分区，吞吐量大的可以调高


# 增加分区数（例如将 test-topic 从 3 个分区增加到 6 个）
kafka-topics.sh --bootstrap-server localhost:9092 --alter --topic test-topic --partitions 6

需要严格保证消息顺序的队列需要设置
kafka-topics.sh --bootstrap-server localhost:9092 --alter --topic index-topic --partitions 1

docker run -d \
--name kafka-0 \
--user root \
-p 9092:9092 \
-p 9093:9093 \
-e KAFKA_CFG_NODE_ID=0 \
-e KAFKA_CFG_PROCESS_ROLES=controller,broker \
-e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@172.20.192.47:9093,1@172.20.192.48:9093,2@172.20.192.49:9093 \
-e KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv \
-e KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
-e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://172.20.192.47:9092 \
-e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT \
-e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-e KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT \
-e KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 \
-e KAFKA_CFG_LOG_DIRS=/bitnami/kafka/data0,/bitnami/kafka/data1,/bitnami/kafka/data2 \
-e KAFKA_CFG_CONTROLLER_ELECTION_TIMEOUT_MS=50000 \
-e KAFKA_CFG_CONTROLLER_STARTUP_TIMEOUT_MS=60000 \
-e KAFKA_CFG_DEFAULT_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_NUM_PARTITIONS=2 \
-v /data/kafka0:/bitnami/kafka/data0 \
-v /data/kafka1:/bitnami/kafka/data1 \
-v /data/kafka2:/bitnami/kafka/data2 \
bitnami/kafka:4.0.0


docker run -d \
--name kafka-1 \
--user root \
-p 9092:9092 \
-p 9093:9093 \
-e KAFKA_CFG_NODE_ID=1 \
-e KAFKA_CFG_PROCESS_ROLES=controller,broker \
-e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@172.20.192.47:9093,1@172.20.192.48:9093,2@172.20.192.49:9093 \
-e KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv \
-e KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
-e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://172.20.192.48:9092 \
-e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT \
-e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-e KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT \
-e KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 \
-e KAFKA_CFG_LOG_DIRS=/bitnami/kafka/data0,/bitnami/kafka/data1,/bitnami/kafka/data2 \
-e KAFKA_CFG_CONTROLLER_ELECTION_TIMEOUT_MS=50000 \
-e KAFKA_CFG_CONTROLLER_STARTUP_TIMEOUT_MS=60000 \
-e KAFKA_CFG_DEFAULT_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_NUM_PARTITIONS=2 \
-v /data/kafka0:/bitnami/kafka/data0 \
-v /data/kafka1:/bitnami/kafka/data1 \
-v /data/kafka2:/bitnami/kafka/data2 \
bitnami/kafka:4.0.0


docker run -d \
--name kafka-2 \
--user root \
-p 9092:9092 \
-p 9093:9093 \
-e KAFKA_CFG_NODE_ID=2 \
-e KAFKA_CFG_PROCESS_ROLES=controller,broker \
-e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@172.20.192.47:9093,1@172.20.192.48:9093,2@172.20.192.49:9093 \
-e KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv \
-e KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
-e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://172.20.192.49:9092 \
-e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT \
-e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-e KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT \
-e KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2 \
-e KAFKA_CFG_LOG_DIRS=/bitnami/kafka/data0,/bitnami/kafka/data1,/bitnami/kafka/data2 \
-e KAFKA_CFG_CONTROLLER_ELECTION_TIMEOUT_MS=50000 \
-e KAFKA_CFG_CONTROLLER_STARTUP_TIMEOUT_MS=60000 \
-e KAFKA_CFG_DEFAULT_REPLICATION_FACTOR=3 \
-e KAFKA_CFG_NUM_PARTITIONS=2 \
-v /data/kafka0:/bitnami/kafka/data0 \
-v /data/kafka1:/bitnami/kafka/data1 \
-v /data/kafka2:/bitnami/kafka/data2 \
bitnami/kafka:4.0.0

上面部署好以后可以加个监控，可以在设备IP:9000查看监控

http://172.20.192.204:9000

docker run -d \
  --name kafka-ui \
  -p 9000:8080 \
  -e KAFKA_CLUSTERS_0_NAME=local \
  -e KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=172.20.192.47:9092,172.20.192.48:9092,172.20.192.49:9092 \
  -e KAFKA_CLUSTERS_0_CONTROLLERSERVERS=172.20.192.47:9093,172.20.192.48:9093,172.20.192.49:9093 \
  provectuslabs/kafka-ui:latest

 docker update --restart=always kafka-ui

网站公告

今日签到

点亮在社区的每一天
去签到

热门文章

最新发布