6月 142019
 

下载Kafka安装包

[root@ip-172-31-13-64 ~]# curl -O https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.4.1/kafka_2.12-2.4.1.tgz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 59.4M  100 59.4M    0     0  1520k      0  0:00:40  0:00:40 --:--:-- 3598k
[root@ip-172-31-13-64 ~]#

使用rz上传JDK并安装

[root@ip-172-31-13-64 ~]# yum -y install lrzsz
[root@ip-172-31-13-64 ~]# rz
rz waiting to receive.
Starting zmodem transfer.  Press Ctrl+C to cancel.
Transferring jdk-8u241-linux-x64.rpm...
  100%  174745 KB     606 KB/sec    00:04:48       0 Errors  

[root@ip-172-31-13-64 ~]#
[root@ip-172-31-13-64 ~]# yum -y install jdk-8u241-linux-x64.rpm

修改配置文件

[root@ip-172-31-13-64 ~]# tar xzf kafka_2.12-2.4.1.tgz
[root@ip-172-31-13-64 ~]# cd kafka_2.12-2.4.1/config/
[root@ip-172-31-13-64 config]# vi server.properties 
broker.id=0
listeners=PLAINTEXT://:9092
zookeeper.connect=172.31.6.164:2181,172.31.2.131:2181,172.31.7.9:2181

[root@ip-172-31-15-223 ~]# tar xzf kafka_2.12-2.4.1.tgz 
[root@ip-172-31-15-223 ~]# cd kafka_2.12-2.4.1/config/
[root@ip-172-31-15-223 config]# vi server.properties
broker.id=1
listeners=PLAINTEXT://:9092
zookeeper.connect=172.31.6.164:2181,172.31.2.131:2181,172.31.7.9:2181

[root@ip-172-31-8-209 ~]# tar xzf kafka_2.12-2.4.1.tgz 
[root@ip-172-31-8-209 ~]# cd kafka_2.12-2.4.1/config/
[root@ip-172-31-8-209 config]# vi server.properties 
broker.id=2
listeners=PLAINTEXT://:9092
zookeeper.connect=172.31.6.164:2181,172.31.2.131:2181,172.31.7.9:2181

启动服务

[root@ip-172-31-13-64 ~]# cd kafka_2.12-2.4.1
[root@ip-172-31-13-64 kafka_2.12-2.4.1]# nohup ./bin/kafka-server-start.sh config/server.properties &
[1] 18985
[root@ip-172-31-13-64 kafka_2.12-2.4.1]# nohup: ignoring input and appending output to ‘nohup.out’

[root@ip-172-31-13-64 kafka_2.12-2.4.1]#


[root@ip-172-31-15-223 ~]# cd kafka_2.12-2.4.1
[root@ip-172-31-15-223 kafka_2.12-2.4.1]# nohup ./bin/kafka-server-start.sh config/server.properties &
[1] 18161
[root@ip-172-31-15-223 kafka_2.12-2.4.1]# nohup: ignoring input and appending output to ‘nohup.out’

[root@ip-172-31-15-223 kafka_2.12-2.4.1]#


[root@ip-172-31-8-209 ~]# cd kafka_2.12-2.4.1
[root@ip-172-31-8-209 kafka_2.12-2.4.1]# nohup ./bin/kafka-server-start.sh config/server.properties &
[4] 17614
[root@ip-172-31-8-209 kafka_2.12-2.4.1]# nohup: ignoring input and appending output to ‘nohup.out’

[root@ip-172-31-8-209 kafka_2.12-2.4.1]#

启动后的日志输出

[root@ip-172-31-13-64 kafka_2.12-2.4.1]# tail -f nohup.out 
[2020-04-13 08:23:29,826] INFO [TransactionCoordinator id=0] Starting up. (kafka.coordinator.transaction.TransactionCoordinator)
[2020-04-13 08:23:29,827] INFO [TransactionCoordinator id=0] Startup complete. (kafka.coordinator.transaction.TransactionCoordinator)
[2020-04-13 08:23:29,834] INFO [Transaction Marker Channel Manager 0]: Starting (kafka.coordinator.transaction.TransactionMarkerChannelManager)
[2020-04-13 08:23:29,882] INFO [ExpirationReaper-0-AlterAcls]: Starting (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)
[2020-04-13 08:23:29,903] INFO [/config/changes-event-process-thread]: Starting (kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread)
[2020-04-13 08:23:29,941] INFO [SocketServer brokerId=0] Started data-plane processors for 1 acceptors (kafka.network.SocketServer)
[2020-04-13 08:23:29,945] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:23:29,945] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:23:29,945] INFO Kafka startTimeMs: 1586766209941 (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:23:29,946] INFO [KafkaServer id=0] started (kafka.server.KafkaServer)

[root@ip-172-31-15-223 kafka_2.12-2.4.1]# tail -f nohup.out 
[2020-04-13 08:23:55,073] INFO [TransactionCoordinator id=1] Starting up. (kafka.coordinator.transaction.TransactionCoordinator)
[2020-04-13 08:23:55,075] INFO [TransactionCoordinator id=1] Startup complete. (kafka.coordinator.transaction.TransactionCoordinator)
[2020-04-13 08:23:55,081] INFO [Transaction Marker Channel Manager 1]: Starting (kafka.coordinator.transaction.TransactionMarkerChannelManager)
[2020-04-13 08:23:55,118] INFO [ExpirationReaper-1-AlterAcls]: Starting (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)
[2020-04-13 08:23:55,138] INFO [/config/changes-event-process-thread]: Starting (kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread)
[2020-04-13 08:23:55,164] INFO [SocketServer brokerId=1] Started data-plane processors for 1 acceptors (kafka.network.SocketServer)
[2020-04-13 08:23:55,168] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:23:55,168] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:23:55,168] INFO Kafka startTimeMs: 1586766235165 (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:23:55,179] INFO [KafkaServer id=1] started (kafka.server.KafkaServer)

[root@ip-172-31-8-209 kafka_2.12-2.4.1]# tail -f nohup.out 
[2020-04-13 08:24:30,928] INFO [TransactionCoordinator id=2] Starting up. (kafka.coordinator.transaction.TransactionCoordinator)
[2020-04-13 08:24:30,929] INFO [Transaction Marker Channel Manager 2]: Starting (kafka.coordinator.transaction.TransactionMarkerChannelManager)
[2020-04-13 08:24:30,929] INFO [TransactionCoordinator id=2] Startup complete. (kafka.coordinator.transaction.TransactionCoordinator)
[2020-04-13 08:24:30,967] INFO [ExpirationReaper-2-AlterAcls]: Starting (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)
[2020-04-13 08:24:30,991] INFO [/config/changes-event-process-thread]: Starting (kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread)
[2020-04-13 08:24:31,008] INFO [SocketServer brokerId=2] Started data-plane processors for 1 acceptors (kafka.network.SocketServer)
[2020-04-13 08:24:31,012] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:24:31,012] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:24:31,012] INFO Kafka startTimeMs: 1586766271008 (org.apache.kafka.common.utils.AppInfoParser)
[2020-04-13 08:24:31,013] INFO [KafkaServer id=2] started (kafka.server.KafkaServer)

查看运行状态

[root@ip-172-31-13-64 ~]# jps
18985 Kafka
23677 Jps
[root@ip-172-31-13-64 ~]#

[root@ip-172-31-15-223 ~]# jps
18161 Kafka
22820 Jps
[root@ip-172-31-15-223 ~]#

[root@ip-172-31-8-209 ~]# jps
22231 Jps
17614 Kafka
[root@ip-172-31-8-209 ~]#

创建测试主题

./bin/kafka-topics.sh --create --bootstrap-server localhost:9092 \
--replication-factor 3 --partitions 1 --topic my-replicated-topic

[root@ip-172-31-13-64 kafka_2.12-2.4.1]# ./bin/kafka-topics.sh --create --bootstrap-server localhost:9092 \
> --replication-factor 3 --partitions 1 --topic my-replicated-topic
[root@ip-172-31-13-64 kafka_2.12-2.4.1]#

查询测试主题描述

bin/kafka-topics.sh --describe --bootstrap-server localhost:9092 \
--topic my-replicated-topic

[root@ip-172-31-13-64 kafka_2.12-2.4.1]# bin/kafka-topics.sh --describe --bootstrap-server localhost:9092 \
> --topic my-replicated-topic
Topic: my-replicated-topic PartitionCount: 1 ReplicationFactor: 3 Configs: segment.bytes=1073741824
Topic: my-replicated-topic Partition: 0 Leader: 0 Replicas: 0,1,2 Isr: 0,1,2
[root@ip-172-31-13-64 kafka_2.12-2.4.1]#

[root@ip-172-31-15-223 kafka_2.12-2.4.1]# bin/kafka-topics.sh --describe --bootstrap-server localhost:9092 \
> --topic my-replicated-topic
Topic: my-replicated-topic PartitionCount: 1 ReplicationFactor: 3 Configs: segment.bytes=1073741824
Topic: my-replicated-topic Partition: 0 Leader: 0 Replicas: 0,1,2 Isr: 0,1,2
[root@ip-172-31-15-223 kafka_2.12-2.4.1]#

[root@ip-172-31-8-209 kafka_2.12-2.4.1]# bin/kafka-topics.sh --describe --bootstrap-server localhost:9092 \
> --topic my-replicated-topic
Topic: my-replicated-topic PartitionCount: 1 ReplicationFactor: 3 Configs: segment.bytes=1073741824
Topic: my-replicated-topic Partition: 0 Leader: 0 Replicas: 0,1,2 Isr: 0,1,2
[root@ip-172-31-8-209 kafka_2.12-2.4.1]#

查询可订阅主题列表

[root@ip-172-31-13-64 kafka_2.12-2.4.1]# ./bin/kafka-topics.sh --list --zookeeper 172.31.6.164:2181,172.31.2.131:2181,172.31.7.9:2181
my-replicated-topic
[root@ip-172-31-13-64 kafka_2.12-2.4.1]#