Flume消费Kafka数据到HDFS

最新推荐文章于 2023-09-07 14:49:45 发布

原创最新推荐文章于 2023-09-07 14:49:45 发布 · 184 阅读

本内容遵循CC 4.0 BY-SA版权协议

本文介绍了如何使用Flume从Kafka消费数据，并将其存储到HDFS。配置包括Source、Channel和Sink三个部分，Source部分设置了两个KafkaSource分别消费不同主题的数据，Channel使用内存型通道，Sink部分则将数据写入HDFS并进行文件滚动策略的设定。

组件定义

a2.sources=r1 r2
a2.channels=c1 c2
a2.sinks=k1 k2

Source配置

a2.sources.r1.type = org.apache.flume.source.kafka.KafkaSource
a2.sources.r1.batchSize = 5000
a2.sources.r1.batchDurationMillis = 2000
a2.sources.r1.kafka.bootstrap.servers = node-201:9092
a2.sources.r1.kafka.zookeeperConnect = node-201:2181
a2.sources.r1.kafka.topics=topic_start

a2.sources.r2.type = org.apache.flume.source.kafka.KafkaSource
a2.sources.r2.batchSize = 5000
a2.sources.r2.batchDurationMillis = 2000
a2.sources.r2.kafka.bootstrap.servers = node-201:9092
a2.sources.r2.kafka.zookeeperConnect = node-201:2181
a2.sources.r2.kafka.topics=topic_event

Channel配置

a2.channels.c1.type=memory
a2.channels.c1.capacity=100000
a2.channels.c1.transactionCapacity=10000

a2.channels.c2.type=memory
a2.channels.c2.capacity=100000
a2.channels.c2.transactionCapacity=10000

Sink配置

a2.sinks.k1.type = hdfs
a2.sinks.k1.hdfs.path = /origin_data/gmall/log/topic_start/%Y-%m-%d
a2.sinks.k1.hdfs.filePrefix = logstart-
a2.sinks.k1.hdfs.round = true
a2.sinks.k1.hdfs.roundValue = 30
a2.sinks.k1.hdfs.roundUnit = second

a2.sinks.k2.type = hdfs
a2.sinks.k2.hdfs.path = /origin_data/gmall/log/topic_event/%Y-%m-%d
a2.sinks.k2.hdfs.filePrefix = logevent-
a2.sinks.k2.hdfs.round = true
a2.sinks.k2.hdfs.roundValue = 30
a2.sinks.k2.hdfs.roundUnit = second

a2.sinks.k1.hdfs.rollInterval = 30
a2.sinks.k1.hdfs.rollSize = 0
a2.sinks.k1.hdfs.rollCount = 0

a2.sinks.k2.hdfs.rollInterval = 30
a2.sinks.k2.hdfs.rollSize = 0
a2.sinks.k2.hdfs.rollCount = 0

a2.sinks.k1.hdfs.fileType = CompressedStream
a2.sinks.k2.hdfs.fileType = CompressedStream

a2.sinks.k1.hdfs.codeC = lzop
a2.sinks.k2.hdfs.codeC = lzop