采用 Spring Integration Kafka实现高吞吐量的分布式发布订阅消息

小灰灰 2021-12-15 00:59 241阅读 0赞

> ### 采用 Spring Integration Kafka实现高吞吐量的分布式发布订阅消息 ###

**【服务端配置】**：Apache官网下最新的二进制版本：kafka\_2.11-2.3.0(内含有zookeeper)

（1）修改config/server.properties 里面的值：

listeners=PLAINTEXT://HOST_NAME(IP):9092
    advertised.listeners=PLAINTEXT://HOST_NAME(IP):9092

其他配置默认

（2） 启动zookeeper:

./bin/zookeeper-server.start.sh ../config/zookeeper.properties &

（3）启动Kafka

./bin/kafka-server-start.sh ../config/server.properties &

（4）创建主题Topic -- myTopic

.bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 4 --topic myTopic

（5） 查看服务端所创建的topic有哪些

.bin/kafka-topics.sh --list --zookeeper localhost:2181

**【pom.xml】jar包依赖：**

<dependency>
    	<groupId>org.springframework.integration</groupId>
    	<artifactId>spring-integration-kafka</artifactId>
    	<version>1.3.0.RELEASE</version>
    </dependency>

**【生产消费（发布/订阅）配置】 ：**

**spring-kafka-consumer.xml:**

<?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
    	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:int="http://www.springframework.org/schema/integration"
    	xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
    	xmlns:task="http://www.springframework.org/schema/task"
    	xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka 
                            http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
                            http://www.springframework.org/schema/integration 
                            http://www.springframework.org/schema/integration/spring-integration.xsd
                            http://www.springframework.org/schema/beans 
                            http://www.springframework.org/schema/beans/spring-beans.xsd
                            http://www.springframework.org/schema/task 
                            http://www.springframework.org/schema/task/spring-task.xsd">
    
    	
    	<int:channel id="inputFromKafka" >
    		<int:dispatcher task-executor="kafkaMessageExecutor" />
    	</int:channel>
        
    	<int-kafka:zookeeper-connect id="zookeeperConnect"
    		zk-connect="192.168.80.150:2181" zk-connection-timeout="6000"
    		zk-session-timeout="6000" zk-sync-time="2000" />
        
    	<int-kafka:inbound-channel-adapter
    		id="kafkaInboundChannelAdapter" kafka-consumer-context-ref="consumerContext"
    		auto-startup="true" channel="inputFromKafka">
    		<int:poller fixed-delay="1" time-unit="MILLISECONDS" />
    	</int-kafka:inbound-channel-adapter>
        <task:executor id="kafkaMessageExecutor" pool-size="8" keep-alive="120" queue-capacity="500" />
    	<bean id="kafkaDecoder"
    		class="org.springframework.integration.kafka.serializer.common.StringDecoder" />
    
    	<bean id="consumerProperties"
    		class="org.springframework.beans.factory.config.PropertiesFactoryBean">
    		<property name="properties">
    			<props>
    				<prop key="auto.offset.reset">smallest</prop>
    				<prop key="socket.receive.buffer.bytes">10485760</prop> 
    				<prop key="fetch.message.max.bytes">5242880</prop>
    				<prop key="auto.commit.interval.ms">1000</prop>
    			</props>
    		</property>
    	</bean>
        
    	<bean id="kafkaConsumerService" class="com.sunney.service.impl.KafkaConsumerServiceImpl" />
        
    	<int:outbound-channel-adapter channel="inputFromKafka"
    		ref="kafkaConsumerService" method="processMessage" />
    
    	<int-kafka:consumer-context id="consumerContext"
    		consumer-timeout="1000" zookeeper-connect="zookeeperConnect"
    		consumer-properties="consumerProperties">
    		<int-kafka:consumer-configurations>
    			<int-kafka:consumer-configuration
    				group-id="default1" value-decoder="kafkaDecoder" key-decoder="kafkaDecoder"
    				max-messages="5000">
    				
    				<int-kafka:topic id="myTopic" streams="4" />
    				<int-kafka:topic id="testTopic" streams="4" />
    			</int-kafka:consumer-configuration>
    		</int-kafka:consumer-configurations>
    	</int-kafka:consumer-context>
    </beans>

**spring-kafka-producer.xml： **

<?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
    	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:int="http://www.springframework.org/schema/integration"
    	xmlns:int-kafka="http://www.springframework.org/schema/integration/kafka"
    	xmlns:task="http://www.springframework.org/schema/task"
    	xsi:schemaLocation="http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd
            http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
            http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
            http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd">
    	
       
    	<bean id="byteArraySerializer" class="org.apache.kafka.common.serialization.ByteArraySerializer"/>
    	<bean id="stringSerializer" class="org.apache.kafka.common.serialization.StringSerializer"/>
    	<bean id="kafkaEncoder" class="org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaEncoder">
            <constructor-arg value="java.lang.String" />
        </bean>
    	<bean id="producerProperties"
    		class="org.springframework.beans.factory.config.PropertiesFactoryBean">
    		<property name="properties">
    			<props>
    				<prop key="topic.metadata.refresh.interval.ms">3600000</prop>
    				<prop key="message.send.max.retries">5</prop>
    				<prop key="serializer.class">kafka.serializer.StringEncoder</prop>
    				<prop key="partitioner.class">com.wanrong.payment.commons.MyPartitioner</prop>
    				<prop key="request.required.acks">-1</prop>
    				<prop key="send.buffer.bytes">5242880</prop>
    				<prop key="request.timeout.ms">10000</prop>
    				<prop key="retry.backoff.ms">200</prop>
    				<prop key="queue.buffering.max.ms">500</prop>
    				<prop key="queue.buffering.max.messages">10000</prop>
    				<prop key="value.serializer">org.apache.kafka.common.serialization.ByteArraySerializer</prop>
    				<prop key="key.serializer">org.apache.kafka.common.serialization.ByteArraySerializer</prop>
    			</props>
    		</property>
    	</bean>
    	
    	
    	
    	<int:channel id="kafkaTopicTest">
    		<int:queue />
    	</int:channel>
    	
    	<int-kafka:outbound-channel-adapter
    		id="kafkaOutboundChannelAdapterTopicTest" kafka-producer-context-ref="producerContextTopicTest"
    		auto-startup="true" channel="kafkaTopicTest" order="3" >
    		<int:poller fixed-delay="1000" time-unit="MILLISECONDS"
    			receive-timeout="1" task-executor="taskExecutor" />
    	</int-kafka:outbound-channel-adapter>
    	<task:executor id="taskExecutor" pool-size="5"
    		keep-alive="120" queue-capacity="500" />
    	
    	<int-kafka:producer-context id="producerContextTopicTest"
    		producer-properties="producerProperties">
    		<int-kafka:producer-configurations>
    		    
    			<int-kafka:producer-configuration
    				broker-list="192.168.80.150:9092"
    				key-serializer="byteArraySerializer"
                    value-class-type="java.lang.String"
                    value-serializer="stringSerializer"
    				topic="testTopic" />
    			<int-kafka:producer-configuration
    				broker-list="192.168.80.150:9092"
    				key-serializer="byteArraySerializer"
                    value-class-type="java.lang.String"
                    value-serializer="stringSerializer"
    				topic="myTopic" />
    		</int-kafka:producer-configurations>
    	</int-kafka:producer-context>
    </beans>

**【注意】： spring-kafka-consumer.xml的auto-startup设置为true 才能保证 消费端口接受到消息**

**【发消息接口 KafkaService】 ：**

package com.sunney.service;
    
    /**
     * 类KafkaService.java的实现描述：发消息接口类
     * @author nazi
     */
    public interface KafkaService {
        /**
         * 发消息
         * @param topic 主题
         * @param obj 发送内容
         */
        public void sendUserInfo(String topic, Object obj);
    }

**对应实现类：**

package com.sunney.service.impl;
    
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.beans.factory.annotation.Qualifier;
    import org.springframework.integration.kafka.support.KafkaHeaders;
    import org.springframework.integration.support.MessageBuilder;
    import org.springframework.messaging.MessageChannel;
    import org.springframework.stereotype.Service;
    
    import com.sunney.service.KafkaService;
    
    /**
     * 类KafkaServiceImpl.java的实现描述：发消息实现类
     * 
     * @author nazi
     */
    @Service("kafkaService")
    public class KafkaServiceImpl implements KafkaService {
    
        // 默认4个分区
        private static final int numPartitions = 4;
        @Autowired
        @Qualifier("kafkaTopicTest")
        MessageChannel           channel;
    
        public void sendUserInfo(String topic, Object obj) {
            // 获取messageKey
            String messageKey = String.valueOf(obj.hashCode());
            // 获取分区号
            int partitionId = partition(messageKey, numPartitions);
    //        partitionId = 0;
            channel.send(MessageBuilder.withPayload(obj).setHeader(KafkaHeaders.TOPIC,
                                                                   topic).setHeader(KafkaHeaders.MESSAGE_KEY,
                                                                                    messageKey).setHeader(KafkaHeaders.PARTITION_ID,
                                                                                                          partitionId).build());
        }
    
        /**
         * 获取分区号
         * 
         * @param key KEY
         * @param numPartitions 分区数
         * @return
         */
        private int partition(Object key, int numPartitions) {
            try {
                long partitionNum = Long.parseLong((String) key);
                return (int) Math.abs(partitionNum % numPartitions);
            } catch (Exception e) {
                return Math.abs(key.hashCode() % numPartitions);
            }
        }
    
    }

**【消息接受类】： **

package com.sunney.service.impl;
    
    import java.util.Collection;
    import java.util.Iterator;
    import java.util.LinkedHashMap;
    import java.util.List;
    import java.util.Map;
    import java.util.Set;
    
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    import com.alibaba.fastjson.JSON;
    import com.sunney.service.KafkaConsumerService;
    import com.sunney.service.UserDto;
    
    /**
     * 类KafkaConsumerService.java的实现描述：消费接收类
     * 
     * @author nazi
     */
    public class KafkaConsumerServiceImpl implements KafkaConsumerService{
    
        static final Logger logger = LoggerFactory.getLogger(KafkaConsumerServiceImpl.class);
        @Override
        public void processMessage(Map<String, Map<Integer, String>> msgs) {
            logger.info("===============================================processMessage===============");
            for (Map.Entry<String, Map<Integer, String>> entry : msgs.entrySet()) {
                logger.info("============Topic:" + entry.getKey());
                LinkedHashMap<Integer, String> messages = (LinkedHashMap<Integer, String>) entry.getValue();
                Set<Integer> keys = messages.keySet();
                for (Integer i : keys)
                    logger.info("======Partition:" + i);
                Collection<String> values = messages.values();
                for (Iterator<String> iterator = values.iterator(); iterator.hasNext();) {
                    String message = "["+iterator.next()+"]";
                    logger.info("=====message:" + message);
                    List<UserDto> userList = JSON.parseArray(message, UserDto.class);  
                    logger.info("=====userList.size:" + userList.size());
                    for(UserDto userDto : userList){
                        logger.info("call userDto:" + userDto.toString());
                    }
    
                }
    
            }
        }
    
    }

**【消费到的消息打印日志】： **

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Nhb3hfbmF6aQ_size_16_color_FFFFFF_t_70][]

**项目发布地址：** [https://github.com/JnCaoxia/kafaDemo][https_github.com_JnCaoxia_kafaDemo]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Nhb3hfbmF6aQ_size_16_color_FFFFFF_t_70]: /images/20211214/dbe9feaf1ad949bf8281384543221ce2.png
[https_github.com_JnCaoxia_kafaDemo]: https://github.com/JnCaoxia/kafaDemo