ELK使用与RabbitMQ的集成

墨蓝 2022-11-21 15:17 163阅读 0赞

# Elasticsearch #

**查看当前节点的所有 Index**  
curl -X GET 'http://localhost:9200/\_cat/indices?v'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70][]  
**列出每个 Index 所包含的 Type**  
curl 'localhost:9200/\_mapping?pretty=true'  
**新建一个名叫weather的 Index**  
curl -X PUT 'localhost:9200/weather'  
**删除这个 Index**  
curl -X DELETE 'localhost:9200/weather'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 1][]

## 中文分词设置 ##

[https://github.com/medcl/elasticsearch-analysis-ik/releases][https_github.com_medcl_elasticsearch-analysis-ik_releases]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 2][]

解压到插件配置目录

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 3][]

重新启动elasticsearch

Could not load plugin descriptor for plugin directory \[.DS\_Store\]

![20201102161631671.png][]

### 新增记录 ###

curl -H 'Content-type: application/json' -XPOST 'http://10.136.60.21:9200/accounts/person' -d '{
     "user": "李四",
     "title": "工程师",
     "desc": "系统管理"
    }'

或者

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 4][]

### 查看记录 ###

curl 'localhost:9200/accounts/person/8hYUiHUB8PWvPlAQQltf?pretty=true'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 5][]

或者

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 6][]

### 更新记录 ###

curl -X PUT 'localhost:9200/accounts/person/8hYUiHUB8PWvPlAQQltf' -H 'Content-Type: application/json' -d'
    {
        "user" : "张三",
        "title" : "工程师",
        "desc" : "数据库管理，软件开发"
    }'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 7][]

### 删除记录 ###

curl -X DELETE 'localhost:9200/accounts/person/8hYUiHUB8PWvPlAQQltf'

## 数据查询 ##

### 匹配所有文档 ###

/Index/Type/\_search

curl -H 'Content-type: application/json' -XGET 'http://10.136.60.21:9200/accounts/person/\_search'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 8][]

### 匹配查询 ###

curl -H 'Content-type: application/json' -XPOST 'http://10.136.60.21:9200/accounts/person/_search' -d '{
     "query": {
      "match": {
       "desc": "管理"
      }
     }
    }'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 9][]

## ELK集成RabbitMQ ##

### RabbitMQ添加队列log.queue ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 10][]

### 添加交换机log.exchange ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 11][]

### log.exchange绑定log.queue ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 12][]

### `logback.xml`文件，配置`appender`输出日志到RabbitMQ ###

<?xml version="1.0" encoding="UTF-8"?>
    <configuration debug="false">
    
        
        <property name="LOG_HOME" value="d:/"/>
    
        
        <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
            <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
                
                <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
            </encoder>
        </appender>
    
        <appender name="RABBITMQ"
                  class="org.springframework.amqp.rabbit.logback.AmqpAppender">
            <layout>
                <pattern><![CDATA[ %d %p %t [%c] - <%m>%n ]]></pattern>
            </layout>
            
            <addresses>10.136.60.21:5672</addresses>
            <username>guest</username>
            <password>guest</password>
            <declareExchange>true</declareExchange>
            <exchangeType>direct</exchangeType>
            <exchangeName>log.exchange</exchangeName>
            <routingKeyPattern>info</routingKeyPattern>
            <generateId>true</generateId>
            <charset>UTF-8</charset>
            <durable>true</durable>
            <deliveryMode>NON_PERSISTENT</deliveryMode>
            <autoDelete>false</autoDelete>
        </appender>
    
        <logger name="com.light.rabbitmq" level="info" additivity="false">
            <appender-ref ref="STDOUT"/>
            <appender-ref ref="RABBITMQ"/>
        </logger>
    
        
        <root level="INFO">
            <appender-ref ref="STDOUT"/>
            <appender-ref ref="RABBITMQ"/>
        </root>
    
    </configuration>

### logstash.conf获取RabbitMQ数据输出至es ###

input{
      rabbitmq{
    		host=>"10.136.60.21"               
    		port=> 5672                     
    		user=>"guest"                   
    		password=>"guest"               
    		queue=>"log.queue"             
    		durable=> true                  
    		codec=>json                     
    		type=> "result"                 
      }
    }
    output {
      elasticsearch {
        hosts => "es:9200"
        index => "springboot-logstash-%{+YYYY.MM.dd}"
      }
    }

### 启动项目 ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 13][]

### logstash日志 ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 14][]

### Kibana查看 ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 15][]

### Cerebro查询 ###

或should

curl -H 'Content-type: application/json' -XPOST 'http://172.20.10.2:9200/springboot-logstash-2020.11.09/_doc/_search' -d '{
     "query": {
      "bool": {
       "should": [
        {
         "match": {
          "message": "INFO"
         }
        },
        {
         "match": {
          "@timestamp": "2020-11-09T07:05:39.067Z"
         }
        }
       ]
      }
     },
     "_source": [
      "@timestamp",
      "message"
     ],
     "from": 0,
     "size": 2,
     "sort": [
      {
       "@timestamp": "desc"
      }
     ]
    }'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 16][]

且must

curl -H 'Content-type: application/json' -XPOST 'http://172.20.10.2:9200/springboot-logstash-2020.11.09/_doc/_search' -d '{
     "query": {
      "bool": {
       "must": [
        {
         "match": {
          "message": "INFO"
         }
        },
        {
         "match": {
          "@timestamp": "2020-11-09T07:05:39.067Z"
         }
        }
       ]
      }
     },
     "_source": [
      "@timestamp",
      "message"
     ],
     "from": 0,
     "size": 2,
     "sort": [
      {
       "@timestamp": "desc"
      }
     ]
    }'

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 17][]

## 分词器设置 ##

{
        "index": {
            "analysis": {
                "analyzer": {
                    "ik_pinyin_analyzer": {
                        "type": "custom",
                        "tokenizer": "ik_max_word",
                        "filter": "pinyin_filter"
                    }
                },
                "filter": {
                    "pinyin_filter": {
                        "type": "pinyin",
                        "keep_first_letter": false
                    }
                }
            }
        }
    }

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 18][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 19][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70]: /images/20221120/e17cc2fc191440099fe3fc336bb43b03.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 1]: /images/20221120/cd3e01f65abf45ae8f267ebb50427a13.png
[https_github.com_medcl_elasticsearch-analysis-ik_releases]: https://github.com/medcl/elasticsearch-analysis-ik/releases
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 2]: /images/20221120/d5c7871456254c92a2023929473c4eaa.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 3]: /images/20221120/70a2a99765d14fd0bb44de1ea5db6b24.png
[20201102161631671.png]: /images/20221120/ef83717ce6ef4fa99dea2d8dc4adf07b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 4]: /images/20221120/7382282253f84f5baeb698699321a8e6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 5]: /images/20221120/6bccf8c175884257b87d3e6be6df47a4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 6]: /images/20221120/c7b15a5ae5b94377b93899ab9e4dbad2.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 7]: /images/20221120/8e23fcbdcb85434ea3a9b408ecc6a724.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 8]: /images/20221120/0e8089c801f84993b2c4cff3338c03e9.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 9]: /images/20221120/837fc087c7b947069228e82bdcce3542.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 10]: /images/20221120/35780adba512455786455082a7a7db99.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 11]: /images/20221120/46e3513df30044d3aa9241dabced13ff.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 12]: /images/20221120/7901c68a45614dd98cbb12467bffefcb.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 13]: /images/20221120/6790e3da2867446cb31b624450e5e22b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 14]: /images/20221120/2338018b474b42a3a9803244e8eff6d1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 15]: /images/20221120/b5cb80942f3c4b1898d1588f8d600b89.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 16]: /images/20221120/df8c5f7379eb4f88afb9edb8ae775105.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 17]: /images/20221120/5d3c0db8359c49cf9a73ba57aee7ca0f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 18]: /images/20221120/342dfc0a82f9417cac56fa22b5aee871.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N0ZXZlX2ZyYW5r_size_16_color_FFFFFF_t_70 19]: /images/20221120/4d498eeae24d427f80d00f1c3e9fa56c.png