Logstash input输入 beats插件和 syslog插件

柔情只为你懂 2023-07-16 07:59 62阅读 0赞

# Logstash input输入 beats插件 和 syslog插件 #

Logstash input多个输入插件同时使用

Logstash -7.2.0

filter使用的插件：grok、kv、urldecode、date、mutate、geoip

## 1、先看总体配置logstash.conf ##

（执行时请去除所有中文注释）

input {
       #beats输入插件
       beats {
    	#绑定主机
         host => "0.0.0.0"
    	 #绑定端口
         port => 5044
    	 #额外添加字段，这里是为了区分来自哪一个插件
    	 add_field => {"[fields][class]" => "beats"}
       }
       
       syslog {
       #绑定端口
    	port => 514
    	#额外添加字段，这里是为了区分来自哪一个插件
    	add_field => {"[fields][class]" => "json"}
       }
    }
    
    filter { 
    	#处理来自beats插件的日志,beats这里收集的是tomcat的日志
    	# 样例：192.168.68.88 - - [16/Mar/2020:11:22:08 +0800] "GET /esws/testService/test?name=天道酬勤&size=50 HTTP/1.1" 200 15315
    	if [fields][class] == "beats"{
    		#grop过滤插件，在编写grop时，可以使用kibana，kibana上有编写工具，无需自己搭建（官方grok速度太慢）
    		grok {
    			#解析Apache日志，自动分割
    			match => { "message" => "%{COMMONAPACHELOG}" }
    		}
    		#键值过滤器
    		kv {
    			#对request字段操作
    			source => "request"
    			按照& ？ 分割
    			field_split => "&?"
    			value_split => "="
    			#选取自己需要的分割后的字段
    			include_keys => ["op","reportlet","formlet"] 
    		} 
    		#解码
    		urldecode {
    			#解码全部字段
    			all_fields => true
    		}
    		#日期处理插件
    		date {
    			#日期匹配，匹配格式可以有多个
    			match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
    			#匹配的日期存储到字段中
    			target => "@timestamp"
    		}
    			
    		# 数据修改
    		mutate{
    			#移除指定字段
    			remove_field => ["agent","beat","offset","tags","prospector","log","ident","[host][name]","[host][hostname]","[host][architecture]","[host][os]","[host][id]","auth","[input][type]"]
    			#复制字段
    			copy => { "@timestamp" => "timestamp" }
    			#copy => { "[fields][fields_type]" => "fields_type" }
    			copy => { "formlet" => "reportlet" }
    		}
    		
    		mutate{
    			#替换
    			gsub => ["reportlet", "%2F", "/"]
    		}
    		
    		if ! [fields_type]  {
    			mutate{
    				copy => { "[fields][fields_type]" => "fields_type" }
    			}
    		}
    
    		date{
    			match => [ "timestamp", "yyyy-MM-dd-HH:mm:ss" ]
    			locale => "cn"
    		}
    		
    		#ip解析，分析IP的位置
    		geoip{
    			source => "clientip"
    		}
    	}
    	#处理来自syslog插件的日志
    	if [fields][class] == "json"{
    		json {
    			source => "message"
    		}
    		
    		if [host] == "192.168.68.100" {
    			mutate{
    				add_field => {"fields_type" => "firewall"}
    			}
    		}
    	}
    	
    	  if ! [fields_type] {
                            mutate{
    				add_field => {"fields_type" => "error-221"}
                            }       
                    } 
    }
    
    output {
    	elasticsearch {
    		action => "index"
    		#填写ES集群
        hosts => ["http://node-01:9200","http://node-02:9200","http://node-03:9200"]
    #ES如果有登陆验证，要配置用户名和密码
    #	user => "admin"
    #	password => "123456"
    		#按字段值，存入不同的索引中
    		index => "%{fields_type}-%{+YYYY-MM}"
    		
    	}
    }

### 2、官方文档： ###

beats：[https://www.elastic.co/guide/en/logstash/current/plugins-inputs-beats.html][https_www.elastic.co_guide_en_logstash_current_plugins-inputs-beats.html]

syslog：[https://www.elastic.co/guide/en/logstash/current/plugins-inputs-syslog.html][https_www.elastic.co_guide_en_logstash_current_plugins-inputs-syslog.html]

### 3 对人工智能感兴趣点下面链接 ###

[现在人工智能非常火爆，很多朋友都想学，但是一般的教程都是为博硕生准备的，太难看懂了。最近发现了一个非常适合小白入门的教程，不仅通俗易懂而且还很风趣幽默。所以忍不住分享一下给大家。点这里可以跳转到教程。][Link 1]

[https://www.cbedai.net/u014646662][Link 1]

[https_www.elastic.co_guide_en_logstash_current_plugins-inputs-beats.html]: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-beats.html
[https_www.elastic.co_guide_en_logstash_current_plugins-inputs-syslog.html]: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-syslog.html
[Link 1]: https://www.cbedai.net/u014646662