ELK之logstash配置

青旅半醒 2023-06-22 03:22 21阅读 0赞

# ELK之logstash配置 #

## 一、logstash直接监听日志文件\` ##

input{
     file {
    	path => "/tmp/logs/info/gateway.log"
    	start_position => "beginning"
    	type => "log1"
       }
     file {
    	path => "/tmp/logs/debug/boot.log"
    	start_position => "beginning"
    	type => "log2"
       }
    }
    output{
    	 if[type] == "log1"{
    	  elasticsearch {
    		   hosts => ["192.167.213.11:9100"]      
    		   index => "gateway-%{+YYYY.MM.dd}"        
    		}
    	}
    	if[type] == "log2"{
    	  elasticsearch {
    		   hosts => ["192.168.213.11:9100"]      
    		   index => "boot-%{+YYYY.MM.dd}"        
    		}
    	}	
    }

## 二、filebeat整合logstash ##

#### filebeat.yml ####

# filebeat.yml
    filebeat.prospectors:
    - type: log
      paths:
        - /tmp/logs/info/gate.log
      input_type: log
      tags: ["log1"]
      fields:
        type: gate
      document_type: gate
      multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
      multiline.negate: true
      multiline.match: after
      
     - type: log
      paths:
       -/tmp/logs/debug/boot.log
      input_type: log
      tags: ["log2"]
      fields:
        type: boot
      document_type: boot
      multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
      multiline.negate: true
      multiline.match: after
      
    fields:
      env: develop-203.11
    output:
      logstash:
        hosts: ["192.168.203.11:5000"]

注：filebeat的文件由于是yml 所以语法严谨一些  
logstash.conf

input{
    		beats {
    		port => 5000
    		codec => "json"
    	}
    	}	
    	filter {
    	grok {
    		match  => {
    			"message" => "(?<date>(\d*[./-]\d*[./-]\d* \d*:\d*:\d*[.,][0-9]+)) .*%{LOGLEVEL:level} .*\- \[%{USERNAME:requestId}\] \[%{HOSTNAME:ip}\] \[%{USERNAME:channel}\] \[%{USERNAME:serviceName}\] \[%{INT:spendTime}\ms] \[%{INT:code}]"
    		}
    		overwrite => ["message"]		
    	}
    	mutate {
    		convert => ["spendTime", "integer"]
    	}
    }
    	output{		
    		 if "log1" in [tags]{
    		  elasticsearch {
    			   hosts => ["192.168.213.11:9200"]      
    			   index => "gate-%{+YYYY.MM.dd}"        
    			}
    		}
    		if "log2" in [tags]{
    		  elasticsearch {
    			   hosts => ["192.168.213.11:9200"]      
    			   index => "boot-%{+YYYY.MM.dd}"        
    			}
    		}		
    	}