logstash + grok 正则语法

你的名字 2022-07-12 18:24 210阅读 0赞

详细正则规则参考:

# 正则语法规则 #

例:

## 日志格式如下 ##

[vclound][2015-11-03 03:35:50,283][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.80]
    [vclound][2015-11-03 03:35:50,381][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=["POST /v2.0/tokens HTTP/1.1" 200 3080]
    [vclound][2015-11-03 03:35:50,384][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.160]
    [vclound][2015-11-03 03:35:50,454][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=["GET /v2/bb0b51d166254dc99bc7462c0ac002ff/servers/b4b530e7-cd9b-42c1-bcd4-a48140726846 HTTP/1.1" 404 73]

## logstash 正则规则参考 ##

(下面代码, 编辑器无法显示, 请点击 view plain 进行阅读)

filter {
      if [type] == "pinyun" {
        grok {
          match => { "message" => "\[%{USERNAME:username}\]\[%{TIMESTAMP_ISO8601:time}\]\[%{LOGLEVEL:loglevel}\]\[%{PROG:filepath}\]\[%{PROG:function}\]\[-\]\[%{BASE16NUM:progid}\]\=\[%{GREEDYDATA:info}\]" }
          add_field => [ "received_at", "%{@timestamp}" ]
          add_field => [ "received_from", "%{host}" ]
        }
      }
    }

注意: 当日志输出有空格, 那么匹配时候就带空格, 如果是特殊字符, 那么就直接匹配该特殊字符

## 输出效果 ##

如下:

{
              "message" => "[vclound][2015-11-03 03:35:50,283][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.80]",
             "@version" => "1",
           "@timestamp" => "2015-11-03T02:01:30.051Z",
                 "type" => "pinyun",
                 "file" => "/apps/logs/uwsgi/uwsgi.log",
                 "host" => "terry-zskvt.vclound.com",
               "offset" => "58995",
             "username" => "vclound",
                 "time" => "2015-11-03 03:35:50,283",
             "loglevel" => "INFO",
             "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203",
             "function" => "_new_conn",
               "progid" => "140192616544000",
                 "info" => "Starting new HTTP connection (1): 240.10.129.80",
          "received_at" => "2015-11-03T02:01:30.051Z",
        "received_from" => "terry-zskvt.vclound.com"
    }
    {
              "message" => "[vclound][2015-11-03 03:35:50,381][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=[\"POST /v2.0/tokens HTTP/1.1\" 200 3080]",
             "@version" => "1",
           "@timestamp" => "2015-11-03T02:01:30.060Z",
                 "type" => "pinyun",
                 "file" => "/apps/logs/uwsgi/uwsgi.log",
                 "host" => "terry-zskvt.vclound.com",
               "offset" => "59181",
             "username" => "vclound",
                 "time" => "2015-11-03 03:35:50,381",
             "loglevel" => "DEBUG",
             "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295",
             "function" => "_make_request",
               "progid" => "140192616544000",
                 "info" => "\"POST /v2.0/tokens HTTP/1.1\" 200 3080",
          "received_at" => "2015-11-03T02:01:30.060Z",
        "received_from" => "terry-zskvt.vclound.com"
    }
    {
              "message" => "[vclound][2015-11-03 03:35:50,384][INFO][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203][_new_conn][-][140192616544000]=[Starting new HTTP connection (1): 240.10.129.160]",
             "@version" => "1",
           "@timestamp" => "2015-11-03T02:01:30.068Z",
                 "type" => "pinyun",
                 "file" => "/apps/logs/uwsgi/uwsgi.log",
                 "host" => "terry-zskvt.vclound.com",
               "offset" => "59362",
             "username" => "vclound",
                 "time" => "2015-11-03 03:35:50,384",
             "loglevel" => "INFO",
             "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:203",
             "function" => "_new_conn",
               "progid" => "140192616544000",
                 "info" => "Starting new HTTP connection (1): 240.10.129.160",
          "received_at" => "2015-11-03T02:01:30.068Z",
        "received_from" => "terry-zskvt.vclound.com"
    }
    {
              "message" => "[vclound][2015-11-03 03:35:50,454][DEBUG][/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295][_make_request][-][140192616544000]=[\"GET /v2/bb0b51d166254dc99bc7462c0ac002ff/servers/b4b530e7-cd9b-42c1-bcd4-a48140726846 HTTP/1.1\" 404 73]",
             "@version" => "1",
           "@timestamp" => "2015-11-03T02:01:30.074Z",
                 "type" => "pinyun",
                 "file" => "/apps/logs/uwsgi/uwsgi.log",
                 "host" => "terry-zskvt.vclound.com",
               "offset" => "59549",
             "username" => "vclound",
                 "time" => "2015-11-03 03:35:50,454",
             "loglevel" => "DEBUG",
             "filepath" => "/usr/lib/python2.6/site-packages/urllib3/connectionpool.py:295",
             "function" => "_make_request",
               "progid" => "140192616544000",
                 "info" => "\"GET /v2/bb0b51d166254dc99bc7462c0ac002ff/servers/b4b530e7-cd9b-42c1-bcd4-a48140726846 HTTP/1.1\" 404 73",
          "received_at" => "2015-11-03T02:01:30.074Z",
        "received_from" => "terry-zskvt.vclound.com"
    }