1 nginx 日志格式配置

[root@elk-5-10 config]# cd /usr/local/nginx/conf/
[root@elk-5-10 conf]# vi nginx.conf

log_format  access  '$http_host $remote_addr - $remote_user [$time_local] "$request" '
             '$status $body_bytes_sent "$http_referer" '
             '"$http_user_agent" $http_x_forwarded_for';

2 日志格式数据样品

2.1 访问日志：

ss00.xxxxxx.me 150.138.154.157 - - [25/Jul/2017:03:02:35 +0800] "GET /csm/7_527.html HTTP/1.1" 304 0 "http://www.twww.com/tetris/page/64000159042/?ad_id=62928537191&cid=62928889880&req_id=0" "Mozilla/5.0 (Linux; Android 6.0.1; Redmi 4X Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36" 183.204.183.118

2.2 错误日志

2017/07/25 15:23:29 [error] 24881#0: *33 open() "/usr/local/nginx-1.12.0/html/favicon.ico" failed (2: No such file or directory), client: 192.168.1.103, server: www.zyb.com, request: "GET /favicon.ico HTTP/1.1", host: "www.zyb.com"

3 logstash 配置文件

input {
    file {
        type => "nginx-access"
        path => "/data/weixin.sys.mingyaohui.com.log"
        start_position => beginning  
    }
    file {
        type => "nginx-error"
        path => "/data/nginx_error.log"
        start_position => beginning
    }
}

filter {
    if [type] == "nginx-access" {
        grok {
            match => ["message","%{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} [%{HTTPDATE:timestamp}] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} %{IPORHOST:forwordip}" ]}
        }
    } else if [type] == "nginx-error" {
        grok {
            match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) [%{LOGLEVEL:severity}] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<clientip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>"%{URI}"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: "%{URI:referrer}")?"]
        }
    }

    # add geo-location info
    geoip {
        source => "clientip"
    }
}

output {
    elasticsearch {
        hosts => ["10.0.0.10"]
        index => "%{type}-%{+YYYY.MM.dd}"
    }
}

4 kibana分析效果图

参考资料

https://grokdebug.herokuapp.com/patterns#

https://github.com/adventure-yunfei/ELK-for-nginx-log/blob/master/logstash.conf