ELK logstash分析 nginx 日志

logstash nginx 日志分析 http://www.cnblogs.com/wshenjin/p/7061500.html

分析逻辑

logstash 监听 nginx 日志
分析每一行 nginx 日志，正则匹配 (logstash.conf grok配置)
拆分 request 的参数 (logstash.conf kv配置)
传输给 ELK 集群前，处理数据类型和模板分析等（logstash.conf output elk 指定 elk_cluster_new.json）
传到 ELK 集群 (logstash.conf output)

nginx 日志记录格式

http
{
        log_format  main  '$http_x_forwarded_for - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_x_forwarded_for" "$http_user_agent"';
}

日志模型

183.202.215.225 - - [20/Jul/2017:14:39:34 +0800] "GET /stat.html?site=shuidigzh&action=http://d.ifeng.com/webppa/315/060109/index.shtml&uid=150
0530496249_uwejx13000&url=http://d.ifeng.com/webppa/hd/gdian/061516/index.shtml&referrer=http://d.ifeng.com/webppa/315/060109/index.shtml&rehos
t=d.ifeng.com&dateline=1500532799523 HTTP/1.1" 200 0 "http://d.ifeng.com/webppa/hd/gdian/061516/index.shtml" "183.202.215.225" "Mozilla/5.0 (Li
nux; Android 4.3; S8-701w Build/HuaweiMediaPad; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.49 MQQBrowser/6.2 TBS/0
43305 Safari/537.36 MicroMessenger/6.5.8.1060 NetType/WIFI Language/zh_CN"

logstash.conf

input {
        file {
                path => "/data/logs/nginx/analys/stat_access.log"
                start_position => "beginning"
                sincedb_path => "/tmp/.logstash_elk_cluster_new_sincedb"
        }
}
}
filter {

        grok {
                match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:logtimestamp}\] \"(?:%{WORD:verb} %{NOTSP
ACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{QS:browser_agent}"   }

        }

        kv {
                source => "request"
                        field_split => "&?"
                        value_split => "="
#remove_field => [ "path", "ident", "auth","message","httpversion","bytes" ]
        }
        urldecode {
                all_fields => true
        }
        geoip {
                source => "clientip"
                        fields => ["country_name","city_name","real_region_name","latitude","longitude","ip","location"]
        }

}
output {
        elasticsearch {
                hosts => ["10.90.32.48:9200"]
                        index => "testlog-%{+YYYY.MM.dd}"
                        template_overwrite => true
                        template => "/data/soft/install/logstash/outputs/elk_cluster_new.json"
        }

}

指定ELK索引和分析数据 elk_cluster_new.json

{
        "template" : "testlog-*",
                "settings" : {
                        "index.refresh_interval" : "5s", 
                        "index.number_of_replicas" : "0"
                },
                "analysis" : {
                        "analyzer" : {
                                "default" : {
                                        "type" : "standard",
                                        "stopwords" : "_none_"
                                }
                        }
                },
                "mappings" : {
                        "_default_" : {
                                "_all" : { "enabled" : false  },
                                "dynamic_templates" : [{
                                        "message_field" : {
                                                "match" : "message",
                                                "match_mapping_type" : "string",
                                                "mapping" : { "type" : "string", "index" : "not_analyzed"  }

                                        }

                                }, {
                                        "string_fields" : {
                                                "match" : "*",
                                                "match_mapping_type" : "string",
                                                "mapping" : { "type" : "string", "index" : "not_analyzed"  }

                                        }

                                }],
                                        "properties" : {
                                                "@timestamp" : { "type" : "date", "format" : "dateOptionalTime"  },
                                                "logtimestamp":{ "type" : "date", "format" : "dd/MMM/yyyy:HH:mm:ss Z"  },
                                                "@version" : { "type" : "integer", "index" : "not_analyzed"  },
                                                "agent" : { "type" : "string", "index" : "not_analyzed"  },
                                                "bytes" : { "type" : "long", "norms" : { "enabled" : false  }  },
                                                "host" : { "type" : "string", "index" : "not_analyzed"  },
                                                "clientip" : { "type" : "ip", "norms" : { "enabled" : false  }  },
                                                "httpversion" : { "type" : "float"  },
                                                "referrer" : { "type" : "string", "index" : "analyzed"  },
                                                "url" : { "type" : "string", "index" : "analyzed"  },
                                                "request" : { "type" : "string", "index" : "not_analyzed", "include_in_all": false  },
                                                "response" : { "type" : "integer", "index" : "not_analyzed"  },
                                                "geoip" : { "type" : "object", "dynamic" : true,  "properties" : { "location" : { "type" : "geo_point"  }  }  },
                                                "verb" : { "type" : "string", "norms" : { "enabled" : false  }  }

                                        }

                        }

                }

}