BlackChen'site

使用ElasticSearch分析 NGINX Access日志

最近在做监控相关的项目. 需要分析NGINXaccess日志来统计入口流量.

日志会统一收集到ElasticSearch集群中. 现在只需要从ElasticSearch中就可以拿到所有数据.

准备

  1. 准备VSCode 软件.

  2. 安装 Elasticsearch for VSCode 插件.
    Elasticsearch for VSCode

  3. 新建一个文件. 设置文件格式为ElasticSearch
    elastic文件

  4. 配置Elasticsearch节点
    shift + command + p 打开控制台,搜索选中 elastic set host
    set host

    config
    如果需要账户认证的话,配置连接需要加账户和密码 例如:
    http://dev_nginx:nginxdev@172.16.110.12:19200

  5. 测试. 根据自己的索引,使用查询语句,查看是否可以查询. 例如查询:

GET /logstash-index-nginx-access-2019.04.10/_search
{
 "query": { 
   "match_all": {    
   }
 },
  "size":1
}

结果:

{
    "took": 5,
    "timed_out": false,
    "_shards": {
        "total": 5,
        "successful": 5,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": 557288,
        "max_score": 1,
        "hits": [
            {
                "_index": "logstash-index-nginx-access-2019.04.10",
                "_type": "doc",
                "_id": "d37YBWoBkKID3eiXbTjv",
                "_score": 1,
                "_source": {
                    "body_bytes_sent": 2239,
                    "beat": {
                        "version": "6.4.2",
                        "name": "xxxxxxxxxxxx",
                        "hostname": "xxxxxxxxxxxx"
                    },
                    "source": "xxxxxxxxxxxx",
                    "time_local": "10/Apr/2019:14:03:49 +0800",
                    "@timestamp": "2019-04-10T06:03:49.000Z",
                    "upstream_response_time": "0.222",
                    "geoip": {
                        "continent_code": "AS",
                        "country_code3": "CN",
                        "region_name": "Shanghai",
                        "country_name": "China",
                        "latitude": 31.0456,
                        "country_code2": "CN",
                        "location": {
                            "lon": 121.3997,
                            "lat": 31.0456
                        },
                        "region_code": "SH",
                        "ip": "210.13.91.146",
                        "longitude": 121.3997,
                        "city_name": "Shanghai",
                        "timezone": "Asia/Shanghai",
                        "coordinates": [
                            121.3997,
                            31.0456
                        ]
                    },
                    "@version": "1",
                    "remote_user": "0",
                    "prospector": {
                        "type": "log"
                    },
                    "input": {
                        "type": "log"
                    },
                    "message": "{xxxxxxxxxxxx}",
                    "request": "GET /xxxxxxxxxxxx HTTP/1.1",
                    "offset": 7059061,
                    "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
                    "host": "xxxxxxxxxxxx.com",
                    "remote_addr1": "210.13.91.146",
                    "http_referer": "https://xxxxxxxxxxxx.html",
                    "tags": [
                        "nginx-log"
                    ],
                    "request_time": 0.222,
                    "remote_addr": "172.18.57.45",
                    "status": 200
                }
            }
        ]
    }
}

分析nginx access日志

上个步骤我们已经可以从elasticsearh集群中获取到数据了. 现在我们分析它.
NGINX access log格式如下.

      {
                "_index": "logstash-index-nginx-access-2019.04.10",
                "_type": "doc",
                "_id": "d37YBWoBkKID3eiXbTjv",
                "_score": 1,
                "_source": {
                    "body_bytes_sent": 2239,
                    "beat": {
                        "version": "6.4.2",
                        "name": "xxxxxxxxxxxxx",
                        "hostname": "xxxxxxxxxxxxx"
                    },
                    "source": "xxxxxxxxxxxxx",
                    "time_local": "10/Apr/2019:14:03:49 +0800",
                    "@timestamp": "2019-04-10T06:03:49.000Z",
                    "upstream_response_time": "0.222",
                    "geoip": {
                        "continent_code": "AS",
                        "country_code3": "CN",
                        "region_name": "Shanghai",
                        "country_name": "China",
                        "latitude": 31.0456,
                        "country_code2": "CN",
                        "location": {
                            "lon": 121.3997,
                            "lat": 31.0456
                        },
                        "region_code": "SH",
                        "ip": "210.13.91.146",
                        "longitude": 121.3997,
                        "city_name": "Shanghai",
                        "timezone": "Asia/Shanghai",
                        "coordinates": [
                            121.3997,
                            31.0456
                        ]
                    },
                    "@version": "1",
                    "remote_user": "0",
                    "prospector": {
                        "type": "log"
                    },
                    "input": {
                        "type": "log"
                    },
                    "message": "{xxxxxxxxxxxxxxxxxxxxxxxxxxxxx}",
                    "request": "GET xxxxxxxxxxxxxxxxxxxxxxxxxxxxx  HTTP/1.1",
                    "offset": 7059061,
                    "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
                    "host": "e-upm.stag.rs.com",
                    "remote_addr1": "xxx.xxx.xxx.xxx",
                    "http_referer": "xxxxxxxxxxxxxxxxxx",
                    "tags": [
                        "nginx-log"
                    ],
                    "request_time": 0.222,
                    "remote_addr": "xxxxxxxxxxx",
                    "status": 200
                }
            },

获取某段时间内的日志

GET /logstash-index-nginx-access*/_search
{
    "query": {
        "bool": {
            "must": [
                {
                    "range": {
                        "@timestamp": {
                            "gte": 1554739200000,
                            "lte": 1554825600000,
                            "format": "epoch_millis"
                        }
                    }
                }
            ]
        }
    }
}

根据时间分组统计

每10分钟的请求量统计.可以做折线图,清晰的表现出每个时间段的请求大小

GET /logstash-index-nginx-access*/_search
{
  "aggs": {
    "x_minute": {
      "date_histogram": {
        "field": "@timestamp",
        "interval": "10m",
        "time_zone": "Asia/Shanghai",
        "min_doc_count": 1
      }
    }
  },
  "size": 0,
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": 1556254705,
              "lte": 1556258306,
              "format": "epoch_second"
            }
          }
        }
      ]
    }
  }
}

结果:

{
    "took": 47,
    "timed_out": false,
    "_shards": {
        "total": 160,
        "successful": 160,
        "skipped": 155,
        "failed": 0
    },
    "hits": {
        "total": 20709,
        "max_score": 0,
        "hits": []
    },
    "aggregations": {
        "x_minute": {
            "buckets": [
                {
                    "key_as_string": "2019-04-26T12:50:00.000+08:00",
                    "key": 1556254200000,
                    "doc_count": 497
                },
                {
                    "key_as_string": "2019-04-26T13:00:00.000+08:00",
                    "key": 1556254800000,
                    "doc_count": 3512
                },
                {
                    "key_as_string": "2019-04-26T13:10:00.000+08:00",
                    "key": 1556255400000,
                    "doc_count": 4391
                },
                {
                    "key_as_string": "2019-04-26T13:20:00.000+08:00",
                    "key": 1556256000000,
                    "doc_count": 3453
                },
                {
                    "key_as_string": "2019-04-26T13:30:00.000+08:00",
                    "key": 1556256600000,
                    "doc_count": 3012
                },
                {
                    "key_as_string": "2019-04-26T13:40:00.000+08:00",
                    "key": 1556257200000,
                    "doc_count": 3396
                },
                {
                    "key_as_string": "2019-04-26T13:50:00.000+08:00",
                    "key": 1556257800000,
                    "doc_count": 2448
                }
            ]
        }
    }
}

根据地区分组

要获取某个地区的的请求量,做类似请求分布图的时候.需要这种查询.

分析到NGINX日志中,有geoip字段,可以根据geoip中的相关内容进行分组.

{
    "geoip": {
        "continent_code": "AS",
        "country_code3": "CN",
        "region_name": "Shanghai",
        "country_name": "China",
        "latitude": 31.0456,
        "country_code2": "CN",
        "location": {
            "lon": 121.3997,
            "lat": 31.0456
        },
        "region_code": "SH",
        "ip": "210.13.91.146",
        "longitude": 121.3997,
        "city_name": "Shanghai",
        "timezone": "Asia/Shanghai",
        "coordinates": [
            121.3997,
            31.0456
        ]
    }
}

请求:


GET /logstash-index-nginx-access*/_search
{
  "aggs": {
    "region_group": {
      "terms": {
        "field": "geoip.region_name.keyword",
        "size":100000,
        "min_doc_count": 1
      }
    }
  },
  "size": 0,
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": 1555233132,
              "lte": 1555319532,
              "format": "epoch_second"
            }
          }
        }
      ],
      "filter": {
            "term":{"geoip.country_name.keyword":"China"}}
    }
  }
}

结果:

{
    "took": 252,
    "timed_out": false,
    "_shards": {
        "total": 160,
        "successful": 160,
        "skipped": 150,
        "failed": 0
    },
    "hits": {
        "total": 159343,
        "max_score": 0,
        "hits": []
    },
    "aggregations": {
        "region_group": {
            "doc_count_error_upper_bound": 0,
            "sum_other_doc_count": 0,
            "buckets": [
                {
                    "key": "Shanghai",
                    "doc_count": 41719
                },
                {
                    "key": "Beijing",
                    "doc_count": 8037
                },
                {
                    "key": "Jiangsu",
                    "doc_count": 7573
                },
                {
                    "key": "Guangdong",
                    "doc_count": 6524
                },
                {
                    "key": "Zhejiang",
                    "doc_count": 5327
                },
                {
                    "key": "Sichuan",
                    "doc_count": 5108
                },
                {
                    "key": "Shandong",
                    "doc_count": 4531
                },
                {
                    "key": "Liaoning",
                    "doc_count": 4278
                },
                {
                    "key": "Henan",
                    "doc_count": 3822
                },
                {
                    "key": "Chongqing",
                    "doc_count": 3810
                },
                {
                    "key": "Fujian",
                    "doc_count": 3723
                },
                {
                    "key": "Hunan",
                    "doc_count": 3515
                },
                {
                    "key": "Hubei",
                    "doc_count": 3465
                },
                {
                    "key": "Anhui",
                    "doc_count": 3463
                },
                {
                    "key": "Tianjin",
                    "doc_count": 3434
                },
                {
                    "key": "Guizhou",
                    "doc_count": 3297
                },
                {
                    "key": "Shaanxi",
                    "doc_count": 3223
                },
                {
                    "key": "Heilongjiang",
                    "doc_count": 2731
                },
                {
                    "key": "Gansu",
                    "doc_count": 2702
                },
                {
                    "key": "Yunnan",
                    "doc_count": 2694
                },
                {
                    "key": "Guangxi",
                    "doc_count": 2504
                },
                {
                    "key": "Hainan",
                    "doc_count": 2464
                },
                {
                    "key": "Qinghai",
                    "doc_count": 2403
                },
                {
                    "key": "Jiangxi",
                    "doc_count": 2209
                },
                {
                    "key": "Hebei",
                    "doc_count": 2097
                },
                {
                    "key": "Shanxi",
                    "doc_count": 2088
                },
                {
                    "key": "Jilin",
                    "doc_count": 2047
                },
                {
                    "key": "Inner Mongolia Autonomous Region",
                    "doc_count": 1808
                },
                {
                    "key": "Ningxia Hui Autonomous Region",
                    "doc_count": 1785
                },
                {
                    "key": "Xinjiang",
                    "doc_count": 1117
                },
                {
                    "key": "Tibet",
                    "doc_count": 860
                }
            ]
        }
    }
}

可以明显的看到根据省份信息,已经做了分组.

评论