使用ELK分析腾讯云CLB日志

缘起

最近在使用腾讯云，想对访问日志进行收集与分析，发现CLB（负责均衡）日志只能保存到COS上面，而且是每个CLB没小时压发送个gz压缩包到COS。

实现方式

CLB配置日志存储到COS，Filebeat客户端CVM安装cosfs挂载COS，并配置Filebeat输出到Elasticsearch集群，最后通过Kibana和Grafana分析。

参考文档

https://www.elastic.co/guide/en/logstash/current/lookup-enrichment.html
https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-template.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v4.1.5-plugins-inputs-file.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v3.1.0-plugins-filters-translate.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v1.0.4-plugins-filters-jdbc_streaming.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v9.2.0-plugins-outputs-elasticsearch.html
/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json

实施步骤

1. COS创建bucket用于存储CLB日志

2. CLB配置存储日志到COS

日志访问：当前仅支持HTTP/HTTPS访问日志的收集，腾讯云默认在CLB底层为客户保留3天的日志；开启日志访问后，日志将存入COS，支持更长期地存储，详情请见https://cloud.tencent.com/document/product/214/10329。

3. 客户端配置COSFS工具

安装Filebeat或Logstash客户端上安装COSFS工具，挂载COS，安装参考https://cloud.tencent.com/document/product/436/6883。

4. 客户端配置传输日志到Elasticsearch集群

a. Logstash主要配置信息如下：

input {
   file{
      path => "/nginxlog/clb_log_bj/*/*/*"   #监控日志路径，可通配
      exclude => "*.gz"                      #排除掉gz压缩文件，因为腾讯CLB日志没小时生成一个压缩包，这边需要脚本自动解压缩
      codec => "json"                        #日志格式为JSON
   }
}

filter {
    jdbc_streaming{                          #使用jdbc_streaming filter是给域名加上部门，默认CLB日志没有部门字段，也可以用translate filter但是不方便动态更新
        jdbc_driver_library => "/etc/logstash/mysql-connector.jar"
        jdbc_driver_class => "com.mysql.jdbc.Driver"
        jdbc_connection_string => "jdbc:mysql://localhost:3306/domain"
        jdbc_user => "domain_rw"
        jdbc_password => "domain_password"
        statement => "select ifnull((select department from domain.domain WHERE url = :server_name),CONVERT('其他' USING utf8)) as department"
        parameters => { "server_name" => "server_name"}
        target => "department"
    }
}

output {
   elasticsearch {
       hosts => ["elasticsearch01:9200","elasticsearch02:9200","elasticsearch03:9200"]
       index => "logstash-nginxlog-%{+YYYY.MM.dd}"
       manage_template => true
       template_overwrite => true
       template_name => "nginx_template"                         #自定义模板名字
       template => "/etc/logstash/templates/nginx_template"      #自定义模板路径
  }
}

b. Nginx模板配置信息如下：

{
"template" : "logstash-nginxlog-*",     #在默认logstash模板上面修改的
"version" : 999,
"settings" : {                          #主要是改动index分片，副本，其他也可以更加需要修改
"index.refresh_interval" : "60s",
"index.number_of_shards": "3",
"index.number_of_replicas": "1"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}

c. Filebeat主配置文件如下：

filebeat.inputs:                     #默认简单需求filebeat就能满足，记录下
- type: log
enabled: true
paths:
- /nginxlog/clb_log_bj/*/*/*         #监控日志路径

exclude_files: ['.gz/pre>]           #排除压缩文件

json.message_key: log                #如下三行设置json格式
json.keys_under_root: true
json.overwrite_keys: true

filebeat.config.modules:             #设置日志字段，可以删除不需要的字段信息
path: ${path.config}/modules.d/*.yml
reload.enabled: true

setup.template.enabled: true         #如下设置索引名称、分片、副本信息
setup.template.name: "filebeat-nginxlog"
setup.template.pattern: "filebeat-nginxlog-*"
setup.template.overwrite: true
setup.template.settings:
index.number_of_shards: 3
index.number_of_replicas: 1

output.elasticsearch:                #如下输出elasticsearch集群配置
hosts: ["elasticsearch01:9200","elasticsearch02:9200","elasticsearch03:9200"]
index: "filebeat-nginxlog-%{+yyyy.MM.dd}"

5. Grafana或者Kibana制图

a. Grafana制图模板如下：

{
  "__inputs": [
    {
      "name": "DS_LOGSTASH-NGINXLOG",
      "label": "logstash-nginxlog",
      "description": "",
      "type": "datasource",
      "pluginId": "elasticsearch",
      "pluginName": "Elasticsearch"
    }
  ],
  "__requires": [
    {
      "type": "datasource",
      "id": "elasticsearch",
      "name": "Elasticsearch",
      "version": "5.0.0"
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "5.0.1"
    },
    {
      "type": "panel",
      "id": "grafana-piechart-panel",
      "name": "Pie Chart",
      "version": "1.3.3"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": "5.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": null,
  "iteration": 1534240475758,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 16,
      "panels": [],
      "repeat": "department",
      "title": "$department",
      "type": "row"
    },
    {
      "aliasColors": {},
      "breakPoint": "50%",
      "cacheTimeout": null,
      "combine": {
        "label": "Others",
        "threshold": 0
      },
      "datasource": "${DS_LOGSTASH-NGINXLOG}",
      "description": "",
      "fontSize": "80%",
      "format": "none",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "id": 6,
      "interval": null,
      "legend": {
        "percentage": true,
        "percentageDecimals": 2,
        "show": true,
        "values": true
      },
      "legendType": "Right side",
      "links": [],
      "maxDataPoints": 3,
      "nullPointMode": "connected",
      "pieType": "pie",
      "strokeWidth": "0",
      "targets": [
        {
          "bucketAggs": [
            {
              "fake": true,
              "field": "http_host.keyword",
              "id": "3",
              "settings": {
                "min_doc_count": 1,
                "order": "desc",
                "orderBy": "_count",
                "size": "10"
              },
              "type": "terms"
            },
            {
              "field": "@timestamp",
              "id": "2",
              "settings": {
                "interval": "auto",
                "min_doc_count": 0,
                "trimEdges": 0
              },
              "type": "date_histogram"
            }
          ],
          "expr": "",
          "format": "time_series",
          "intervalFactor": 1,
          "metrics": [
            {
              "field": "select field",
              "id": "1",
              "type": "count"
            }
          ],
          "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
          "refId": "A",
          "timeField": "@timestamp"
        }
      ],
      "title": "域名访问比例图",
      "type": "grafana-piechart-panel",
      "valueName": "total"
    },
    {
      "aliasColors": {},
      "breakPoint": "50%",
      "cacheTimeout": null,
      "combine": {
        "label": "Others",
        "threshold": 0
      },
      "datasource": "${DS_LOGSTASH-NGINXLOG}",
      "fontSize": "80%",
      "format": "none",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "id": 4,
      "interval": null,
      "legend": {
        "percentage": true,
        "percentageDecimals": 2,
        "show": true,
        "sortDesc": true,
        "values": false
      },
      "legendType": "Right side",
      "links": [],
      "maxDataPoints": 3,
      "nullPointMode": "connected",
      "pieType": "pie",
      "strokeWidth": "0",
      "targets": [
        {
          "bucketAggs": [
            {
              "fake": true,
              "field": "status.keyword",
              "id": "3",
              "settings": {
                "min_doc_count": 1,
                "order": "desc",
                "orderBy": "_count",
                "size": "10"
              },
              "type": "terms"
            },
            {
              "field": "@timestamp",
              "id": "2",
              "settings": {
                "interval": "auto",
                "min_doc_count": 0,
                "trimEdges": 0
              },
              "type": "date_histogram"
            }
          ],
          "expr": "",
          "format": "time_series",
          "intervalFactor": 1,
          "metrics": [
            {
              "field": "select field",
              "id": "1",
              "type": "count"
            }
          ],
          "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
          "refId": "A",
          "timeField": "@timestamp"
        }
      ],
      "title": "状态码比例图",
      "type": "grafana-piechart-panel",
      "valueName": "total"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_LOGSTASH-NGINXLOG}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 10
      },
      "id": 2,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "bucketAggs": [
            {
              "fake": true,
              "field": "http_host.keyword",
              "id": "3",
              "settings": {
                "min_doc_count": 1,
                "order": "desc",
                "orderBy": "_count",
                "size": "10"
              },
              "type": "terms"
            },
            {
              "field": "@timestamp",
              "id": "2",
              "settings": {
                "interval": "auto",
                "min_doc_count": 0,
                "trimEdges": 0
              },
              "type": "date_histogram"
            }
          ],
          "expr": "",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "",
          "metrics": [
            {
              "field": "select field",
              "id": "1",
              "type": "count"
            }
          ],
          "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
          "refId": "A",
          "timeField": "@timestamp"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeShift": null,
      "title": "域名访问趋势TOP10",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": true,
      "dashLength": 10,
      "dashes": false,
      "datasource": "${DS_LOGSTASH-NGINXLOG}",
      "fill": 1,
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 10
      },
      "hideTimeOverride": false,
      "id": 18,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": false,
        "max": true,
        "min": false,
        "rightSide": true,
        "show": true,
        "total": false,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "nullPointMode": "null",
      "percentage": false,
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [
        {
          "alias": "Count"
        }
      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "bucketAggs": [
            {
              "field": "@timestamp",
              "id": "2",
              "settings": {
                "interval": "auto",
                "min_doc_count": 0,
                "trimEdges": 0
              },
              "type": "date_histogram"
            }
          ],
          "expr": "",
          "format": "time_series",
          "intervalFactor": 1,
          "metrics": [
            {
              "field": "request_time",
              "id": "1",
              "meta": {},
              "settings": {
                "percents": [
                  "90",
                  "95",
                  "99",
                  "99.9",
                  "99.99",
                  "100"
                ]
              },
              "type": "percentiles"
            }
          ],
          "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
          "refId": "A",
          "timeField": "@timestamp"
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt"
        },
        {
          "colorMode": "warning",
          "fill": false,
          "line": false,
          "op": "gt",
          "value": 5000
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "延迟",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "decimals": null,
          "format": "s",
          "label": "",
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
        }
      ]
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 17
      },
      "id": 14,
      "panels": [
        {
          "columns": [],
          "datasource": "logstash-nginxlog",
          "fontSize": "100%",
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 70
          },
          "id": 12,
          "links": [],
          "pageSize": null,
          "scroll": true,
          "showHeader": true,
          "sort": {
            "col": 1,
            "desc": true
          },
          "styles": [
            {
              "alias": "Time",
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "link": false,
              "pattern": "Time",
              "type": "date"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "number",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "bucketAggs": [
                {
                  "field": "http_user_agent.keyword",
                  "id": "2",
                  "settings": {
                    "min_doc_count": 1,
                    "order": "desc",
                    "orderBy": "_count",
                    "size": "10"
                  },
                  "type": "terms"
                }
              ],
              "expr": "",
              "format": "table",
              "intervalFactor": 1,
              "metrics": [
                {
                  "field": "select field",
                  "id": "1",
                  "type": "count"
                }
              ],
              "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
              "refId": "A",
              "timeField": "@timestamp"
            }
          ],
          "title": "域名访问Agent TOP 10",
          "transform": "table",
          "transparent": false,
          "type": "table"
        },
        {
          "columns": [],
          "datasource": "logstash-nginxlog",
          "fontSize": "100%",
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 12,
            "y": 70
          },
          "id": 8,
          "links": [],
          "pageSize": null,
          "scroll": true,
          "showHeader": true,
          "sort": {
            "col": 1,
            "desc": true
          },
          "styles": [
            {
              "alias": "Time",
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Time",
              "type": "number",
              "unit": "none"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "number",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "bucketAggs": [
                {
                  "field": "remote_addr.keyword",
                  "id": "2",
                  "settings": {
                    "min_doc_count": 1,
                    "order": "desc",
                    "orderBy": "_count",
                    "size": "10"
                  },
                  "type": "terms"
                }
              ],
              "expr": "",
              "format": "table",
              "intervalFactor": 1,
              "metrics": [
                {
                  "field": "select field",
                  "id": "1",
                  "type": "count"
                }
              ],
              "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
              "refId": "A",
              "timeField": "@timestamp"
            }
          ],
          "title": "域名客户端IP TOP 10",
          "transform": "table",
          "type": "table"
        },
        {
          "columns": [],
          "datasource": "logstash-nginxlog",
          "fontSize": "100%",
          "gridPos": {
            "h": 9,
            "w": 24,
            "x": 0,
            "y": 79
          },
          "id": 10,
          "links": [],
          "pageSize": null,
          "scroll": true,
          "showHeader": true,
          "sort": {
            "col": 1,
            "desc": true
          },
          "styles": [
            {
              "alias": "Time",
              "dateFormat": "YYYY-MM-DD HH:mm:ss",
              "pattern": "Time",
              "type": "date"
            },
            {
              "alias": "",
              "colorMode": null,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "decimals": 2,
              "pattern": "/.*/",
              "thresholds": [],
              "type": "number",
              "unit": "short"
            }
          ],
          "targets": [
            {
              "bucketAggs": [
                {
                  "fake": true,
                  "field": "request.keyword",
                  "id": "3",
                  "settings": {
                    "min_doc_count": 1,
                    "order": "desc",
                    "orderBy": "_count",
                    "size": "10"
                  },
                  "type": "terms"
                }
              ],
              "expr": "",
              "format": "table",
              "intervalFactor": 1,
              "metrics": [
                {
                  "field": "select field",
                  "id": "1",
                  "type": "count"
                }
              ],
              "query": "http_host.keyword:$hostname  AND status.keyword:$status  AND  upstream_addr.keyword:$upstream_addr  AND department.department.keyword:$department",
              "refId": "A",
              "timeField": "@timestamp"
            }
          ],
          "title": "域名Request TOP 10",
          "transform": "table",
          "type": "table"
        }
      ],
      "title": "Row title",
      "type": "row"
    }
  ],
  "refresh": false,
  "schemaVersion": 16,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_LOGSTASH-NGINXLOG}",
        "hide": 0,
        "includeAll": true,
        "label": "业务",
        "multi": true,
        "name": "department",
        "options": [],
        "query": "{\"find\": \"terms\", \"field\": \"department.department.keyword\"}",
        "refresh": 1,
        "regex": "",
        "sort": 2,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_LOGSTASH-NGINXLOG}",
        "hide": 0,
        "includeAll": true,
        "label": null,
        "multi": true,
        "name": "hostname",
        "options": [],
        "query": "{\"find\": \"terms\", \"field\": \"http_host.keyword\"}",
        "refresh": 1,
        "regex": "",
        "sort": 3,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_LOGSTASH-NGINXLOG}",
        "hide": 0,
        "includeAll": true,
        "label": null,
        "multi": true,
        "name": "status",
        "options": [],
        "query": "{\"find\": \"terms\", \"field\": \"status.keyword\" ,\"query\": \"http_host:$hostname\"}",
        "refresh": 1,
        "regex": "",
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "${DS_LOGSTASH-NGINXLOG}",
        "hide": 0,
        "includeAll": true,
        "label": null,
        "multi": true,
        "name": "upstream_addr",
        "options": [],
        "query": "{\"find\": \"terms\", \"field\": \"upstream_addr.keyword\",\"query\": \"http_host:$hostname\"}",
        "refresh": 1,
        "regex": "",
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "腾讯云CLB日志分析",
  "uid": "7PJIMq5ik",
  "version": 8
}