Graylog服务收集k8s容器日志

前言

  在采集k8s pod日志的时候有许多种方案，比如：边车模式，elk ,efk ,本文结合efk的方式，使用 graylog 服务，通过fluentd来进行日志收集
  服务器压力个人评估，graylog 可以采取集群的方式，服务压力主要在 ES 上，可以通过graylog服务，建立多个索引来对应k8s的命名空间

graylog服务组件

  graylog-server: 服务端，日志采集入口;
  mongodb：用户/角色/权限数据，存配置、用户、Stream 规则、Dashboard 定义、告警历史、Sidecar 标签；
  Elasticsearch: 真正存日志、倒排索引、聚合计算；Graylog **对其版本有严格要求**
  nxlog,fluentd,logstash: 日志采集agent

架构图

# docker部署graylog-Server

docker-compose.yaml

点击展开docker-compose.yaml 文本

mkdir -p /data/graylog && cd /data/graylog

cat >docker-compose.yaml<<'EOF'
version: '3'
services:
  graylog-nginx:
    restart: always
    image: docker.1ms.run/nginx:1.26.3
    container_name: graylog-nginx
    ports:
      - "0.0.0.0:19009:19009"
    volumes:
      - ./nginx/conf.d:/etc/nginx/conf.d
      - /etc/localtime:/etc/localtime
      - /etc/timezone:/etc/timezone
    networks:
      graylog_net:
        ipv4_address: 10.19.200.29
    #logging:
    #  driver: gelf
    #  options:
    #    gelf-address: "tcp://192.168.123.201:8161"
    #    tag: "graylog-nginx"

  graylog-server:
    container_name: graylog-server
    image: docker.1ms.run/graylog/graylog:5.2.9
    environment:
      - TZ=Asia/Shanghai
      - GRAYLOG_PASSWORD_SECRET=Dajidali@1qaz2wsx
      - GRAYLOG_ROOT_PASSWORD_SHA2=6b242e0ce165c53fa9b8bda602dbab5db796d18890410c2052d77fc9ba1aaddd
      - GRAYLOG_HTTP_EXTERNAL_URI=http://127.0.0.1:9009/
      - GRAYLOG_ELASTICSEARCH_HOSTS=http://elasticsearch:9200
      - GRAYLOG_MONGODB_URI=mongodb://mongo:27017/graylog
    user: "0"
    entrypoint: /usr/bin/tini -- wait-for-it elasticsearch:9200 --  /docker-entrypoint.sh
    #如果不是容器部署es,修改 elasticsearch 映射IP
    extra_hosts:
      - "mongo:10.19.200.110"
      - "elasticsearch:10.19.200.112"
    volumes:
      - /etc/localtime:/etc/localtime
      - /etc/timezone:/etc/timezone
      - ./graylog/config/graylog.conf:/usr/share/graylog/data/config/graylog.conf
      #- ./graylog/data:/usr/share/graylog/data
#    network_mode: host
    ports:
#      - "9009:9009"  # Web interface,通过nginx代理
      - "0.0.0.0:8161:8161"
      - "0.0.0.0:8162:8162"
    networks:
      graylog_net:
        ipv4_address: 10.19.200.10
    deploy:
      resources:
         limits:
            memory: 2G
         reservations:
            memory: 1G



  graylog-elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
    container_name: graylog-elasticsearch
    user: "1000:1000"
    environment:
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms1536m -Xmx1536m"
    ulimits:
      memlock:
        soft: 65536
        hard: 65536
    volumes:
      - ./elasticsearch/data:/usr/share/elasticsearch/data
      - /etc/localtime:/etc/localtime
      - /etc/timezone:/etc/timezone
    #ports:
    #  - "127.0.0.1:9200:9200"
    networks:
      graylog_net:
        ipv4_address: 10.19.200.112
    deploy:
      resources:
        limits:
          memory: 2G

  graylog-mongo:
    image: docker.1ms.run/mongo:5
    container_name: graylog-mongo
    volumes:
      - ./mongodata:/data/db
      - /etc/localtime:/etc/localtime
      - /etc/timezone:/etc/timezone
    networks:
      graylog_net:
        ipv4_address: 10.19.200.110

networks:
  graylog_net:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 10.19.200.0/24
EOF

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

启动

docker-compose -f docker-compose.yaml up -d

# 登录graylog-server

http://xxx:19009/loki/
admin
Dajidali@1qaz2wsx

1
2
3

# 配置graylog-server

system/inputs

# 配置input

配置input(gelf-tcp)

注意： 这个端口需要和docker 映射的端口一致，在配置fluentd 的configmap的时候，需要调整IP和端口

# k8s部署fluentd组件

命名空间: logging

根据实际情况修改 fluentd-configmap.yaml 最后 10行内容

kubectl create ns logging
kubectl apply -f fluentd-configmap.yaml
kubectl apply -f fluentd-daemonset.yaml
kubectl get pod -n logging

1
2
3
4

# 测试创建pod,给需要收集日志的pod打标签

创建deployment,给deployment打标签: "logging=true"

kubectl create deployment test01 --image=docker.1ms.run/nginx:1.26.3
kubectl patch deployment test01 -p '{"spec":{"template":{"metadata":{"labels":{"logging":"true"}}}}}'

1
2

# 查看graylog-server接收日志情况

到此，说明已经成功将日志收集到 graylog-server

# 创建streams测试

再次创建一个测试的容器，比如我想收集命名空间：platform
可能理解误区：根据命名空间单独创建索引，stream，这个其实可以都写在一个stream里面，根据stream里面的规则来判断日志的最终写入

kubectl create  ns platform
kubectl -n platform create deployment test02 --image=docker.1ms.run/nginx:1.26.3
kubectl -n platform patch deployment test02 -p '{"spec":{"template":{"metadata":{"labels":{"logging":"true"}}}}}'

1
2
3

graylog-server创建的时候会在es创建默认的索引，es数据量过多可能会查询过慢的问题，根据自己的需求创建索引，比如按命名空间创建es索引

system/indices