filebeat部署

# filebeat部署

日志收集，结合开源项目 ZLT

# 制作filebeat镜像

FROM debian:jessie
# 在官网下载同版本LINUX64-BIT的sha替换
ENV FILEBEAT_VERSION=7.5.1 
ENV FILEBEAT_SHA1=daf1a5e905c415daf68a8192a069f913a1d48e2c79e270da118385ba12a93aaa91bda4953c3402a6f0abf1c177f7bcc916a70bcac41977f69a6566565a8fae9c


RUN set -x && \
 apt-get update && \
 apt-get install -y wget && \
 wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz -O /opt/filebeat.tar.gz && \
 cd /opt && \
 echo "${FILEBEAT_SHA1} filebeat.tar.gz" | sha512sum -c - && \
 tar xzvf filebeat.tar.gz && \
 cd filebeat-* && \
 cp filebeat /bin && \
 cd /opt && \
 rm -rf filebeat* && \
 apt-get purge -y wget && \
 apt-get autoremove -y && \
 apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
COPY docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

docker-entrypoint.sh脚本

vi docker-entrypoint.sh

#!/bin/bash

ENV=${ENV:-"test"} 
PROJ_NAME=${PROJ_NAME:-"no-define"} 
MULTILINE=${MULTILINE:-"^\["}  
#MULTILINE=${MULTILINE:-"^\d{2}"}  

cat > /etc/filebeat.yaml << EOF
filebeat.inputs:
- type: log
  fields_under_root: true
  fields:
    topic: logm-${PROJ_NAME}
  paths:
    - /logm/*.log
    - /logm/*/*.log
    - /logm/*/*/*.log
    - /logm/*/*/*/*.log
    - /logm/*/*/*/*/*.log
  scan_frequency: 120s
  max_bytes: 10485760
  multiline.pattern: '$MULTILINE'
  multiline.negate: true
  multiline.match: after
  multiline.max_lines: 100
- type: log
  fields_under_root: true
  fields:
    topic: logu-${PROJ_NAME}
  paths:
    - /logu/*.log
    - /logu/*/*.log
    - /logu/*/*/*.log
    - /logu/*/*/*/*.log
    - /logu/*/*/*/*/*.log
    - /logu/*/*/*/*/*/*.log
output.kafka:
  hosts: ["hdss14-21.host.com:9092"]
  topic: k8s-fb-$ENV-%{[topic]}
  # kafka版本超过2.0，默认写2.0.0 
  version: 2.0.0
  required_acks: 0
  max_message_bytes: 10485760
EOF

set -xe

# If user don't provide any command
# Run filebeat
if [[ "$1" == "" ]]; then
     exec filebeat  -c /etc/filebeat.yaml 
else
    # Else allow the user to run arbitrarily commands like bash
    exec "$@"
fi

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

构建镜像

docker build . -t harbor.yfklife.cn/devops/filebeat:v7.5.1
docker push harbor.yfklife.cn/devops/filebeat:v7.5.1

1
2

# 实战日志收集（microservices-platform项目）

# 准备资源清单

制作镜像

个人存储包下载跳转

wget https://XXXXXXX/zlt-d907c13.zip #我个人制作好的包和配置

#修改mysql，redis连接信息
application-dev.properties

#修改nacos连接信息
bootstrap-dev.properties

#修改前端api地址,
grep -rl 192.168.14.200:9900 zlt-web/ #修改你的网关域名地址,对应traefik路由配置：http://zlt.yfklife.cn/api

#进入到目录：gateway-service，user-center，zlt-uaa，zlt-web 进行构建镜像

1
2
3
4
5
6
7
8
9
10
11
12

项目部署

kubectl create ns zlt
cd zlt-k8syaml
ls -l
for i in `ls`;do kubectl apply -f $i;done

1
2
3
4

zlt-gateway-dp-svc.yml示例

vi zlt-gateway-dp-svc.yml

kind: Service
apiVersion: v1
metadata: 
  name: zlt-gateway
  namespace: zlt
spec:
  ports:
  - protocol: TCP
    port: 9900
    targetPort: 9900
  selector: 
    app: zlt-gateway
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: zlt-gateway
  namespace: zlt
  labels: 
    name: zlt-gateway
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: zlt-gateway
  template:
    metadata:
      labels: 
        app: zlt-gateway
        name: zlt-gateway
    spec:
      containers:
      - name: zlt-gateway
        image: harbor.yfklife.cn/zlt/sc-gateway:5.3.0
        ports:
        - containerPort: 9900
          protocol: TCP
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /logm
          name: logm
      - name: filebeat
        image: harbor.yfklife.cn/devops/filebeat2:v7.5.1
        imagePullPolicy: IfNotPresent
        env:
        - name: ENV
          value: test  #区分后面logstaah环境
        - name: PROJ_NAME
          value: zlt-gateway
        volumeMounts:
        - mountPath: /logm        
          name: logm
      volumes:
      - emptyDir: {}
        name: logm
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

# 个人存储下载地址。。。

点击，获取下面下载资源

如果此文章对您有帮助，点击 -->> 请博主喝咖啡

上次更新: 2023/11/20, 16:23:11

← kafka和kakfa-manager部署 es-logstash-kibana部署日志展示→