Filebeat 部署文档

cd /usr/local/srcwget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.15.2-linux-x86_64.tar.gz

cd /usr/local/srctar xzvf filebeat-8.15.2-linux-x86_64.tar.gzmv filebeat-8.15.2-linux-x86_64 /usr/local/filebeat-8.15.2cd /usr/local/ln -s filebeat-8.15.2 filebeat

mkdir -p /data/filebeat_data/{conf,data,tmp}cd /usr/local/filebeat# 配置文件mv filebeat.yml /data/filebeat_data/conf/ln -s /data/filebeat_data/conf/filebeat.yml /usr/local/filebeat/filebeat.yml# 日志目录mkdir -p /data/logs/filebeatln -s /data/logs/filebeat /usr/local/filebeat/logs# 数据目录ln -s /data/filebeat_data/data /usr/local/filebeat/data# modules.d目录mv modules.d /data/filebeat_data/ln -s /data/filebeat_data/modules.d /usr/local/filebeat/modules.d

cat <<"EOF" | tee /usr/lib/systemd/system/filebeat.service[Unit]Description=Filebeat sends log files to Logstash or directly to Elasticsearch.Documentation=https://www.elastic.co/products/beats/filebeatWants=network-online.targetAfter=network-online.target[Service]WorkingDirectory=/usr/local/filebeatEnvironment="BEAT_CONFIG_OPTS=-c /usr/local/filebeat/filebeat.yml"ExecStart=/usr/local/filebeat/filebeat --environment systemd $BEAT_CONFIG_OPTS Restart=always[Install]WantedBy=multi-user.targetEOF

path.home: /usr/local/filebeatpath.config: /usr/local/filebeatpath.data: /usr/local/filebeat/datapath.logs: /usr/local/filebeat/logslogging.level: infologging.to_files: truelogging.files:  path: /usr/local/filebeat/logs  name: filebeat.log  keepfiles: 7  permissions: 0644filebeat.config.modules:  path: /usr/local/filebeat/modules.d/*.yml  reload.enabled: true  reload.period: 10s

filebeat.inputs:- type: log  enabled: true  paths:    - /var/log/history.log  processors:    - add_fields:        target: log        fields:          type: "linux-history"

filebeat.inputs:- type: log  enabled: true  paths:    - /data/bizlogs/tomcat/catalina.out  encoding: gbk  processors:    - add_fields:        target: log        fields:          type: "java"          app_name: "tomcat"  multiline.pattern: '^(\d\d){1,2}-(?:0[1-9]|1[0-2])-(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)'  multiline.negate: true  multiline.match: after- type: log  enabled: true  paths:    - /data/bizlogs/java/*/*.log  processors:    - add_fields:        target: log        fields:          type: "java"          app_name: "jar"  multiline.pattern: '^(\d\d){1,2}-(?:0[1-9]|1[0-2])-(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)'  multiline.negate: true  multiline.match: after

cd /usr/local/filebeat/echo '# Module: mysql# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-module-mysql.html- module: mysql  error:    enabled: true    var.paths: ["/data/logs/mysql/3306/error.log*"]  slowlog:    enabled: true    var.paths: ["/data/logs/mysql/3306/slow.log*"]' > modules.d/mysql.yml./filebeat modules enable mysql./filebeat modules list

filebeat.autodiscover:  providers:    - type: docker      containers:        path: "/var/lib/docker/containers/"      hints.enabled: true      # 选项为true，默认所有容器开启采集      # 选项为false，配置了co.elastic.logs/enabled: true的容器才开启采集      hints.default_config.enabled: false

filebeat.autodiscover:  providers:    - type: kubernetes      add_resource_metadata:        deployment: true      hints.enabled: true      hints.default_config:        type: container        paths:          - /var/log/containers/*.log  # CRI path        processors:        - add_fields:            target: ""            fields:              log_type: "k8slog"        multiline:           pattern: ^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2} #匹配Java日志开头时间           negate: true # 正则是否开启，默认false不开启           match: after

filebeat.inputs:- type: container  enabled: true  paths:    - /var/log/containers/*.log  processors:    - script:        lang: javascript        id: k8s_metadata        source: >          function process(event) {            event.Tag("js");            var path = event.Get('log.file.path');            var file_name = path.split('/')[4]            var pod_str = file_name.split('.')[0]            var pod_info = pod_str.split('_')            var container_info = pod_info[pod_info.length-1].split('-')            event.Put('k8s.pod_name', pod_info[0])            event.Put('k8s.namesapce_name', pod_info[1]);            event.Put('k8s.container_name', container_info.slice(0,-1).join('-'));            event.Put('k8s.container_id', container_info[container_info.length-1]);          }

output.elasticsearch:  hosts: ["http://127.0.0.1:9200"]  username: "elastic"  password: "YOUR_PASSWORD"

output.logstash:  hosts:    - "127.0.0.1:5044"

output.kafka:  hosts: ['127.0.0.1:9092']  topic: "mysql-slow-log"

systemctl daemon-reloadsystemctl enable filebeat --now