docker-compose部署elfk

忘是亡心i 2023-06-30 07:00 59阅读 0赞

通过docker-compose部署elfk。elfk最新版本：7.5.1

mkdir -p /home/elfk/{ elasticsearch,logstash,kibana,filebeat}/config
    
    mkdir /home/elfk/elasticsearch/data /home/elfk/logstash/pipeline
    
    cd /home/elfk
    
    echo 'ELK_VERSION=7.5.1' > .env

tree .
    
    .
    ├── docker-compose.yml
    ├── elasticsearch
    │   ├── config
    │   │   └── elasticsearch.yml
    │   ├── data
    │   └── Dockerfile
    ├── filebeat
    │   ├── config
    │   │   └── filebeat.yml
    │   └── Dockerfile
    ├── kibana
    │   ├── config
    │   │   └── kibana.yml
    │   └── Dockerfile
    └── logstash
        ├── config
        │   └── logstash.yml
        ├── Dockerfile
        └── pipeline
            └── logstash.conf
    
    10 directories, 10 files

--------------------

### elasticsearch ###

Dockerfile

vim /home/elfk/elasticsearch/Dockerfile

ARG ELK_VERSION=7.5.1
    
    # https://github.com/elastic/elasticsearch-docker
    # FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
    FROM elasticsearch:${ELK_VERSION}
    # Add your elasticsearch plugins setup here
    # Example: RUN elasticsearch-plugin install analysis-icu

elasticsearch.yml

vim /home/elfk/elasticsearch/config/elasticsearch.yml

---
    ## Default Elasticsearch configuration from Elasticsearch base image.
    ## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
    #
    cluster.name: "docker-cluster"
    network.host: 0.0.0.0
    
    ## X-Pack settings
    ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
    #
    xpack.license.self_generated.type: trial                #trial为试用版，一个月期限，可更改为basic版本
    xpack.security.enabled: true
    xpack.monitoring.collection.enabled: true
    
    http.cors.enabled: true
    http.cors.allow-origin: "*"
    http.cors.allow-headers: Authorization,X-Requested-With,Content-Length,Content-Type

--------------------

### logstash ###

Dockerfile

vim /home/elfk/logstash/Dockerfile

ARG ELK_VERSION=7.5.1
    
    # https://github.com/elastic/logstash-docker
    # FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
    FROM logstash:${ELK_VERSION}
    
    # Add your logstash plugins setup here
    # Example: RUN logstash-plugin install logstash-filter-json
    
    RUN logstash-plugin install logstash-filter-multiline \
        && logstash-plugin install logstash-output-zabbix

logstash.yml

vim /home/elfk/logstash/config/logstash.yml

---
    ## Default Logstash configuration from Logstash base image.
    ## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
    #
    http.host: "0.0.0.0"
    xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
    
    ## X-Pack security credentials
    #
    xpack.monitoring.enabled: true
    xpack.monitoring.elasticsearch.username: elastic
    xpack.monitoring.elasticsearch.password: changeme
    xpack.monitoring.collection.interval: 10s

pipeline

vim /home/elfk/logstash/pipeline/logstash.conf

input {
        tcp {
        	port => 5000
        }
    }
    
    #input {
    #    beats {
    #        port => 5044
    #    }
    #}
    
    output {
        elasticsearch {
        	hosts => "elasticsearch:9200"
        	user => "elastic"
        	password => "changeme"
        }
    }

--------------------

### kibana ###

Dockerfile

vim /home/elfk/kibana/Dockerfile

ARG ELK_VERSION=7.5.1
    
    # https://github.com/elastic/kibana-docker
    # FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
    FROM kibana:${ELK_VERSION}
    
    # Add your kibana plugins setup here
    # Example: RUN kibana-plugin install <name|url>

kibana.yml

vim /home/elfk/kibana/config/kibana.yml

---
    ## Default Kibana configuration from Kibana base image.
    ## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
    #
    server.name: kibana
    server.host: "0"
    elasticsearch.hosts: [ "http://elasticsearch:9200" ]
    xpack.monitoring.ui.container.elasticsearch.enabled: true
    
    ## X-Pack security credentials
    #
    elasticsearch.username: elastic
    elasticsearch.password: changeme

--------------------

### docker-compose.yml ###

vim /home/elfk/docker-compose.yml

version: '3.7'
    
    services:
      elasticsearch:
        build:
          context: elasticsearch/
          args:
            ELK_VERSION: $ELK_VERSION
        volumes:
          - type: bind
            source: ./elasticsearch/config/elasticsearch.yml
            target: /usr/share/elasticsearch/config/elasticsearch.yml
            read_only: true
          - type: volume
            source: elasticsearch
            target: /usr/share/elasticsearch/data
        ports:
          - "9200:9200"
          - "9300:9300"
        environment:
          ES_JAVA_OPTS: "-Xmx256m -Xms256m"
          ELASTIC_PASSWORD: changeme
          discovery.type: single-node
        networks:
          - elk
    
      logstash:
        depends_on:
          - elasticsearch
        build:
          context: logstash/
          args:
            ELK_VERSION: $ELK_VERSION
        volumes:
          - type: bind
            source: ./logstash/config/logstash.yml
            target: /usr/share/logstash/config/logstash.yml
            read_only: true
          - type: bind
            source: ./logstash/pipeline
            target: /usr/share/logstash/pipeline
            read_only: true
        ports:
          - "5000:5000"
          - "9600:9600"
        environment:
          LS_JAVA_OPTS: "-Xmx256m -Xms256m"
        networks:
          - elk
    
      kibana:
        depends_on:
          - elasticsearch
        build:
          context: kibana/
          args:
            ELK_VERSION: $ELK_VERSION
        volumes:
          - type: bind
            source: ./kibana/config/kibana.yml
            target: /usr/share/kibana/config/kibana.yml
            read_only: true
        ports:
          - "5601:5601"
        networks:
          - elk
    
    volumes:
      elasticsearch:
        driver: local
        driver_opts:
          type: none
          o: bind
          device: /home/elfk/elasticsearch/data
          
    networks:
      elk:
        driver: bridge

if [ $(grep 'vm.max_map_count' /etc/sysctl.conf |wc -l) -eq 0 ] ; \
    then echo 'vm.max_map_count=655360' >> /etc/sysctl.conf; \
    fi
    
    sysctl -p
    
    docker-compose up --build -d

docker ps
    
    CONTAINER ID        IMAGE                COMMAND                  CREATED             STATUS              PORTS                                                      NAMES
    448e2f464297        elfk_logstash        "/usr/local/bin/dock…"   12 seconds ago      Up 9 seconds        0.0.0.0:5000->5000/tcp, 0.0.0.0:9600->9600/tcp, 5044/tcp   elfk_logstash_1
    ca391653eb39        elfk_kibana          "/usr/local/bin/dumb…"   12 seconds ago      Up 9 seconds        0.0.0.0:5601->5601/tcp                                     elfk_kibana_1
    2cd23fd30f61        elfk_elasticsearch   "/usr/local/bin/dock…"   14 seconds ago      Up 12 seconds       0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp             elfk_elasticsearch_1
    
    netstat -lntp
    
    Active Internet connections (only servers)
    Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
    tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      52770/sshd          
    tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      23263/master        
    tcp6       0      0 :::9200                 :::*                    LISTEN      127138/docker-proxy 
    tcp6       0      0 :::9300                 :::*                    LISTEN      127126/docker-proxy 
    tcp6       0      0 :::22                   :::*                    LISTEN      52770/sshd          
    tcp6       0      0 ::1:25                  :::*                    LISTEN      23263/master        
    tcp6       0      0 :::9600                 :::*                    LISTEN      127275/docker-proxy 
    tcp6       0      0 :::5601                 :::*                    LISTEN      127237/docker-proxy 
    tcp6       0      0 :::5000                 :::*                    LISTEN      127300/docker-proxy

--------------------

### filebeat ###

Dockerfile

vim /home/elfk/filebeat/Dockerfile

ARG ELK_VERSION=7.5.1
    
    # https://github.com/elastic/beats-docker
    # FROM docker.elastic.co/beats/filebeat:${ELK_VERSION}
    FROM elastic/filebeat:${ELK_VERSION}
    COPY config/filebeat.yml /usr/share/filebeat/filebeat.yml

filebeat.yml

vim /home/elfk/filebeat/config/filebeat.yml

---
    filebeat.inputs:
    - type: log
      enabled: false
      paths:
        - /var/log/*.log
    
    filebeat.config.modules:
      path: ${ path.config}/modules.d/*.yml
      reload.enabled: false
    
    processors:
      - add_host_metadata: ~
      - add_cloud_metadata: ~
      
    output.elasticsearch:
      hosts: ['elasticsearch:9200']
      username: elastic
      password: changeme
     
    #output.logstash:
      # The Logstash hosts
      #hosts: ["localhost:5044"]

cd filebeat/
    
    docker build -t elfk_filebeat:latest .

--------------------

### 日志收集 ###

*  登录kibana：

地址：`ip:5601`

账号：`elastic`，密码：`changeme`

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70]

*  修改filebeat配置：

docker run -d \
      --name=filebeat \
      --user=root \
      -v /var/lib/docker/containers:/var/lib/docker/containers:ro \
      -v /var/run/docker.sock:/var/run/docker.sock:ro \
      -v /home/logs/nginx.log:/home/logs/nginx.log:rw \
      elfk_filebeat:latest filebeat -e -strict.perms=false \
      -E output.elasticsearch.hosts=["localhost:9200"]
    
    docker exec -it filebeat bash
    
    vi /usr/share/filebeat/filebeat.yml

---
    filebeat.inputs:
    - type: log
      enabled: false
      paths:
        - /var/log/*.log
    
    - type: log
      enabled: true
      paths:
        - /home/logs/nginx.log
    
    filebeat.config.modules:
      path: ${ path.config}/modules.d/*.yml
      reload.enabled: false
    
    processors:
      - add_host_metadata: ~
      - add_cloud_metadata: ~
     
    output.elasticsearch:
      hosts: ['elasticsearch:9200']
      username: elastic
      password: changeme
    
    #output.logstash:
      #hosts: ["localhost:5044"]
      #enabled: true
      #worker: 1
      #compression_level: 3

docker restart filebeat
    
    mkdir /home/logs
    
    echo '该消息通过filebeat容器到达es容器，111！！！' > /home/logs/nginx.log
    
    echo '该消息通过filebeat容器到达es容器，222！！！' >> /home/logs/nginx.log
    
    echo '该消息通过filebeat容器到达es容器，333！！！' >> /home/logs/nginx.log

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70 1]

kibana创建index，可以看到一个filebeat index自动生成，因为没有在filebeat配置文件中自定义index。

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70 2]

这就达到了收集日志的目的。

将所有要收集的日志全部存放在宿主机的一个目录下，然后将该目录映射到filebeat容器内（注意给`rw`权限），接着在filebeat配置文件中配置收集并自定义index，从而完成日志收集。

这里只是演示filebeat将日志直接传输到elasticsearch的过程，其实并不太合适，因为日志没有做任何处理。

应该将filebeat的日志传输至logstash，在logstash中做日志处理，然后由logstash传输至elasticsearch，最后在kibana上展示出来，这样还可以结合zabbix完成错误日志告警的目的。

已存放至个人gitgub：[docker-compose][]

--------------------

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200115180557104.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/2020011518061977.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70 2]: https://img-blog.csdnimg.cn/20200115180628752.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz,size_16,color_FFFFFF,t_70
[docker-compose]: https://github.com/Tobewont/docker-compose