docker安装ELK

女爷i 2022-06-09 07:00 390阅读 0赞

第一步：创建目录和配置文件  
mkdir-p /data/server/nginx/conf/  
vim nginx.conf  
\--------------------------------------nginx.conf-------------------------  
worker\_processes 1;  
  
events \{  
    worker\_connections  1024;  
\}  
  
http \{  
    include       mime.types;  
    default\_type  application/octet-stream;  
  
    \# 定义日志格式，和你的 Logstash 中的 grok 过滤规则相匹配  
    log\_format  logstash  '$remote\_addr - $remote\_user \[$time\_local\] "$request" '  
                '$status $body\_bytes\_sent "$http\_referer" '  
                '"$http\_user\_agent" "$http\_x\_forwarded\_for" '  
                '"$http\_host" "$upstream\_addr" '  
                '$request\_time $upstream\_response\_time';  
  
    access\_log  /var/log/nginx/access.log  logstash; \# 设置指定的日志格式  
    error\_log   /var/log/nginx/error.log;  
  
    server \{  
       server\_name  localhost;  
  
        listen                  80;  
        charset utf-8;  
  
        location / \{  
        root   /usr/share/nginx/html;  
        index  index.html index.htm;  
    \}  
  
    \}  
\}  
\------------------------------------------------------------------------  
  
  
第二步：创建目录并授权  
mkdir -p /data/server/nginx/logs  
touch access.log  
chmod 777 access.log  
  
第三步：拉取nginx镜像并执行  
docker pull nginx  
docker run --name some-nginx  -p 80:80 --add-host www.github01.com:192.168.20.38 -v /data/server/nginx/conf/nginx.conf:/etc/nginx/nginx.conf:ro -v /data/server/nginx/logs/access.log:/var/log/nginx/access.log -d nginx  
  
第4步：拉取es镜像并执行,并放通iptables  
docker run -d --name myes -p 9200 -p 9300 --add-host www.github.com:192.168.20.38 elasticsearch  
iptables -I INPUT 2 -p tcp -m state --state NEW --dport 9200 -j ACCEPT  
  
第5步：拉取kibana镜像并执行  
docker run --name mykibana   -e ELASTICSEARCH\_URL=http://www.github.com:9200 -p 5601:5601 --add-host www.github.com:192.168.20.38 -d kibana  
  
第6步：拉取zookeeper镜像、kafka镜像，并执行  
docker pull wurstmeister/zookeeper  
docker pull wurstmeister/kafka  
docker run --name zookeeper -p 2181:2181 -t -d wurstmeister/zookeeper  
docker run --name kafka -e HOST\_IP=192.168.20.38 -e KAFKA\_ADVERTISED\_PORT=9092 -e KAFKA\_BROKER\_ID=1 -e ZK=zk -p 9092:9092 --link zookeeper:zk -t -d wurstmeister/kafka  
  
第7步：进入kafka，创建topic  
docker exec -it $\{CONTAINER ID\} /bin/bash  
bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 1 --topic test  
bin/kafka-topics.sh --list --zookeeper zookeeper:2181  
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test  
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning  
  
  
第8步：创建目录，写配置文件，  
mkdir -p /data/server/logstash/conf/  
vim logstash.conf  
\----------------------------logstash.conf------------------------------  
input \{  
    file \{  
        path => \["/data/server/nginx/logs/access.log"\]  
        type => "nginx\_log"  
        start\_position => "beginning"  
    \}  
\}  
  
\#filter \{  
\#\}  
  
output \{  
   elasticsearch \{  
       hosts => \["www.github.com:9200"\]  
        index => "logstash-nginx"  
        \}  
   \#stdout \{  
        \#codec => rubydebug  
   \# \}  
\}  
\-------------------------logstash.conf ------------------------------------------  
\--------------------------------------------------------------------------------  
input \{  
    file \{  
        path => \["/data/server/nginx/logs/access.log"\]  
        type => "nginx\_log"  
        start\_position => "beginning"  
    \}  
\}  
  
\#filter \{  
\#\}  
  
output \{  
        kafka \{  
                topic\_id => "test"  
                bootstrap\_servers => "192.168.20.38:9092"  
        \}  
\}  
\--------------------------------------------------------------------------------  
  
\---------------------------------------------------------------------------------  
input \{  
 kafka \{  
        zk\_connect => "192.168.20.38:2181"  
        \#group\_id => ""  
        topic\_id => "test"  
        codec => "plain"  
        reset\_beginning => false  
        consumer\_threads => 5  
        decorate\_events => true  
        \#add\_field => \{ "\[@metadata\]\[type\]" => "syslog" \}  
      \}  
\}  
  
\#filter \{  
\#\}  
  
output \{  
   elasticsearch \{  
       hosts => \["www.github.com:9200"\]  
        index => "logstash-nginx"  
        \}  
   \#stdout \{  
        \#codec => rubydebug  
   \# \}  
\}  
\----------------------------------------------------------------------------------  
第9步，执行镜像：  
\#docker run -it -d --name=my\_logstash  --add-host www.github.com:192.168.20.38 -v /data/server/logstash/conf:/config-dir -v /data/server/nginx/logs/access.log:/data/server/nginx/logs/access.log logstash -f /config-dir/logstash.conf  
docker run -it -d --name=my\_logstash  --add-host www.github.com:192.168.20.38 --link kafka -v /data/server/logstash/conf:/config-dir -v /data/server/nginx/logs/access.log:/data/server/nginx/logs/access.log logstash -f /config-dir/logstash.conf  
docker run -it -d --name=my\_kafka\_logstash  --add-host www.github.com:192.168.20.38 --link kafka -v /data/server/logstash/kafka\_conf:/config-dir   logstash  -f /config-dir/logstash.conf