kubernetes集群EFK日志系统搭建

怼烎@ 2021-11-10 19:36 373阅读 0赞

# 日志收集架构 #

Kubernetes 集群本身不提供日志收集的解决方案，一般来说有主要的3种方案来做日志收集：

*  在节点上运行一个 agent 来收集日志
 *  在 Pod 中包含一个 sidecar 容器来收集应用日志
 *  直接在应用程序中将日志信息推送到采集后端

# 搭建 EFK 日志系统 #

elasticsearch安装使用集群外部环境

192.168.1.122  9200

kubernetes集群创建名称空间

kubectl create namespace logging

## 创建configmap,添加fluentd配置文件 ##

注意：修改host 和port

## 部署fluentd ##

[root@k8s-master ~]# cat fluentd-configmap.yaml 
    kind: ConfigMap
    apiVersion: v1
    metadata:
      name: fluentd-config
      namespace: logging
      labels:
        addonmanager.kubernetes.io/mode: Reconcile
    data:
      system.conf: |-
        <system>
          root_dir /tmp/fluentd-buffers/
        </system>
      containers.input.conf: |-
        <source>
          @id fluentd-containers.log
          @type tail
          path /var/log/containers/*.log
          pos_file /var/log/es-containers.log.pos
          time_format %Y-%m-%dT%H:%M:%S.%NZ
          localtime
          tag raw.kubernetes.*
          format json
          read_from_head true
        </source>
        # Detect exceptions in the log output and forward them as one log entry.
        <match raw.kubernetes.**>
          @id raw.kubernetes
          @type detect_exceptions
          remove_tag_prefix raw
          message log
          stream stream
          multiline_flush_interval 5
          max_bytes 500000
          max_lines 1000
        </match>
      system.input.conf: |-
        # Logs from systemd-journal for interesting services.
        <source>
          @id journald-docker
          @type systemd
          filters [{ "_SYSTEMD_UNIT": "docker.service" }]
          <storage>
            @type local
            persistent true
          </storage>
          read_from_head true
          tag docker
        </source>
        <source>
          @id journald-kubelet
          @type systemd
          filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
          <storage>
            @type local
            persistent true
          </storage>
          read_from_head true
          tag kubelet
        </source>
      forward.input.conf: |-
        # Takes the messages sent over TCP
        <source>
          @type forward
        </source>
      output.conf: |-
        # Enriches records with Kubernetes metadata
        <filter kubernetes.**>
          @type kubernetes_metadata
        </filter>
        <match **>
          @id elasticsearch
          @type elasticsearch
          @log_level info
          include_tag_key true
          host 192.168.1.122
          port 9200
          logstash_format true
          request_timeout    30s
          <buffer>
            @type file
            path /var/log/fluentd-buffers/kubernetes.system.buffer
            flush_mode interval
            retry_type exponential_backoff
            flush_thread_count 2
            flush_interval 5s
            retry_forever
            retry_max_interval 30
            chunk_limit_size 2M
            queue_limit_length 8
            overflow_action block
          </buffer>
        </match>
    [root@k8s-master ~]# cat fluentd-daemonset.yaml 
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: fluentd-es
      namespace: logging
      labels:
        k8s-app: fluentd-es
        kubernetes.io/cluster-service: "true"
        addonmanager.kubernetes.io/mode: Reconcile
    ---
    kind: ClusterRole
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: fluentd-es
      labels:
        k8s-app: fluentd-es
        kubernetes.io/cluster-service: "true"
        addonmanager.kubernetes.io/mode: Reconcile
    rules:
    - apiGroups:
      - ""
      resources:
      - "namespaces"
      - "pods"
      verbs:
      - "get"
      - "watch"
      - "list"
    ---
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: fluentd-es
      labels:
        k8s-app: fluentd-es
        kubernetes.io/cluster-service: "true"
        addonmanager.kubernetes.io/mode: Reconcile
    subjects:
    - kind: ServiceAccount
      name: fluentd-es
      namespace: logging
      apiGroup: ""
    roleRef:
      kind: ClusterRole
      name: fluentd-es
      apiGroup: ""
    ---
    apiVersion: apps/v1
    kind: DaemonSet
    metadata:
      name: fluentd-es
      namespace: logging
      labels:
        k8s-app: fluentd-es
        version: v2.0.4
        kubernetes.io/cluster-service: "true"
        addonmanager.kubernetes.io/mode: Reconcile
    spec:
      selector:
        matchLabels:
          k8s-app: fluentd-es
          version: v2.0.4
      template:
        metadata:
          labels:
            k8s-app: fluentd-es
            kubernetes.io/cluster-service: "true"
            version: v2.0.4
          # This annotation ensures that fluentd does not get evicted if the node
          # supports critical pod annotation based priority scheme.
          # Note that this does not guarantee admission on the nodes (#40573).
          annotations:
            scheduler.alpha.kubernetes.io/critical-pod: ''
        spec:
          serviceAccountName: fluentd-es
          containers:
          - name: fluentd-es
            image: cnych/fluentd-elasticsearch:v2.0.4
            env:
            - name: FLUENTD_ARGS
              value: --no-supervisor -q
            resources:
              limits:
                memory: 500Mi
              requests:
                cpu: 100m
                memory: 200Mi
            volumeMounts:
            - name: varlog
              mountPath: /var/log
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: config-volume
              mountPath: /etc/fluent/config.d
          nodeSelector:
            beta.kubernetes.io/fluentd-ds-ready: "true"
          tolerations:
          - key: node-role.kubernetes.io/master
            operator: Exists
            effect: NoSchedule
          terminationGracePeriodSeconds: 30
          volumes:
          - name: varlog
            hostPath:
              path: /var/log
          - name: varlibdockercontainers
            hostPath:
              path: /var/lib/docker/containers
          - name: config-volume
            configMap:
              name: fluentd-config
    [root@k8s-master ~]#

创建节点标签

[root@k8s-master ~]# kubectl label nodes k8s-master beta.kubernetes.io/fluentd-ds-ready=true
    [root@k8s-master ~]# kubectl label nodes k8s-node1 beta.kubernetes.io/fluentd-ds-ready=true
    
    [root@k8s-master ~]# kubectl get nodes --show-labels
    NAME         STATUS   ROLES    AGE   VERSION   LABELS
    k8s-master   Ready    master   45d   v1.13.1   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/fluentd-ds-ready=true,beta.kubernetes.io/os=linux,kubernetes.io/hostname=k8s-master,node-role.kubernetes.io/master=
    k8s-node1    Ready    <none>   45d   v1.13.1   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/fluentd-ds-ready=true,beta.kubernetes.io/os=linux,kubernetes.io/hostname=k8s-node1
    [root@k8s-master ~]#

### 　最后应用配置文件 ###

kubectl apply  -f fluentd-daemonset.yaml

## 查看pods情况 ##

[root@k8s-master ~]# kubectl get pods -n logging
    NAME               READY   STATUS    RESTARTS   AGE
    fluentd-es-pjcpx   1/1     Running   0          72m
    fluentd-es-x5bck   1/1     Running   0          72m
    [root@k8s-master ~]#

最后就可以在kibana的dashboard上展示

如下过滤kubernetes集群入口ingressgateway日志信息

![1252720-20190729185105199-1761921930.png][]

转载于:https://www.cnblogs.com/xuliang666/p/11265773.html

[1252720-20190729185105199-1761921930.png]: /images/20211109/db896b94d3b346639be807f6656f4c7f.png