[kubernetes]-k8s安装alertmanager和prometheus-webhook-dingtalk

朱雀 2022-11-13 04:20 485阅读 0赞

#### 安装alertmanager ####

##### 创建存放数据及插件的文件夹 #####

# 在指定的node上创建文件夹
    mkdir -p /data/k8s/alertmanager
    chown -R 65534:root  alertmanager

##### 创建alertmanager-cm.yaml #####

---
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: alertmanager-config
      namespace: kube-ops
    data:
      alertmanager.yml: |-
    
            # 全局配置项
            global:
              resolve_timeout: 5m # 处理超时时间，默认为5min
              smtp_from: 'it@hz-health.cn'
              smtp_smarthost: 'smtp.exmail.qq.com:465'
              #smtp_smarthost: 'smtp.aliyun.com:465'
              smtp_auth_username: 'it@hz-health.cn'
              smtp_auth_password: 'yourpassword'
              smtp_require_tls: false
    
            # 定义路由树信息
            route:
              group_by: [alertname]  # 报警分组依据
              receiver: ops_notify   # 设置默认接收人
              group_wait: 30s        # 最初即第一次等待多久时间发送一组警报的通知
              group_interval: 60s    # 在发送新警报前的等待时间 下一次报警开车时间
              repeat_interval: 1h    # 重复发送告警时间。默认1h       第一次报警时间为group_interval  重复报警的时间为group_interval+repeat_interval
              routes:
    
              - receiver: ops_notify  # 基础告警通知
                group_wait: 10s
                match_re:
                  alertname: 实例存活告警|磁盘使用率告警   # 匹配告警规则中的名称发送
    
              - receiver: info_notify  # 消息告警通知
                group_wait: 10s
                match_re:
                  alertname: 内存使用率告警|CPU使用率告警
    
            # 定义基础告警接收者
            receivers:
            - name: ops_notify
              webhook_configs:
              - url: http://prometheus-webhook-dingtalk.ihaozhuo.com/dingtalk/ops_dingding/send
                send_resolved: true  # 警报被解决之后是否通知
              email_configs:
              # - to: '423308591@qq.com‘  如果两个邮箱中间需要空格
              - to: 'it@hz-health.cn, xujiamin@hz-health.cn'
                send_resolved: true
    
            # 定义消息告警接收者
            - name: info_notify
              webhook_configs:
              - url: http://prometheus-webhook-dingtalk.ihaozhuo.com/dingtalk/info_dingding/send
                send_resolved: true
    
            # 一个inhibition规则是在与另一组匹配器匹配的警报存在的条件下，使匹配一组匹配器的警报失效的规则。两个警报必须具有一组相同的标签。
            inhibit_rules:
              - source_match:
                  severity: 'critical'
                target_match:
                  severity: 'warning'
                equal: ['alertname', 'dev', 'instance']

##### 创建alertmanager-deploy.yaml #####

---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: alertmanager
      namespace: kube-ops
      labels:
        app: alertmanager
    spec:
      replicas: 1
      revisionHistoryLimit: 2
      selector:
        matchLabels:
          app: alertmanager
      minReadySeconds: 0
      strategy:
        type: RollingUpdate
        rollingUpdate:
          maxUnavailable: 1
          maxSurge: 1
      template:
        metadata:
          labels:
            app: alertmanager
        spec:
          affinity:         #亲和性的调度设置
            nodeAffinity:   #策略为节点亲和性
              requiredDuringSchedulingIgnoredDuringExecution:      #亲和性的硬策略
                nodeSelectorTerms:   #这里不再使用nodeselector，使用这个参数可以进行相对简单的逻辑运算
                - matchExpressions:     #匹配表达式
                  - key: kubernetes.io/hostname      #具体匹配规则(可以通过kubectl get node --show-labels找到相应规则)
                    operator: In  #不在，简单的来说就是不在k8s-04节点
                    values:
                    - prod-k8s-n006
    
          containers:
          - name: alertmanager
            image: prom/alertmanager:v0.21.0
            args:
            - "--config.file=/etc/alertmanager/alertmanager.yml"
            - "--storage.path=/alertmanager/data"
            resources:
              limits:
                memory: "256Mi"
                cpu: "100m"
            readinessProbe:
              tcpSocket:
                port: 9093
              initialDelaySeconds: 30
              periodSeconds: 10
            livenessProbe:
              tcpSocket:
                port: 9093
              initialDelaySeconds: 60
              periodSeconds: 10
            ports:
            - containerPort: 9093
              protocol: TCP
              name: http
    
            volumeMounts:
            - name: data
              mountPath: /alertmanager/data
            - name: config-volume
              mountPath: "/etc/alertmanager/alertmanager.yml"
              subPath: alertmanager.yml
          volumes:
          - name: data
            hostPath:
          # 宿主上目录位置
              path: /data/k8s/alertmanager
              type: DirectoryOrCreate
          - configMap:
              name: alertmanager-config
            name: config-volume

##### 创建alertmanager-svc.yaml #####

---
    apiVersion: v1
    kind: Service
    metadata:
      name: alertmanager
      namespace: kube-ops
      labels:
        app: alertmanager
    spec:
      selector:
        app: alertmanager
      type: NodePort
      ports:
      - port: 80
        protocol: TCP
        targetPort: 9093

##### 创建alertmanager-ingress.yaml #####

---
    apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
    # 通过添加下面的annotations 来开启白名单
    # 关闭80强制跳转443 为ingress配置增加注解（annotations）：nginx.ingress.kubernetes.io/ssl-redirect: 'false' 就可以禁止http强制跳转至https
      annotations:
        #nginx.ingress.kubernetes.io/whitelist-source-range: "60.191.70.64/29, xx.xxx.0.0/16"
        nginx.ingress.kubernetes.io/ssl-redirect: 'false'
      name: prod-alertmanager
      namespace: kube-ops
    spec:
      rules:
        - host: alertmanager.ihaozhuo.com
          http:
            paths:
            - path: /
              backend:
                serviceName: alertmanager
                servicePort: 80

##### 运行alertmanager #####

kubectl apply -f ./

#### 安装prometheus-webhook-dingtalk ####

先看一下prometheus-webhook-dingtalk的参数 打算通过configmap的方式挂载config.yml并开启ui。

之前都是直接启动的时候通过命令`/srv/alertmanager/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile=ops_dingding=https://oapi.dingtalk.com/robot/send?access_token=1234567890 --web.enable-ui` 启动，研究这个config花了点时间

![1a842c1bbf776307701f2a535a69965d.png][]

##### 制作一个prometheus-webhook-dingtalk的镜像 #####

###### 创建Dockerfile ######

FROM centos:centos7.4.1708
    MAINTAINER PDABC Enterprise Container Images <jiaminxu@hz-health.cn>
    
    ADD  prometheus-webhook-dingtalk /etc/prometheus-webhook-dingtalk
    RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
    
    WORKDIR /etc/prometheus-webhook-dingtalk
    
    #公开端口
    EXPOSE 8060
    
    USER root
    #设置启动命令
    ENTRYPOINT ["sh", "-c","/etc/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk"]
    # 想写死的话 也是可以的。
    #ENTRYPOINT ["sh", "-c","/etc/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile=ops_dingding=https://oapi.dingtalk.com/robot/send?access_token=1234567890   --web.enable-ui "]

###### 编译并上床镜像 ######

docker build -t registry.cn-shanghai.aliyuncs.com/yjk-datag/prometheus-webhook-dingtalk:v3  .
    docker push registry.cn-shanghai.aliyuncs.com/yjk-datag/prometheus-webhook-dingtalk:v3

##### 创建prometheus-webhook-dingtalk-cm.yaml #####

https://github.com/timonwong/prometheus-webhook-dingtalk

apiVersion: v1
    kind: ConfigMap
    metadata:
      name: prometheus-webhook-dingtalk-config
      namespace: kube-ops
    data:
      config.yml: |
        targets:
          # 这个是我用到的
          ops_dingding:
            url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
          webhook2:
            url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
          webhook_legacy:
            url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
            # Customize template content
            message:
              # Use legacy template
              title: '{
       { template "default.title" . }}'
              text: '{
       { template "default.content" . }}'
          webhook_mention_all:
            url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
            mention:
              all: true
          webhook_mention_users:
            url: https://oapi.dingtalk.com/robot/send?access_token=10bda98979ae2155b6822b699cde1841d4fbd8514c0441bbbb4485caddf3a388x #这是是钉钉机器人的webhook
            mention:
              mobiles: ['15xxxxxxxx0']

##### 创建prometheus-webhook-dingtalk-deploy.yaml #####

apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: prometheus-webhook-dingtalk
      namespace: kube-ops
      labels:
        app: prometheus-webhook-dingtalk
    spec:
      replicas: 1
      revisionHistoryLimit: 2
      selector:
        matchLabels:
          app: prometheus-webhook-dingtalk
      minReadySeconds: 0
      strategy:
        type: RollingUpdate
        rollingUpdate:
          maxUnavailable: 1
          maxSurge: 1
      template:
        metadata:
          labels:
            app: prometheus-webhook-dingtalk
        spec:
          affinity:         #亲和性的调度设置
            nodeAffinity:   #策略为节点亲和性
              requiredDuringSchedulingIgnoredDuringExecution:      #亲和性的硬策略
                nodeSelectorTerms:   #这里不再使用nodeselector，使用这个参数可以进行相对简单的逻辑运算
                - matchExpressions:     #匹配表达式
                  - key: kubernetes.io/hostname      #具体匹配规则(可以通过kubectl get node --show-labels找到相应规则)
                    operator: In  #不在，简单的来说就是不在k8s-04节点
                    values:
                    - prod-k8s-n006
    
          containers:
          - name: prometheus-webhook-dingtalk
            image: registry.cn-shanghai.aliyuncs.com/yjk-datag/prometheus-webhook-dingtalk:v3
            args:
            # - "--web.enable-ui" # 这个web ui貌似在这里添加不生效。我也用不上 可以在构建镜像的时候加上试试
            - "----config.file=/etc/prometheus-webhook-dingtalk/config.yml"
            resources:
              limits:
                memory: "256Mi"
                cpu: "100m"
            readinessProbe:
              tcpSocket:
                port: 8060
              initialDelaySeconds: 30
              periodSeconds: 10
            livenessProbe:
              tcpSocket:
                port: 8060
              initialDelaySeconds: 60
              periodSeconds: 10
            ports:
            - containerPort: 8060
              protocol: TCP
              name: http
    
            volumeMounts:
            - name: config-volume
              mountPath: "/etc/prometheus-webhook-dingtalk/config.yml"
              subPath: config.yml
          volumes:
          - configMap:
              name: prometheus-webhook-dingtalk-config
            name: config-volume

##### 创建prometheus-webhook-dingtalk-svc.yaml #####

---
    apiVersion: v1
    kind: Service
    metadata:
      name: prometheus-webhook-dingtalk
      namespace: kube-ops
      labels:
        app: prometheus-webhook-dingtalk
    spec:
      selector:
        app: prometheus-webhook-dingtalk
      type: NodePort
      ports:
      - port: 80
        protocol: TCP
        targetPort: 8060

创建prometheus-webhook-dingtalk-ingress.yaml

---
    apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
    # 通过添加下面的annotations 来开启白名单
    # 关闭80强制跳转443 为ingress配置增加注解（annotations）：nginx.ingress.kubernetes.io/ssl-redirect: 'false' 就可以禁止http强制跳转至https
      annotations:
        #nginx.ingress.kubernetes.io/whitelist-source-range: "60.191.70.64/29, xx.xxx.0.0/16"
        nginx.ingress.kubernetes.io/ssl-redirect: 'false'
      name: prod-prometheus-webhook-dingtalk
      namespace: kube-ops
    spec:
      rules:
        - host: prometheus-webhook-dingtalk.ihaozhuo.com
          http:
            paths:
            - path: /
              backend:
                serviceName: prometheus-webhook-dingtalk
                servicePort: 80

运行prometheus-webhook-dingtalk

kubectl apply -f ./

![6552433a1ff01f972226d34c643092ab.png][]

##### 测试告警 #####

![b722874ee59e4bf6e209e41b993cc5e2.png][]

![3bb97ed3785a3892dd6fbfc466d325c4.png][]

![30c722243b3fb0599213ff171a861e47.png][]

##### 修改prometheus-webhook-dingtalk的svc为headless #####

好处是可以不用域名访问 可以使用prometheus-webhook-dingtalk-headless.kube-ops.svc.cluster.local:8060 来请求

创建prometheus-webhook-dingtalk-headless-svc.yaml

---
    apiVersion: v1
    kind: Service
    metadata:
      name: prometheus-webhook-dingtalk-headless
      namespace: kube-ops
      labels:
        app: prometheus-webhook-dingtalk
    spec:
      ports:
      - port: 80
        name: http
      # clusterIP 设置为 None
      clusterIP: None
      selector:
        app: prometheus-webhook-dingtalk

kubectl apply -f ./

应用之后 通过其他容器测试

![7213b2c8a71d487c4997cd718777985c.png][]

[1a842c1bbf776307701f2a535a69965d.png]: /images/20221022/e24f6cd35c0c4070a2c5a5fbed8266f0.png
[6552433a1ff01f972226d34c643092ab.png]: /images/20221022/ad9f6088b64049f7be8fd2acb67526f5.png
[b722874ee59e4bf6e209e41b993cc5e2.png]: /images/20221022/fbd3b63d194b4d56bfe09979fef20467.png
[3bb97ed3785a3892dd6fbfc466d325c4.png]: /images/20221022/37f83d7652e145e4a9ff42c285bbfa29.png
[30c722243b3fb0599213ff171a861e47.png]: /images/20221022/99c7c8c7aa7a410ebc2d2e606eb97306.png
[7213b2c8a71d487c4997cd718777985c.png]: /images/20221022/65a9271215d0418ab2c8f3dce7b36bd9.png