K8s之三master的安装-yellowcong

亦凉 2022-05-13 12:12 403阅读 0赞

> 安装etcd的时候，一定要注意的时防火墙关闭，其次如果时使用云服务器的时候，安全组，一定要开了，我就发生了这个惨案。

## 架构 ##

<table> 
 <thead> 
  <tr> 
   <th>ip</th> 
   <th>主机名</th> 
   <th>服务</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>172.21.16.17</td> 
   <td>ba-k8s-master-node3</td> 
   <td>主节点，etcd</td> 
  </tr> 
  <tr> 
   <td>172.21.16.9</td> 
   <td>ba-k8s-master-node1</td> 
   <td>主节点，etcd</td> 
  </tr> 
  <tr> 
   <td>172.21.16.3</td> 
   <td>ba-k8s-master-node2</td> 
   <td>主节点，etcd</td> 
  </tr> 
  <tr> 
   <td>172.21.16.6</td> 
   <td>haproxy（或keepalived）</td> 
   <td>主节点，etcd</td> 
  </tr> 
 </tbody> 
</table>

## 1.etcd 安装 ##

### 1.1 安装cfssl ###

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
    wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
    wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
    
    #添加执行权限
    chmod a+x cfssl_linux-amd64
    chmod a+x cfssljson_linux-amd64
    chmod a+x cfssl-certinfo_linux-amd64
    
    #拷贝到/usr/bin
    mv cfssl_linux-amd64 /usr/bin/cfssl
    mv cfssljson_linux-amd64 /usr/bin/cfssljson
    mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

![这里写图片描述][70]

### 安装脚本 ###

#创建目录
    mkdir  -p  /opt/k8s/ssl  &&  cd  /opt/k8s/ssl

### 1.3 生成证书 ###

#### 1.3.1 ####

cat >  ca-config.json <<EOF
    {
    "signing": {
    "default": {
      "expiry": "8760h"
    },
    "profiles": {
      "kubernetes-Soulmate": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "8760h"
      }
    }
    }
    }
    EOF

#### 1.3.2ca-csr.json ####

cat >  ca-csr.json <<EOF
    {
    "CN": "kubernetes-Soulmate",
    "key": {
    "algo": "rsa",
    "size": 2048
    },
    "names": [
    {
      "C": "CN",
      "ST": "shanghai",
      "L": "shanghai",
      "O": "k8s",
      "OU": "System"
    }
    ]
    }
    EOF

#完成后会在当前目录下看到生成的签名的CA证书ca.pem和对应的密钥证书ca-key.pem。
    cfssl gencert -initca ca-csr.json | cfssljson -bare ca

![这里写图片描述][70 1]

#### 1.3.3配置etcd的证书请求文件 ####

这个地方，需要将ip替换为master节点的列表

cat > etcd-csr.json <<EOF
    {
      "CN": "etcd",
      "hosts": [
        "127.0.0.1",
        "172.21.16.17",
        "172.21.16.9",
        "172.21.16.3"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "shanghai",
          "L": "shanghai",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }
    EOF

### 生成etcd证书 ###

指定证书，以及刚刚的配置文件。

cfssl gencert -ca=ca.pem \
    -ca-key=ca-key.pem \
    -config=ca-config.json \
    -profile=kubernetes-Soulmate etcd-csr.json | cfssljson -bare etcd
    
    
    #创建etcd的目录
    mkdir -p /etc/etcd/ssl
    
    #拷贝到etcd的目录
    cp etcd.pem etcd-key.pem ca.pem /etc/etcd/ssl/

![这里写图片描述][70 2]

拷贝etcd的证书到etcd目录  
![这里写图片描述][70 3]

### 拷贝到其他地址 ###

#创建文件夹
    ssh ba-k8s-master-node2 "mkdir -p /etc/etcd/ssl/"
    ssh ba-k8s-master-node3 "mkdir -p /etc/etcd/ssl/"
    
    #拷贝到某个节点上
    scp /etc/etcd/ssl/* root@ba-k8s-master-node2:/etc/etcd/ssl/
    scp /etc/etcd/ssl/* root@ba-k8s-master-node3:/etc/etcd/ssl/
    
    #给node2 和node3安装etcd
    ssh ba-k8s-master-node2 "yum install etcd -y"
    ssh ba-k8s-master-node3 "yum install etcd -y"
    
    
    #创建文件目录
    ssh ba-k8s-master-node2 "mkdir -p /var/lib/etcd"
    ssh ba-k8s-master-node3 "mkdir -p /var/lib/etcd"

## 安装etcd ##

yum install etcd -y

架构

<table> 
 <thead> 
  <tr> 
   <th>etcd名称</th> 
   <th>ip</th> 
   <th>主机名</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>172.21.16.9-name</td> 
   <td>172.21.16.9</td> 
   <td>ba-k8s-master-node1</td> 
  </tr> 
  <tr> 
   <td>172.21.16.3-name</td> 
   <td>172.21.16.3</td> 
   <td>ba-k8s-master-node2</td> 
  </tr> 
  <tr> 
   <td>172.21.16.17-name</td> 
   <td>172.21.16.17</td> 
   <td>ba-k8s-master-node3</td> 
  </tr> 
 </tbody> 
</table>

### 172.21.16.9 etcd service配置 ###

cat <<EOF >/etc/systemd/system/etcd.service
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    Documentation=https://github.com/coreos
    
    [Service]
    Type=notify
    WorkingDirectory=/var/lib/etcd/
    ExecStart=/usr/bin/etcd \
      --name 172.21.16.9-name \
      --cert-file=/etc/etcd/ssl/etcd.pem \
      --key-file=/etc/etcd/ssl/etcd-key.pem \
      --peer-cert-file=/etc/etcd/ssl/etcd.pem \
      --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
      --trusted-ca-file=/etc/etcd/ssl/ca.pem \
      --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
      --initial-advertise-peer-urls https://172.21.16.9:2380 \
      --listen-peer-urls https://172.21.16.9:2380 \
      --listen-client-urls https://172.21.16.9:2379,http://127.0.0.1:2379 \
      --advertise-client-urls https://172.21.16.9:2379 \
      --initial-cluster-token etcd-cluster-0 \
      --initial-cluster 172.21.16.9-name=https://172.21.16.9:2380,172.21.16.17-name=https://172.21.16.17:2380,172.21.16.3-name=https://172.21.16.3:2380 \
      --initial-cluster-state new \
      --peer-client-cert-auth=true \
      --client-cert-auth=true \
      --data-dir=/data/etcd_data
    Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target
    EOF

\#\#\#172.21.16.3（172.21.16.3-name）etcd service配置

cat <<EOF >/etc/systemd/system/etcd.service
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    Documentation=https://github.com/coreos
    
    [Service]
    Type=notify
    WorkingDirectory=/var/lib/etcd/
    ExecStart=/usr/bin/etcd \
      --name 172.21.16.3-name \
      --cert-file=/etc/etcd/ssl/etcd.pem \
      --key-file=/etc/etcd/ssl/etcd-key.pem \
      --peer-cert-file=/etc/etcd/ssl/etcd.pem \
      --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
      --trusted-ca-file=/etc/etcd/ssl/ca.pem \
      --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
      --initial-advertise-peer-urls https://172.21.16.3:2380 \
      --listen-peer-urls https://172.21.16.3:2380 \
      --listen-client-urls https://172.21.16.3:2379,http://127.0.0.1:2379 \
      --advertise-client-urls https://172.21.16.3:2379 \
      --initial-cluster-token etcd-cluster-0 \
      --initial-cluster 172.21.16.9-name=https://172.21.16.9:2380,172.21.16.17-name=https://172.21.16.17:2380,172.21.16.3-name=https://172.21.16.3:2380 \
      --initial-cluster-state new \
      --peer-client-cert-auth=true \
      --client-cert-auth=true \
      --data-dir=/data/etcd_data
    Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target
    EOF

\#\#\#172.21.16.17 （172.21.16.17-name）etcd service配置

cat <<EOF >/etc/systemd/system/etcd.service
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    Documentation=https://github.com/coreos
    
    [Service]
    Type=notify
    WorkingDirectory=/var/lib/etcd/
    ExecStart=/usr/bin/etcd \
      --name 172.21.16.17-name \
      --cert-file=/etc/etcd/ssl/etcd.pem \
      --key-file=/etc/etcd/ssl/etcd-key.pem \
      --peer-cert-file=/etc/etcd/ssl/etcd.pem \
      --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
      --trusted-ca-file=/etc/etcd/ssl/ca.pem \
      --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
      --initial-advertise-peer-urls https://172.21.16.17:2380 \
      --listen-peer-urls https://172.21.16.17:2380 \
      --listen-client-urls https://172.21.16.17:2379,http://127.0.0.1:2379 \
      --advertise-client-urls https://172.21.16.17:2379 \
      --initial-cluster-token etcd-cluster-0 \
      --initial-cluster 172.21.16.9-name=https://172.21.16.9:2380,172.21.16.17-name=https://172.21.16.17:2380,172.21.16.3-name=https://172.21.16.3:2380 \
      --initial-cluster-state new \
      --peer-client-cert-auth=true \
      --client-cert-auth=true \
      --data-dir=/data/etcd_data
    Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target
    EOF

### 添加自启动 ###

#覆盖原始配置
    mv /etc/systemd/system/etcd.service /usr/lib/systemd/system/
    
    #重载配置
    systemctl daemon-reload 
    
    #重启，并设定为开机启动
    systemctl start etcd && systemctl enable etcd
    systemctl status etcd

![这里写图片描述][70 4]

### 健康检查 ###

etcdctl --endpoints=https://172.21.16.9:2379,https://172.21.16.17:2379,https://172.21.16.3:2379 \
      --ca-file=/etc/etcd/ssl/ca.pem \
      --cert-file=/etc/etcd/ssl/etcd.pem \
      --key-file=/etc/etcd/ssl/etcd-key.pem  cluster-health

![这里写图片描述][70 5]

### 部署失败 ###

#停止服务
    systemctl stop etcd
    
    #移除文件配置
    rm -rf /data/data_etcd
    
    #冲洗配置服务，然后启动
    systemctl start etcd

## 安装keepalive ##

yum install -y keepalived && systemctl enable keepalived

\#\#安装keepalive  
安装keepalived的时候，一定要确认注意不是阿里云 ，腾讯云的虚拟主机，这两个不适合安装，需要单独购买ip，这个还不如直接购买阿里云提供的lsb服务  
参考文章： https://blog.csdn.net/yelllowcong/article/details/82693084

如果安装不了keepalvied ，我们可以使用 nginx和haproxy替代

### haproxy 配置 ###

配置的这个是基于tcp代理的，只需要修改ip和端口号即可

#/etc/haproxy/haproxy.cfg
    global
      log 127.0.0.1 local0 err
      maxconn 50000
      uid 99
      gid 99
      #daemon
      nbproc 1
      pidfile haproxy.pid
    
    defaults
      mode http
      log 127.0.0.1 local0 err
      maxconn 50000
      retries 3
      timeout connect 5s
      timeout client 30s
      timeout server 30s
      timeout check 2s
    
    #统计页面配置
    listen admin_stats
      #采用7层代理
      mode http
      #绑定端口
      bind 0.0.0.0:8080
      #错误日志记录  
      log 127.0.0.1 local0 err
      stats refresh 30s
      #状态的uri的前缀
      stats uri     /haproxy-status
      #统计页面密码框上提示文本  
      stats realm   Haproxy\ Statistics
      #用户名和密码
      #设置监控页面的用户和密码:admin,可以设置多个用户名  
      stats auth    yellowcong:yellowcong
      #隐藏版本信息
      stats hide-version
      #设置手工启动/禁用
      stats admin if TRUE
    
    #前端配置
    frontend k8s-https
      bind *:6443 
      mode tcp
      #maxconn 50000
      default_backend k8s-https
      
    #后端配置
    backend k8s-https
      mode tcp
      #balance roundrobin
      stick-table type ip size 200k expire 30m
      stick on src
      #option ssl-hello-chk
      server lab1 172.21.16.17:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
      server lab2 172.21.16.9:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
      server lab3 172.21.16.3:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3

![在这里插入图片描述][70 6]

## Docker安装 ##

不同的k8s版本，对于docker的版本是有要求的，这点需要注意了。如果已经安装过了的，需要删除docker,同时还要清除docker的本地仓库。

更多请参考 https://blog.csdn.net/yelllowcong/article/details/80599256

yum autoremove docker-ce -y
    
    #清空之前安装的docker镜像
    rm -rf /var/lib/docker
    
    #安装相关rpm
    yum install https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm 
    
    #安装服务
    yum install docker-ce-17.03.2.ce-1.el7.centos -y
    
    #重启docker, 查看docker版本
    systemctl restart docker && systemctl enable docker
    docker -v

![在这里插入图片描述][70 7]

## K8s安装 ##

安装步骤，具体可以看 https://blog.csdn.net/yelllowcong/article/details/80715398

### 1.配置config.yaml ###

参数说明

<table> 
 <thead> 
  <tr> 
   <th>参数</th> 
   <th>意义</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>etcd.endpoints</td> 
   <td>etcd的列表</td> 
  </tr> 
  <tr> 
   <td>etcd.endpoints.caFile</td> 
   <td></td> 
  </tr> 
  <tr> 
   <td>etcd.endpoints.certFile</td> 
   <td></td> 
  </tr> 
  <tr> 
   <td>etcd.endpoints.keyFile</td> 
   <td></td> 
  </tr> 
 </tbody> 
</table>

配置文件

#创建文件夹
    mkdir -p /opt/k8s/
    
    #添加配置我文件
    cat <<EOF > /opt/k8s/config.yaml
    apiVersion: kubeadm.k8s.io/v1alpha1
    kind: MasterConfiguration
    etcd:
      endpoints:
      - https://172.21.16.9:2379
      - https://172.21.16.17:2379
      - https://172.21.16.3:2379
      caFile: /etc/etcd/ssl/ca.pem
      certFile: /etc/etcd/ssl/etcd.pem
      keyFile: /etc/etcd/ssl/etcd-key.pem
      dataDir: /var/lib/etcd
    networking:
      podSubnet: 10.244.0.0/16
    kubernetesVersion: 1.10.4
    api:
      advertiseAddress: "172.21.16.6"
    token: "b99a00.a144ef80536d4344"
    tokenTTL: "0s"
    apiServerCertSANs:
    - ba-k8s-master-node1
    - ba-k8s-master-node2
    - ba-k8s-master-node3
    - 172.21.16.9
    - 172.21.16.17
    - 172.21.16.3
    - 172.21.16.6
    featureGates:
      CoreDNS: true
    imageRepository: "k8s.gcr.io"
    imagePullPolicy: IfNotPresent
    featureGates:
      CoreDNS: true
    EOF

![在这里插入图片描述][70 8]

### 配置cni ###

#配置cni
    cat >  /etc/cni/net.d/10-flannel.conf <<EOF
    {
      "name": "cb0",
      "type": "flannel",
      "delegate": {
        "isDefaultGateway": true
      }
    }
    EOF

### 安装集群 ###

k8s 10.4版本，最大支持docker 17.3

#k8s 10.4版本，最大支持docker 17.3
    #可以看出，service默认网段是10.96.0.0/12
    kubeadm init --help
    
    /etc/systemd/system/kubelet.service.d/10-kubeadm.conf默认dns地址cluster-dns=10.96.0.10
    
    #初始化服务器
    kubeadm init  --config /opt/k8s/config.yaml
    
    #查看kubelet情况
    systemctl status kubelet.service
    
    #拷贝
    #创建文件夹
    mkdir -p $HOME/.kube
    
    #拷贝文件
    cp /etc/kubernetes/admin.conf $HOME/.kube/config

![这里写图片描述][70 9]

初始化好了后  
![这里写图片描述][70 10]

### 安装第二，三个节点 ###

一定要拷贝证书和基础配置。都需要完成基础的docker设定，k8s设定。

scp -r pki root@ba-k8s-master-node2:/data/
    scp -r /opt/k8s/config.yaml root@ba-k8s-master-node2:/data/
    
    #拷贝到k8s里面,由于是https的，如果不配置这个，就会导致访问不了k8s的servicer的服务。
    cp -r /data/pki /etc/kubernetes/
    cp /data/config.yaml /opt/k8s/
    
    #配置cni
    mkdir -p  /etc/cni/net.d/
    cat >  /etc/cni/net.d/10-flannel.conf <<EOF
    {
      "name": "cb0",
      "type": "flannel",
      "delegate": {
        "isDefaultGateway": true
      }
    }
    EOF
    
    #初始化集群
    kubeadm init  --config /opt/k8s/config.yaml

\#\#\#配置config

mkdir -p $HOME/.kube
      sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
      sudo chown $(id -u):$(id -g) $HOME/.kube/config

可以看到三节点，安装成功  
![在这里插入图片描述][70 11]

### 删除k8s服务 ###

当我们部署错误了，可以通过这个清空，然后重新部署服务。

kubeadm reset
    rm -rf /var/lib/cni/
    rm -f $HOME/.kube/config

## 常见问题 ##

### 1.docker version is greater than the most recently validated version ###

[WARNING SystemVerification]: docker version is greater than the most recently validated version. Docker version: 18.06.1-ce. Max validated version: 17.03
    [WARNING FileExisting-crictl]: crictl not found in system path

![这里写图片描述][70 12]

导致这个问题的原因是，docker的版本太高了。

#安装指定docker
    yum list docker-ce --showduplicates | sort -r
    
    #安装docker
    yum install -y docker-ce-17.03.3.ce-1.el7

查看列表  
![这里写图片描述][70 13]

### 2.Error: Package: docker-ce-17.03.3.ce-1.el7.x86\_64 (docker-ce-stable) ###

docker安装失败,由于没有找到相关依赖

yum autoremove docker-ce -y
    
    #清空之前安装的docker版本
    rm -rf /var/lib/docker
    
    
    #安装相关rpm
    yum install https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm 
    
    #安装服务
    yum install docker-ce-17.03.2.ce-1.el7.centos

![这里写图片描述][70 14]

### 3 \[graphdriver\] prior storage driver overlay2 failed: driver not supported ###

导致这个问题，是由于之前版本的docker数据没有删除，导致的，解决办法，`rm -rf /var/lib/docker`

[graphdriver] prior storage driver overlay2 failed: driver not supported

![这里写图片描述][70 15]

### 4 network plugin is not ready: cni config uninitialized ###

Sep 16 18:20:19 ba-k8s-master-node1 kubelet[2456]: E0916 18:20:19.439385    2456 certificate_manager.go:299] Failed while requesting a signed certificate from the master: cannot create certificate signing request: Post https://172.21.16.15:6443/apis/certificates.k8s.io/v1beta1/certificatesigningrequests: dial tcp 172.21.16.15:6443: i/o timeout
    Sep 16 18:20:21 ba-k8s-master-node1 kubelet[2456]: W0916 18:20:21.573568    2456 cni.go:171] Unable to update cni config: No networks found in /etc/cni/net.d
    Sep 16 18:20:21 ba-k8s-master-node1 kubelet[2456]: E0916 18:20:21.573702    2456 kubelet.go:2130] Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
    Sep 16 18:20:22 ba-k8s-master-node1 kubelet[2456]: I0916 18:20:22.444939    2456 kubelet_node_status.go:271] Setting node annotation to enable volume controller attach/detach
    Sep 16 18:20:22 ba-k8s-master-node1 kubelet[2456]: I0916 18:20:22.444947    2456 kubelet_node_status.go:271] Setting node annotation to enable volume controller attach/detach
    Sep 16 18:20:23 ba-k8s-master-node1 kubelet[2456]: I0916 18:20:23.444976    2456 kubelet_node_status.go:271] Setting node annotation to enable volume controller attach/detach

导致这个问题的原因是cni没有安装上导致的

cat >  /etc/cni/net.d/10-flannel.conf <<EOF
    {
      "name": "cb0",
      "type": "flannel",
      "delegate": {
        "isDefaultGateway": true
      }
    }
    EOF

![这里写图片描述][70 16]

## 参考文章 ##

https://blog.csdn.net/qq\_24513043/article/details/82459443

https://www.cnblogs.com/ericnie/p/7694592.html  
https://blog.csdn.net/github\_35614077/article/details/81673890  
http://blog.51cto.com/irow10/2055064  
http://www.zyizou.com/archives/599

https://www.hi-linux.com/posts/49138.html

[70]: /images/20220513/1da8967baee247dbac0fa2a37fe37ae7.png
[70 1]: /images/20220513/3ce668bc272e4b73892202b8eda7d61d.png
[70 2]: /images/20220513/941eb317ad7b4a4fb1f8ebf943a7596c.png
[70 3]: /images/20220513/81a5008ea89644c7b15d7911c238fed1.png
[70 4]: /images/20220513/1d2e1a9f04e24e18892fcaf7aee28882.png
[70 5]: /images/20220513/37c9a67760574d03bb24481c0f3cf076.png
[70 6]: /images/20220513/ba9d22aa5e6940f2a61688296aa61441.png
[70 7]: /images/20220513/e63aa574a2034d6d847714f160b62484.png
[70 8]: /images/20220513/d89ddc1e828740a798574ea2008b0e53.png
[70 9]: /images/20220513/2c030be61a5f458cba59fee4ec93bf37.png
[70 10]: /images/20220513/1fee6355597245399a3e0def3f3ae0c5.png
[70 11]: /images/20220513/3e36b6e225b744c4a60c855d487c3fab.png
[70 12]: /images/20220513/512173ee008f4fdba11ae0a7b8c06b0a.png
[70 13]: /images/20220513/c247084d9cf544bb881502921e9e2739.png
[70 14]: /images/20220513/0bb4fc69a01a4112b14887de0093869c.png
[70 15]: /images/20220513/c25570d4d6e949e0a2de0a725377d8a1.png
[70 16]: /images/20220513/3f4ed6bfd625410f839a983a52c88f75.png