基于二进制安装 kubernetes 1.9.6

偏执的太偏执、 2022-05-25 09:46 207阅读 0赞

**环境说明**

系统

[root@k8s-master-47 ~]# cat /etc/redhat-release
    CentOS Linux release 7.5.1804 (Core)

节点

[root@k8s-master-49 ~]# cat /etc/hosts
    127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
    ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
    192.168.1.47 k8s-master-47
    192.168.1.48 k8s-master-48
    192.168.1.49 k8s-master-49
    192.168.1.50 k8s-master-50

**创建 验证**  
这里使用 CloudFlare 的 PKI 工具集 cfssl 来生成 Certificate Authority (CA) 证书和秘钥文件。

安装 cfssl

mkdir -p /opt/local/cfssl
    
    cd /opt/local/cfssl
    
    wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
    mv cfssl_linux-amd64 cfssl
    
    wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
    mv cfssljson_linux-amd64 cfssljson
    
    wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
    mv cfssl-certinfo_linux-amd64 cfssl-certinfo
    
    chmod +x *
    
    [root@k8s-master-47 cfssl]# ls -l 总用量 18808 -rwxr-xr-x 1 root root 10376657 3月 30 2016 cfssl -rwxr-xr-x 1 root root 6595195 3月 30 2016 cfssl-certinfo -rwxr-xr-x 1 root root 2277873 3月 30 2016 cfssljson

创建 CA 证书配置

mkdir /opt/ssl
    
    cd /opt/ssl

# config.json 文件
    
    vi  config.json
    
    
    {
      "signing": {
        "default": {
          "expiry": "87600h"
        },
        "profiles": {
          "kubernetes": {
            "usages": [
                "signing",
                "key encipherment",
                "server auth",
                "client auth"
            ],
            "expiry": "87600h"
          }
        }
      }
    }

# csr.json 文件
    
    vi csr.json
    
    {
      "CN": "kubernetes",
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

生成 CA 证书和私钥

cd /opt/ssl/
    
    /opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca
    
    
    [root@k8s-master-47 ssl]# /opt/local/cfssl/cfssl gencert -initca csr.json | /opt/local/cfssl/cfssljson -bare ca
    2018/05/12 10:01:27 [INFO] generating a new CA key and certificate from CSR
    2018/05/12 10:01:27 [INFO] generate received request
    2018/05/12 10:01:27 [INFO] received CSR
    2018/05/12 10:01:27 [INFO] generating key: rsa-2048
    2018/05/12 10:01:27 [INFO] encoded CSR
    2018/05/12 10:01:27 [INFO] signed certificate with serial number 566945822389148790180083993246699565817633919023
    [root@k8s-master-47 ssl]# ls -l
    总用量 20
    -rw-r--r-- 1 root root 1005 5月 12 10:01 ca.csr
    -rw------- 1 root root 1679 5月 12 10:01 ca-key.pem
    -rw-r--r-- 1 root root 1363 5月 12 10:01 ca.pem
    -rw-r--r-- 1 root root 292 5月 12 09:57 config.json
    -rw-r--r-- 1 root root 210 5月 12 09:58 csr.json

分发证书

# 创建证书目录
    mkdir -p /etc/kubernetes/ssl
    
    # 拷贝所有文件到目录下
    cp *.pem /etc/kubernetes/ssl
    cp ca.csr /etc/kubernetes/ssl
    
    # 这里要将文件拷贝到所有的k8s 机器上
    
    
    scp * 192.168.1.48:/etc/kubernetes/ssl/
    
    scp * 192.168.1.49:/etc/kubernetes/ssl/
    scp * 1192.168.1.50:/etc/kubernetes/ssl/

**安装 docker**

> 所有服务器预先安装 docker-ce ，官方1.9 中提示， 目前 k8s 支持最高 Docker versions 1.11.2, 1.12.6, 1.13.1, and 17.03.1

yum install docker -y

启动docker

systemctl enable docker
    systemctl start docker systemctl status docker

# 如果报错 请使用
    journalctl -f -t docker  和 journalctl -u docker 来定位问题

**etcd 集群**

> 官方地址 [https://github.com/coreos/etcd/releases][https_github.com_coreos_etcd_releases]

安装etcd

yum install etcd3 -y

创建 etcd 证书

> etcd 证书这里，默认配置三个，后续如果需要增加，更多的 etcd 节点 这里的认证IP 请多预留几个，以备后续添加能通过认证，不需要重新签发

cd /opt/ssl/
    
    vi etcd-csr.json
    
    {
      "CN": "etcd",
      "hosts": [
        "127.0.0.1",
        "192.168.1.47",
        "192.168.1.48",
        "192.168.1.49"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

# 生成 etcd   密钥
     /opt/local/cfssl/cfssl gencert -ca=/opt/ssl/ca.pem \
      -ca-key=/opt/ssl/ca-key.pem \   -config=/opt/ssl/config.json \   -profile=kubernetes etcd-csr.json | /opt/local/cfssl/cfssljson -bare etcd

# 查看生成
    
    [root@k8s-master-47 ssl]# ls etcd*
    etcd.csr  etcd-csr.json  etcd-key.pem  etcd.pem
    
    
    
    # 拷贝到etcd服务器
    
    # etcd-1 192.168.1.47
    cp etcd*.pem /etc/kubernetes/ssl/
    
    # etcd-2
    scp etcd*.pem 192.168.1.48:/etc/kubernetes/ssl/
    
    # etcd-3
    scp etcd*.pem 192.168.1.49:/etc/kubernetes/ssl/
    
    
    
    # 如果 etcd 非 root 用户，读取证书会提示没权限
    
    chmod 644 /etc/kubernetes/ssl/etcd-key.pem

修改 etcd 配置

> 由于 etcd 是最重要的组件，所以 –data-dir 请配置到其他路径中

# 创建 etcd data 目录， 并授权
    
    useradd etcd
    
    mkdir -p /opt/etcd
    
    chown -R etcd:etcd /opt/etcd

# etcd-1
    
    
    vi /etc/systemd/system/etcd.service
    
    
    
    
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    
    [Service]
    Type=notify
    WorkingDirectory=/opt/etcd/
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/usr/bin/etcd \
      --name=etcd1 \   --cert-file=/etc/kubernetes/ssl/etcd.pem \   --key-file=/etc/kubernetes/ssl/etcd-key.pem \   --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \   --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \   --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \   --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \   --initial-advertise-peer-urls=https://192.168.1.47:2380 \   --listen-peer-urls=https://192.168.1.47:2380 \   --listen-client-urls=https://192.168.1.47:2379,http://127.0.0.1:2379 \   --advertise-client-urls=https://192.168.1.47:2379 \   --initial-cluster-token=k8s-etcd-cluster \   --initial-cluster=etcd1=https://192.168.1.47:2380,etcd2=https://192.168.1.48:2380,etcd3=https://192.168.1.49:2380 \   --initial-cluster-state=new \   --data-dir=/opt/etcd/ Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

# etcd-2
    
    
    vi /etc/systemd/system/etcd.service
    
    
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    
    [Service]
    Type=notify
    WorkingDirectory=/opt/etcd/
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/usr/bin/etcd \
      --name=etcd2 \   --cert-file=/etc/kubernetes/ssl/etcd.pem \   --key-file=/etc/kubernetes/ssl/etcd-key.pem \   --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \   --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \   --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \   --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \   --initial-advertise-peer-urls=https://192.168.1.48:2380 \   --listen-peer-urls=https://192.168.1.48:2380 \   --listen-client-urls=https://192.168.1.48:2379,http://127.0.0.1:2379 \   --advertise-client-urls=https://192.168.1.48:2379 \   --initial-cluster-token=k8s-etcd-cluster \   --initial-cluster=etcd1=https://192.168.1.47:2380,etcd2=https://192.168.1.48:2380,etcd3=https://192.168.1.49:2380 \   --initial-cluster-state=new \   --data-dir=/opt/etcd/ Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

# etcd-3
    
    
    vi /etc/systemd/system/etcd.service
    
    
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target
    
    [Service]
    Type=notify
    WorkingDirectory=/opt/etcd/
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/usr/bin/etcd \
      --name=etcd3 \   --cert-file=/etc/kubernetes/ssl/etcd.pem \   --key-file=/etc/kubernetes/ssl/etcd-key.pem \   --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \   --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \   --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \   --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \   --initial-advertise-peer-urls=https://192.168.1.49:2380 \   --listen-peer-urls=https://192.168.1.49:2380 \   --listen-client-urls=https://192.168.1.49:2379,http://127.0.0.1:2379 \   --advertise-client-urls=https://192.168.1.49:2379 \   --initial-cluster-token=k8s-etcd-cluster \   --initial-cluster=etcd1=https://192.168.1.47:2380,etcd2=https://192.168.1.48:2380,etcd3=https://192.168.1.49:2380 \   --initial-cluster-state=new \   --data-dir=/opt/etcd/ Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

启动 etcd

systemctl daemon-reload
    systemctl enable etcd
    systemctl start etcd systemctl status etcd

# 如果报错 请使用
    journalctl -f -t etcd  和 journalctl -u etcd 来定位问题

**验证 etcd 集群状态**

查看 etcd 集群状态

etcdctl --endpoints=https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379\
            --cert-file=/etc/kubernetes/ssl/etcd.pem \
            --ca-file=/etc/kubernetes/ssl/ca.pem \
            --key-file=/etc/kubernetes/ssl/etcd-key.pem \
            cluster-health
    
    [root@k8s-master-47 ssl]# etcdctl --endpoints=https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379\
    >         --cert-file=/etc/kubernetes/ssl/etcd.pem \
    >         --ca-file=/etc/kubernetes/ssl/ca.pem \
    >         --key-file=/etc/kubernetes/ssl/etcd-key.pem \
    >         cluster-health
    member 26127bd44cbdf882 is healthy: got healthy result from https://192.168.1.49:2379
    member d3051b9a3c6d69af is healthy: got healthy result from https://192.168.1.48:2379
    member e847aa46eb7ec814 is healthy: got healthy result from https://192.168.1.47:2379
    cluster is healthy

查看成员

etcdctl --endpoints=https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379\
            --cert-file=/etc/kubernetes/ssl/etcd.pem \         --ca-file=/etc/kubernetes/ssl/ca.pem \         --key-file=/etc/kubernetes/ssl/etcd-key.pem \         member list
    
    [root@k8s-master-47 ssl]# etcdctl --endpoints=https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379\
    >         --cert-file=/etc/kubernetes/ssl/etcd.pem \
    >         --ca-file=/etc/kubernetes/ssl/ca.pem \
    >         --key-file=/etc/kubernetes/ssl/etcd-key.pem \
    >         member list
    26127bd44cbdf882: name=etcd3 peerURLs=https://192.168.1.49:2380 clientURLs=https://192.168.1.49:2379 isLeader=true
    d3051b9a3c6d69af: name=etcd2 peerURLs=https://192.168.1.48:2380 clientURLs=https://192.168.1.48:2379 isLeader=false
    e847aa46eb7ec814: name=etcd1 peerURLs=https://192.168.1.47:2380 clientURLs=https://192.168.1.47:2379 isLeader=false

## 配置 Kubernetes 集群 ##

**Master and Node**

> Master 需要部署 kube-apiserver , kube-scheduler , kube-controller-manager 这三个组件。 kube-scheduler 作用是调度pods分配到那个node里，简单来说就是资源调度。 kube-controller-manager 作用是 对 deployment controller , replication controller, endpoints controller, namespace controller, and serviceaccounts controller等等的循环控制，与kube-apiserver交互。

**安装组件**

# 从github 上下载版本
    
    cd /tmp
    
    wget https://dl.k8s.io/v1.9.6/kubernetes-server-linux-amd64.tar.gz
    
    tar -xzvf kubernetes-server-linux-amd64.tar.gz
    
    cd kubernetes
    
    cp -r server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler,kubectl,kubelet} /usr/local/bin/
    
    
    scp kubernetes-server-linux-amd64.tar.gz 192.168.1.48:/tmp/
    scp kubernetes-server-linux-amd64.tar.gz 192.168.1.49:/tmp/
    scp kubernetes-server-linux-amd64.tar.gz 192.168.1.50:/tmp/
    #192.168.1.50
    cp server/bin/{kube-proxy,kubelet,kubectl} /usr/local/bin/

**创建 admin 证书**  
kubectl 与 kube-apiserver 的安全端口通信，需要为安全通信提供 TLS 证书和秘钥。

cd /opt/ssl/
    
    vi admin-csr.json
    
    
    {
      "CN": "admin",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "system:masters",
          "OU": "System"
        }
      ]
    }

# 生成 admin 证书和私钥
    cd /opt/ssl/
    
    /opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/opt/ssl/config.json \
      -profile=kubernetes admin-csr.json | /opt/local/cfssl/cfssljson -bare admin
    
    
    # 查看生成
    
    [root@k8s-master-47 ssl]# ls admin*
    admin.csr  admin-csr.json  admin-key.pem  admin.pem
    
    cp admin*.pem /etc/kubernetes/ssl/
    
    scp admin*.pem 192.168.1.48:/etc/kubernetes/ssl/
    scp admin*.pem 192.168.1.49:/etc/kubernetes/ssl/
    scp admin*.pem 192.168.1.50:/etc/kubernetes/ssl/

配置 kubectl kubeconfig 文件

> 生成证书相关的配置文件存储与 /root/.kube 目录中

# 配置 kubernetes 集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 
    
    # 配置 客户端认证
    
    kubectl config set-credentials admin \
      --client-certificate=/etc/kubernetes/ssl/admin.pem \   --embed-certs=true \   --client-key=/etc/kubernetes/ssl/admin-key.pem 
    
    
    kubectl config set-context kubernetes \
      --cluster=kubernetes \   --user=admin 
    
    kubectl config use-context kubernetes
    mkdir ~/.kube
    scp ~/.kube/* root@192.168.1.48:~/.kube/
    scp ~/.kube/* root@192.168.1.49:~/.kube/
    scp ~/.kube/* root@192.168.1.50:~/.kube/

创建 kubernetes 证书

cd /opt/ssl
    
    vi kubernetes-csr.json
    
    {
      "CN": "kubernetes",
      "hosts": [
        "127.0.0.1",
        "192.168.1.49",
        "192.168.1.47",
        "192.168.1.48",
        "10.254.0.1",
        "kubernetes",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local"
      ],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }
    
    
    ## 这里 hosts 字段中 三个 IP 分别为 127.0.0.1 本机， 192.168.1.49 和 192.168.1.48,192.168.1.47 为 Master 的IP，多个Master需要写多个。 10.254.0.1 为 kubernetes SVC 的 IP， 一般是 部署网络的第一个IP , 如: 10.254.0.1 ， 在启动完成后，我们使用 kubectl get svc ， 就可以查看到

**生成 kubernetes 证书和私钥**

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \   -config=/opt/ssl/config.json \   -profile=kubernetes kubernetes-csr.json | /opt/local/cfssl/cfssljson -bare kubernetes 
    # 查看生成
    
    [root@k8s-master-47 ssl]# ls -l kubernetes*
    -rw-r--r-- 1 root root 1261 5月 12 10:56 kubernetes.csr -rw-r--r-- 1 root root 479 5月 12 10:54 kubernetes-csr.json -rw------- 1 root root 1679 5月 12 10:56 kubernetes-key.pem -rw-r--r-- 1 root root 1635 5月 12 10:56 kubernetes.pem 
    
    # 拷贝到目录
    cp kubernetes*.pem /etc/kubernetes/ssl/
    
    scp kubernetes*.pem 192.168.1.48:/etc/kubernetes/ssl/
    scp kubernetes*.pem 192.168.1.49:/etc/kubernetes/ssl/
    scp kubernetes*.pem 192.168.1.50:/etc/kubernetes/ssl/

配置 kube-apiserver

> kubelet 首次启动时向 kube-apiserver 发送 TLS Bootstrapping 请求，kube-apiserver 验证 kubelet 请求中的 token 是否与它配置的 token 一致，如果一致则自动为 kubelet生成证书和秘钥。

# 生成 token
    
    [root@k8s-master-47 ssl]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
    f5c4ec31a1812819fc04bf060954357e
    
    
    # 创建 token.csv 文件
    
    cd /opt/ssl
    
    vi token.csv
    
    f5c4ec31a1812819fc04bf060954357e,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
    
    
    # 拷贝
    
    cp token.csv /etc/kubernetes/
    
    scp token.csv root@192.168.1.48:/etc/kubernetes/
    scp token.csv root@192.168.1.49:/etc/kubernetes/

# 生成高级审核配置文件
    
    cd /etc/kubernetes
    
    
    cat >> audit-policy.yaml <<EOF
    # Log all requests at the Metadata level.
    apiVersion: audit.k8s.io/v1beta1
    kind: Policy
    rules:
    - level: Metadata
    EOF
    
    
    
    
    # 拷贝
    
    scp audit-policy.yaml root@192.168.1.48:/etc/kubernetes/
    scp audit-policy.yaml root@192.168.1.49:/etc/kubernetes/

创建 kube-apiserver.service 文件

# 自定义 系统 service 文件一般存于 /etc/systemd/system/ 下
    # 配置为 各自的本地 IP
    
    vi /etc/systemd/system/kube-apiserver.service
    
    [Unit]
    Description=Kubernetes API Server
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=network.target
    
    [Service]
    User=root
    ExecStart=/usr/local/bin/kube-apiserver \
      --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \   --advertise-address=192.168.1.47 \   --allow-privileged=true \   --apiserver-count=3 \   --audit-policy-file=/etc/kubernetes/audit-policy.yaml \   --audit-log-maxage=30 \   --audit-log-maxbackup=3 \   --audit-log-maxsize=100 \   --audit-log-path=/var/log/kubernetes/audit.log \   --authorization-mode=Node,RBAC \   --bind-address=0.0.0.0 \   --secure-port=6443 \   --client-ca-file=/etc/kubernetes/ssl/ca.pem \   --enable-swagger-ui=true \   --etcd-cafile=/etc/kubernetes/ssl/ca.pem \   --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \   --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \   --etcd-servers=https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379 \   --event-ttl=1h \   --kubelet-https=true \   --insecure-bind-address=127.0.0.1 \   --insecure-port=8080 \   --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \   --service-cluster-ip-range=10.254.0.0/18 \   --service-node-port-range=30000-32000 \   --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \   --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \   --enable-bootstrap-token-auth \   --token-auth-file=/etc/kubernetes/token.csv \   --v=1 Restart=on-failure
    RestartSec=5
    Type=notify
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

# k8s 1.8 开始需要 添加 --authorization-mode=Node
    # k8s 1.8 开始需要 添加 --admission-control=NodeRestriction
    # k8s 1.8 开始需要 添加 --audit-policy-file=/etc/kubernetes/audit-policy.yaml
    
    # 这里面要注意的是 --service-node-port-range=30000-32000
    # 这个地方是 映射外部端口时 的端口范围，随机映射也在这个范围内映射，指定映射端口必须也在这个范围内。

**启动 kube-apiserver**

systemctl daemon-reload
    systemctl enable kube-apiserver
    systemctl start kube-apiserver
    systemctl status kube-apiserver

配置 kube-controller-manager

# 创建 kube-controller-manager.service 文件
    
    vi /etc/systemd/system/kube-controller-manager.service
    
    
    [Unit]
    Description=Kubernetes Controller Manager
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    
    [Service]
    ExecStart=/usr/local/bin/kube-controller-manager \
      --address=0.0.0.0 \   --master=http://127.0.0.1:8080 \   --allocate-node-cidrs=true \   --service-cluster-ip-range=10.254.0.0/18 \   --cluster-cidr=10.254.64.0/18 \   --cluster-name=kubernetes \   --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \   --root-ca-file=/etc/kubernetes/ssl/ca.pem \   --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \   --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \   --leader-elect=true \   --v=1 Restart=on-failure
    RestartSec=5
    
    [Install]
    WantedBy=multi-user.target

启动 kube-controller-manager

systemctl daemon-reload
    systemctl enable kube-controller-manager
    systemctl start kube-controller-manager
    systemctl status kube-controller-manager

配置 kube-scheduler

# 创建 kube-cheduler.service 文件
    
    vi /etc/systemd/system/kube-scheduler.service
    
    
    [Unit]
    Description=Kubernetes Scheduler
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    
    [Service]
    ExecStart=/usr/local/bin/kube-scheduler \
      --address=0.0.0.0 \   --master=http://127.0.0.1:8080 \   --leader-elect=true \   --v=1 Restart=on-failure
    RestartSec=5
    
    [Install]
    WantedBy=multi-user.target

启动 kube-scheduler

systemctl daemon-reload
    systemctl enable kube-scheduler
    systemctl start kube-scheduler
    systemctl status kube-scheduler

**验证 Master 节点**

kubectl get componentstatuses

![k8s][]

**配置 kubelet**

> kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要先将 bootstrap token 文件中的 kubelet-bootstrap 用户赋予 system:node-bootstrapper 角色，然后 kubelet 才有权限创建认证请求(certificatesigningrequests)。

# 先创建认证请求
    # user 为 master 中 token.csv 文件里配置的用户
    # 只需创建一次就可以
    
    kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

创建 kubelet kubeconfig 文件

# 配置集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 \   --kubeconfig=bootstrap.kubeconfig 
    # 配置客户端认证
    
    kubectl config set-credentials kubelet-bootstrap \
      --token=f5c4ec31a1812819fc04bf060954357e \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \   --user=kubelet-bootstrap \   --kubeconfig=bootstrap.kubeconfig 
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
    
    # 拷贝生成的 bootstrap.kubeconfig 文件
    
    mv bootstrap.kubeconfig /etc/kubernetes/
    
    scp bootstrap.kubeconfig root@192.168.1.48:/etc/kubernetes/
    scp bootstrap.kubeconfig root@192.168.1.49:/etc/kubernetes/
    scp bootstrap.kubeconfig root@192.168.1.50:/etc/kubernetes/

创建 kubelet.service 文件

# 创建 kubelet 目录
    
    > 配置为 node 本机 IP
    
    mkdir /var/lib/kubelet
    
    vi /etc/systemd/system/kubelet.service
    
    
    [Unit]
    Description=Kubernetes Kubelet
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=docker.service
    Requires=docker.service
    
    [Service]
    WorkingDirectory=/var/lib/kubelet
    ExecStart=/usr/local/bin/kubelet \
      --cgroup-driver=systemd \   --hostname-override=k8s-master-47 \   --pod-infra-container-image=jicki/pause-amd64:3.0 \ //生产环境最好改为自己的   --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \   --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \   --cert-dir=/etc/kubernetes/ssl \   --cluster_dns=10.254.0.2 \   --cluster_domain=cluster.local \   --hairpin-mode promiscuous-bridge \   --allow-privileged=true \   --fail-swap-on=false \   --serialize-image-pulls=false \   --logtostderr=true \   --max-pods=512 \   --v=1 
    [Install]
    WantedBy=multi-user.target

# 如上配置:
    kubernetes-64    本机hostname
    10.254.0.2       预分配的 dns 地址
    cluster.local.   为 kubernetes 集群的 domain
    jicki/pause-amd64:3.0  这个是 pod 的基础镜像，既 gcr 的 gcr.io/google_containers/pause-amd64:3.0 镜像， 下载下来修改为自己的仓库中的比较快。

启动 kubelet

systemctl daemon-reload
    systemctl enable kubelet
    systemctl start kubelet systemctl status kubelet

# 如果报错 请使用
    journalctl -f -t kubelet  和 journalctl -u kubelet 来定位问题

配置 TLS 认证

root@k8s-master-47 bin]# kubectl get csr
    NAME                                                  AGE      REQUESTOR           CONDITION
    node-csr-4TzK8vJVRpC0rOPYtrmOCIaks8oKjGsEJrgNzO5bQMU   25m       kubelet-bootstrap   Pending
    node-csr-HUpLNUw3a4zbZnun9cFM5U4sOz-R3sd-Qu9r3bBl9zY   21m       kubelet-bootstrap   Pending
    node-csr-VeQYPKbJa_Vqcusy1iFPrXFv22sPcYhnYMF81qF63B0   4m        kubelet-bootstrap   Pending
    node-csr-howo-xX8bQtqjmE6DLuYMg6KTv0nSY-BKa6SOnp0s94   21m       kubelet-bootstrap   Pending
    
    # 增加 认证
    
    kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve

验证 nodes

[root@k8s-master-47 bin]# kubectl get nodes
    NAME            STATUS    ROLES     AGE       VERSION
    k8s-master-47   Ready     <none>    13s       v1.9.6
    k8s-master-48   Ready     <none>    12s       v1.9.6
    k8s-master-49   Ready     <none>    13s       v1.9.6
    k8s-master-50   Ready     <none>    13s       v1.9.6
    
    
    # 成功以后会自动生成配置文件与密钥
    
    # 配置文件
    
    ls /etc/kubernetes/kubelet.kubeconfig   
    /etc/kubernetes/kubelet.kubeconfig
    
    
    # 密钥文件 这里注意如果 csr 被删除了，请删除如下文件，并重启 kubelet 服务
    
    ls /etc/kubernetes/ssl/kubelet*
    /etc/kubernetes/ssl/kubelet-client.crt  /etc/kubernetes/ssl/kubelet.crt
    /etc/kubernetes/ssl/kubelet-client.key  /etc/kubernetes/ssl/kubelet.key

## 配置 kube-proxy ##

**创建 kube-proxy 证书**

# 证书方面由于我们node端没有装 cfssl
    # 我们回到 master 端 机器 去配置证书，然后拷贝过来
    
    cd /opt/ssl
    
    
    vi kube-proxy-csr.json
    
    {
      "CN": "system:kube-proxy",
      "hosts": [],
      "key": {
        "algo": "rsa",
        "size": 2048
      },
      "names": [
        {
          "C": "CN",
          "ST": "ShenZhen",
          "L": "ShenZhen",
          "O": "k8s",
          "OU": "System"
        }
      ]
    }

生成 kube-proxy 证书和私钥

/opt/local/cfssl/cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem \
      -ca-key=/etc/kubernetes/ssl/ca-key.pem \
      -config=/opt/ssl/config.json \
      -profile=kubernetes  kube-proxy-csr.json | /opt/local/cfssl/cfssljson -bare kube-proxy
    
    # 查看生成
    ls kube-proxy*
    kube-proxy.csr  kube-proxy-csr.json  kube-proxy-key.pem  kube-proxy.pem
    
    # 拷贝到目录
    
    cp kube-proxy* /etc/kubernetes/ssl/
    
    scp kube-proxy* root@192.168.1.48:/etc/kubernetes/ssl/
    scp kube-proxy* root@192.168.1.49:/etc/kubernetes/ssl/
    scp kube-proxy* root@192.168.1.50:/etc/kubernetes/ssl/

创建 kube-proxy kubeconfig 文件

# 配置集群
    
    kubectl config set-cluster kubernetes \
      --certificate-authority=/etc/kubernetes/ssl/ca.pem \   --embed-certs=true \   --server=https://127.0.0.1:6443 \   --kubeconfig=kube-proxy.kubeconfig 
    
    # 配置客户端认证
    
    kubectl config set-credentials kube-proxy \
      --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \   --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \   --embed-certs=true \   --kubeconfig=kube-proxy.kubeconfig 
    
    # 配置关联
    
    kubectl config set-context default \
      --cluster=kubernetes \   --user=kube-proxy \   --kubeconfig=kube-proxy.kubeconfig 
    
    
    # 配置默认关联
    kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
    
    # 拷贝到需要的 node 端里
    cp kube-proxy.kubeconfig /etc/kubernetes/
    scp kube-proxy.kubeconfig root@192.168.1.48:/etc/kubernetes/
    scp kube-proxy.kubeconfig root@192.168.1.49:/etc/kubernetes/
    scp kube-proxy.kubeconfig root@192.168.1.50:/etc/kubernetes/

创建 kube-proxy.service 文件

> 1.9 官方 ipvs 已经 beta , 尝试开启 ipvs 测试一下, 官方 –feature-gates=SupportIPVSProxyMode=false 默认是 false 的， 需要打开 –feature-gates=SupportIPVSProxyMode=true –masquerade-all 必须添加这项配置，否则 创建 svc 在 ipvs 不会添加规则  
> 打开 ipvs 需要安装 ipvsadm 软件， 在 node 中安装  
> yum install ipvsadm -y

# 创建 kube-proxy 目录
    
    mkdir -p /var/lib/kube-proxy
    
    
    vi /etc/systemd/system/kube-proxy.service
    
    [Unit]
    Description=Kubernetes Kube-Proxy Server
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=network.target
    
    [Service]
    WorkingDirectory=/var/lib/kube-proxy
    ExecStart=/usr/local/bin/kube-proxy \
      --bind-address=192.168.1.47 \   --hostname-override=k8s-master-47 \   --cluster-cidr=10.254.64.0/18 \   --masquerade-all \   --feature-gates=SupportIPVSProxyMode=true \   --proxy-mode=ipvs \   --ipvs-min-sync-period=5s \   --ipvs-sync-period=5s \   --ipvs-scheduler=rr \   --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \   --logtostderr=true \   --v=1 Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

启动 kube-proxy

systemctl daemon-reload
    systemctl enable kube-proxy
    systemctl start kube-proxy
    systemctl status kube-proxy

# 检查 ipvs
    
    [root@k8s-master-47 bin]# ipvsadm -L -n
    IP Virtual Server version 1.2.1 (size=4096)
    Prot LocalAddress:Port Scheduler Flags
      -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

# 如果报错 请使用
    journalctl -f -t kube-proxy  和 journalctl -u kube-proxy 来定位问题

## Node 端 ##

> 单 Node 部分 需要部署的组件有 docker flannel kubelet kube-proxy 这几个组件。 Node 节点 基于 Nginx 负载 API 做 Master HA

# master 之间除 api server 以外其他组件通过 etcd 选举，api server 默认不作处理；在每个 node 上启动一个 nginx，每个 nginx 反向代理所有 api server，node 上 kubelet、kube-proxy 连接本地的 nginx 代理端口，当 nginx 发现无法连接后端时会自动踢掉出问题的 api server，从而实现 api server 的 HA

**发布证书**

# ALL node
    
    mkdir -p /etc/kubernetes/ssl/
    
    scp ca.pem kube-proxy.pem kube-proxy-key.pem  node-*:/etc/kubernetes/ssl/

**创建Nginx 代理**

> 在每个 node 都必须创建一个 Nginx 代理， 这里特别注意， 当 Master 也做为 Node 的时候 不需要配置 Nginx-proxy

# 创建配置目录
    mkdir -p /etc/nginx
    
    # 写入代理配置
    cat << EOF >> /etc/nginx/nginx.conf
    error_log stderr notice;
    
    worker_processes auto;
    events {
      multi_accept on;
      use epoll;
      worker_connections 1024;
    }
    
    stream {
        upstream kube_apiserver {
            least_conn;
            server 192.168.1.47:6443;
            server 192.168.1.48:6443;
            server 192.168.1.49:6443;
        }
    
        server {
            listen        0.0.0.0:6443;
            proxy_pass    kube_apiserver;
            proxy_timeout 10m;
            proxy_connect_timeout 1s;
        }
    }
    EOF
    
    # 更新权限
    chmod +r /etc/nginx/nginx.conf

# 配置 Nginx 基于 docker 进程，然后配置 systemd 来启动
    
    cat << EOF >> /etc/systemd/system/nginx-proxy.service
    [Unit]
    Description=kubernetes apiserver docker wrapper
    Wants=docker.socket
    After=docker.service
    
    [Service]
    User=root
    PermissionsStartOnly=true
    ExecStart=/usr/bin/docker run -p 127.0.0.1:6443:6443 \\
                                  -v /etc/nginx:/etc/nginx \\
                                  --name nginx-proxy \\
                                  --net=host \\
                                  --restart=on-failure:5 \\
                                  --memory=512M \\
                                  nginx:1.13.7-alpine
    ExecStartPre=-/usr/bin/docker rm -f nginx-proxy
    ExecStop=/usr/bin/docker stop nginx-proxy
    Restart=always
    RestartSec=15s
    TimeoutStartSec=30s
    
    [Install]
    WantedBy=multi-user.target
    EOF

# 启动 Nginx
    
    systemctl daemon-reload
    systemctl start nginx-proxy
    systemctl enable nginx-proxy
    systemctl status nginx-proxy
    
    [root@k8s-node-1 nginx]# docker ps 
    CONTAINER ID        IMAGE                 COMMAND                  CREATED             STATUS              PORTS               NAMES
    fceae8e7cb06        nginx:1.13.7-alpine   "nginx -g 'daemon ..."   22 seconds ago      Up 22 seconds                           nginx-proxy

配置 Kubelet.service 文件

# 创建目录
    
    mkdir /var/lib/kubelet
    
    vi /etc/systemd/system/kubelet.service
    
    
    [Unit]
    Description=Kubernetes Kubelet
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=docker.service
    Requires=docker.service
    
    [Service]
    WorkingDirectory=/var/lib/kubelet
    ExecStart=/usr/local/bin/kubelet \
      --cgroup-driver=systemd \   --hostname-override=k8s-node-1 \   --pod-infra-container-image=jicki/pause-amd64:3.0 \   --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \   --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \   --cert-dir=/etc/kubernetes/ssl \   --cluster_dns=10.254.0.2 \   --cluster_domain=cluster.local \   --hairpin-mode promiscuous-bridge \   --allow-privileged=true \   --fail-swap-on=false \   --serialize-image-pulls=false \   --logtostderr=true \   --max-pods=512 \   --v=1 
    [Install]
    WantedBy=multi-user.target

# 启动 kubelet 与 kube-proxy
    
    systemctl daemon-reload
    systemctl start kubelet
    systemctl status kubelet

配置 kube-proxy.service

# 创建 kube-proxy 目录
    
    mkdir -p /var/lib/kube-proxy
    
    
    vi /etc/systemd/system/kube-proxy.service
    
    [Unit]
    Description=Kubernetes Kube-Proxy Server
    Documentation=https://github.com/GoogleCloudPlatform/kubernetes
    After=network.target
    
    [Service]
    WorkingDirectory=/var/lib/kube-proxy
    ExecStart=/usr/local/bin/kube-proxy \
      --bind-address=192.168.1.51 \   --hostname-override=k8s-node-1 \   --cluster-cidr=10.254.64.0/18 \   --masquerade-all \   --feature-gates=SupportIPVSProxyMode=true \   --proxy-mode=ipvs \   --ipvs-min-sync-period=5s \   --ipvs-sync-period=5s \   --ipvs-scheduler=rr \   --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \   --logtostderr=true \   --v=2 Restart=on-failure
    RestartSec=5
    LimitNOFILE=65536
    
    [Install]
    WantedBy=multi-user.target

启动kube-proxy

# 启动
    
    systemctl start kube-proxy
    systemctl status kube-proxy

Master 配置 TLS 认证

[root@k8s-master-48 bin]# kubectl get csr
    NAME                                                   AGE       REQUESTOR           CONDITION
    node-csr-mq6AUUNf17Fso5wJecgFkm7FtUUIMypO39IBTMCuglA   1m        kubelet-bootstrap   Pending
    
    # 增加 认证
    
    [root@k8s-master-47 ~]# kubectl get csr | grep Pending | awk '{print $1}' | xargs kubectl certificate approve
    certificatesigningrequest "node-csr-mq6AUUNf17Fso5wJecgFkm7FtUUIMypO39IBTMCuglA" approved

[root@k8s-master-49 bin]# kubectl get nodes
    NAME            STATUS    ROLES     AGE       VERSION
    k8s-master-47   Ready     <none>    5h        v1.9.6
    k8s-master-48   Ready     <none>    5h        v1.9.6
    k8s-master-49   Ready     <none>    5h        v1.9.6
    k8s-master-50   Ready     <none>    5h        v1.9.6
    k8s-node-1      Ready     <none>    45s       v1.9.6

限制 POD 的调度

> k8s-master-47 只做 master 不做 pod 调度，所以禁止调度到 k8s-master-47中, Pod 的调度是通过 kubelet 服务来启动的，但是不启动 kubelet 的话，节点在 node 里是不可见的。

root@k8s-master-49 bin]# kubectl cordon k8s-master-47
    node "k8s-master-47" cordoned
    [root@k8s-master-49 bin]# kubectl get nodes
    NAME            STATUS                     ROLES     AGE       VERSION
    k8s-master-47   Ready,SchedulingDisabled   <none>    5h        v1.9.6
    k8s-master-48   Ready                      <none>    5h        v1.9.6
    k8s-master-49   Ready                      <none>    5h        v1.9.6
    k8s-master-50   Ready                      <none>    5h        v1.9.6
    k8s-node-1      Ready                      <none>    2m        v1.9.6

## 配置 Flannel 网络 ##

> flannel 网络只部署在 kube-proxy 相关机器

安装flannel

yum install flannel -y

# 配置 flannel 网段
    
    etcdctl --endpoints=https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379\ --cert-file=/etc/kubernetes/ssl/etcd.pem \ --ca-file=/etc/kubernetes/ssl/ca.pem \ --key-file=/etc/kubernetes/ssl/etcd-key.pem \ set /flannel/network/config \ '{"Network":"10.254.64.0/18","SubnetLen":24,"Backend":{"Type":"host-gw"}}'

# 修改 flanneld 配置
    
    vi /etc/sysconfig/flanneld
    
    
    # Flanneld configuration options 
    
    # etcd 地址
    
    FLANNEL_ETCD_ENDPOINTS="https://192.168.1.47:2379,https://192.168.1.48:2379,https://192.168.1.49:2379"
    
    # 配置为上面的路径 flannel/network
    FLANNEL_ETCD_PREFIX="/flannel/network"
    
    # 其他的配置，可查看 flanneld --help,这里添加了 etcd ssl 认证
    FLANNEL_OPTIONS="-ip-masq=true -etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/kubernetes/ssl/etcd.pem -etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem -iface=enp0s8"

# 启动 flannel 
    
    systemctl daemon-reload
    systemctl enable flanneld
    systemctl start flanneld
    systemctl status flanneld

# 如果报错 请使用
    journalctl -f -t flanneld  和 journalctl -u flanneld 来定位问题

# 配置完毕，重启 docker
    
    systemctl daemon-reload
    systemctl restart docker
    systemctl status docker
    
    
    # 重启 kubelet
    
    systemctl daemon-reload
    systemctl restart kubelet
    systemctl status kubelet
    
    # 验证 网络
    
    ifconfig  查看  docker0 网络 是否已经更改为配置IP网段 
    [root@k8s-node-1 kubernetes]# ifconfig 
    docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
            inet 10.254.73.1  netmask 255.255.255.0  broadcast 0.0.0.0
            ether 02:42:9e:0f:61:b1  txqueuelen 0  (Ethernet)
            RX packets 0  bytes 0 (0.0 B)
            RX errors 0  dropped 0  overruns 0  frame 0
            TX packets 0  bytes 0 (0.0 B)
            TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

## 测试集群 ##

# 创建一个 nginx deplyment
    
    apiVersion: extensions/v1beta1 
    kind: Deployment 
    metadata: 
      name: nginx-dm
    spec: 
      replicas: 2
      template: 
        metadata: 
          labels: 
            name: nginx 
        spec: 
          containers: 
            - name: nginx 
              image: nginx:alpine 
              imagePullPolicy: IfNotPresent
              ports: 
                - containerPort: 80
    
    ---
    
    apiVersion: v1 
    kind: Service
    metadata: 
      name: nginx-svc 
    spec: 
      ports: 
        - port: 80
          targetPort: 80
          protocol: TCP 
      selector: 
        name: nginx

root@k8s-master-47 nginx-deployment]# kubectl get pods -owide
    NAME                        READY     STATUS    RESTARTS   AGE       IP             NODE
    nginx-dm-84f8f49555-49fg9   1/1       Running   0          33s       10.254.73.2    k8s-node-1
    nginx-dm-84f8f49555-9c6j4   1/1       Running   0          33s       10.254.111.2   k8s-master-48
    
    root@k8s-master-47 nginx-deployment]# kubectl get svc -o wide
    NAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE       SELECTOR
    kubernetes   ClusterIP   10.254.0.1     <none>        443/TCP   7h        <none>
    nginx-svc    ClusterIP   10.254.23.37   <none>        80/TCP    1m        name=nginx

[root@k8s-master-48 bin]# curl 10.254.111.2
    <!DOCTYPE html>
    <html>
    <head>
    <title>Welcome to nginx!</title>
    <style> body { width: 35em; margin: 0 auto; font-family: Tahoma, Verdana, Arial, sans-serif; } </style>
    </head>
    <body>
    <h1>Welcome to nginx!</h1>
    <p>If you see this page, the nginx web server is successfully installed and
    working. Further configuration is required.</p>
    
    <p>For online documentation and support please refer to
    <a href="http://nginx.org/">nginx.org</a>.<br/>
    Commercial support is available at
    <a href="http://nginx.com/">nginx.com</a>.</p>
    
    <p><em>Thank you for using nginx.</em></p>
    </body>
    </html>

## 配置 CoreDNS ##

> 官方 地址 [https://coredns.io][https_coredns.io]

下载镜像

# 官方镜像
    coredns/coredns:1.0.4
    
    
    # 我的镜像
    jicki/coredns:1.0.4

载 yaml 文件

wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed
    
    mv coredns.yaml.sed coredns.yaml

# vi coredns.yaml
    
    ...
    data:
      Corefile: |
        .:53 {
            errors
            health
            kubernetes cluster.local 10.254.0.0/18 {
              pods insecure
              upstream /etc/resolv.conf
              fallthrough in-addr.arpa ip6.arpa
            }
    ...        
            image: jicki/coredns:1.0.4
    ...        
      clusterIP: 10.254.0.2

# 配置说明 
    
    
    # 这里 kubernetes cluster.local 为 创建 svc 的 IP 段
    
    kubernetes cluster.local 10.254.0.0/18 
    
    # clusterIP 为 指定 DNS 的 IP
    
    clusterIP: 10.254.0.2

导入 yaml 文件

[root@k8s-master-47 coredns]# kubectl apply -f coredns.yaml
    serviceaccount "coredns" created
    clusterrole "system:coredns" created
    clusterrolebinding "system:coredns" created
    configmap "coredns" created
    deployment "coredns" created
    service "kube-dns" created

查看 kubedns 服务

root@k8s-master-47 coredns]# kubectl get pod,svc -n kube-system -owide
    NAME                          READY     STATUS    RESTARTS   AGE       IP             NODE
    po/coredns-64b597b598-hkxmv   1/1       Running   0          31s       10.254.73.3    k8s-node-1
    po/coredns-64b597b598-n5f4r   1/1       Running   0          31s       10.254.106.2   k8s-master-49
    
    NAME           TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGE       SELECTOR
    svc/kube-dns   ClusterIP   10.254.0.2   <none>        53/UDP,53/TCP   31s       k8s-app=kube-dns

检查日志

2018/05/12 10:53:24 [INFO] plugin/reload: Running configuration MD5 = fe3c1515172455f38e799f20bfaef909 .:53 CoreDNS-1.1.2 linux/amd64, go1.10.1, 582f91f3

验证 dns 服务

> 在验证 dns 之前，在 dns 未部署之前创建的 pod 与 deployment 等，都必须删除，重新部署，否则无法解析

[root@k8s-master-47 nginx-deployment]# kubectl delete -f .
    deployment "nginx-dm" deleted
    service "nginx-svc" deleted
    [root@k8s-master-47 nginx-deployment]# kubectl create -f .
    deployment "nginx-dm" created
    service "nginx-svc" created

# 创建一个 pods 来测试一下 dns 
    
    apiVersion: v1
    kind: Pod
    metadata:
      name: alpine
    spec:
      containers:
      - name: alpine
        image: alpine
        command:
        - sh
        - -c
        - while true; do sleep 1; done
    
    
    
    # 查看 创建的服务
    
    root@k8s-master-47 nginx-deployment]# kubectl get pods,svc
    NAME                           READY     STATUS    RESTARTS   AGE
    po/alpine                      1/1       Running   0          8s
    po/nginx-dm-84f8f49555-79cdg   1/1       Running   0          8s
    po/nginx-dm-84f8f49555-tl9bb   1/1       Running   0          8s
    
    NAME             TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE
    svc/kubernetes   ClusterIP   10.254.0.1     <none>        443/TCP   7h
    svc/nginx-svc    ClusterIP   10.254.56.44   <none>        80/TCP    8s
    
    
    
    # 测试
    
    [root@k8s-master-47 ~]# kubectl exec -it alpine nslookup nginx-svc
    nslookup: can't resolve '(null)': Name does not resolve Name: nginx-svc Address 1: 10.254.40.179 nginx-svc.default.svc.cluster.local [root@k8s-master-47 yaml]# kubectl exec -it alpine nslookup kubernetes nslookup: can't resolve '(null)': Name does not resolve
    
    Name:      kubernetes
    Address 1: 10.254.0.1 kubernetes.default.svc.cluster.local

## 部署 Ingress 与 Dashboard ##

下载 dashboard 镜像

k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3

下载 yaml 文件

curl -O https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml

导入 yaml

kubectl create -f kubernetes-dashboard.yaml
    
    secret "kubernetes-dashboard-certs" created
    serviceaccount "kubernetes-dashboard" created
    role "kubernetes-dashboard-minimal" created
    rolebinding "kubernetes-dashboard-minimal" created
    deployment "kubernetes-dashboard" created
    service "kubernetes-dashboard" created
    
    [root@k8s-master-47 dashboard]# kubectl get pods,svc -n kube-system
    NAME                                       READY     STATUS    RESTARTS   AGE
    po/coredns-64b597b598-hkxmv                1/1       Running   0          1d
    po/coredns-64b597b598-n5f4r                1/1       Running   0          1d
    po/kubernetes-dashboard-5bd6f767c7-qndsk   1/1       Running   0          11m
    
    NAME                       TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
    svc/kube-dns               ClusterIP   10.254.0.2      <none>        53/UDP,53/TCP   1d
    svc/kubernetes-dashboard   ClusterIP   10.254.44.121   <none>        443/TCP         11m

访问dashboard,参考官网[Accessing-Dashboard][]

访问[https://192.168.1.50:30746/\#!/login][https_192.168.1.50_30746_login]

![dash][]  
获取token

[root@k8s-master-47 .kube]# kubectl get secret --all-namespaces kubernetes-dashboard-token-5nplq -o yaml
    error: a resource cannot be retrieved by name across all namespaces
    [root@k8s-master-47 .kube]# kubectl get secret -n kube-system kubernetes-dashboard-token-5nplq -o yaml
    apiVersion: v1
    data:
      ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR3akNDQXFxZ0F3SUJBZ0lVWTA2NjNKb1M1d2svWnhCRzJXSTladERMVUM4d0RRWUpLb1pJaHZjTkFRRUwKQlFBd1p6RUxNQWtHQTFVRUJoTUNRMDR4RVRBUEJnTlZCQWdUQ0ZOb1pXNWFhR1Z1TVJFd0R3WURWUVFIRXdoVAphR1Z1V21obGJqRU1NQW9HQTFVRUNoTURhemh6TVE4d0RRWURWUVFMRXdaVGVYTjBaVzB4RXpBUkJnTlZCQU1UCkNtdDFZbVZ5Ym1WMFpYTXdIaGNOTVRnd05URXlNREUxTmpBd1doY05Nak13TlRFeE1ERTFOakF3V2pCbk1Rc3cKQ1FZRFZRUUdFd0pEVGpFUk1BOEdBMVVFQ0JNSVUyaGxibHBvWlc0eEVUQVBCZ05WQkFjVENGTm9aVzVhYUdWdQpNUXd3Q2dZRFZRUUtFd05yT0hNeER6QU5CZ05WQkFzVEJsTjVjM1JsYlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1ClpYUmxjekNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFPYjhmNWFPL2ZUMkI2ODIKSTQ5OG5OZjBVZXJTcHg0eEFyeUVNU3JtVHVMWGFhaHRVRC9nQ1RISjg5U3NRMldvUklkQjVTRkNmU1pHR2dBSgpnR0hFZEtZbzJyVWlVTUFoNDZYKy9HM0ljc1VsOG5xWnJ3L2VTOUpCa1BKTCtxK2lxYy9pakhaVTVrVXRFUjFaCm41L3VtbVlhWUxxU1VJTU1SOXhXbVVNNUVURnRZVmU3NVZadHF4MkZRMHhsRUg3MW5hWE95YWQxaWZLeDlhRUkKaEZpbTdPQ1JNNnBLay8wUVNNSjVtbVhkT2pzK0x3cUN2bU51Nk5LV1pvN1VUZUVPcWZnUVlZcW1makJuamxPbQp4L0FjcXg1dUllSmhTY1FSVFNIY2M4b0UyR0g0SGtZVG1GSWZ4WjJkRWVXdGprMGtTVFZMYUNxMWpiMGlseUhlCnQ2TzJjVWNDQXdFQUFhTm1NR1F3RGdZRFZSMFBBUUgvQkFRREFnRUdNQklHQTFVZEV3RUIvd1FJTUFZQkFmOEMKQVFJd0hRWURWUjBPQkJZRUZFbmh5WG5NSUFQWkJmTG9IRk1kVCtJM2ZPR0hNQjhHQTFVZEl3UVlNQmFBRkVuaAp5WG5NSUFQWkJmTG9IRk1kVCtJM2ZPR0hNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUROZVR6VFFjcktRMWgrCkxmaEpxWHJQRVoxTk5FdWcvYmtBdkI1R2t4akt2dEhuV1NTQ1l1TWxrV0h2eUp3Tm9TZUo5Rk9xU3lMeFFWNFoKdWhDeVhRUG14U1lZOExiaXoyWXg1dHhOQ3grNllkRVZyQXVMTGFZK0NDRGV6SzdPRUhoYVMvZjk5MWRLaDEybQpEeG1HRytBaTVkSGE3b0VwS1VLNWh3SkhYTzhCN05pQWd5ZGN4ZHJTVDJYR1JoRmRsVmd3dTE3b3BvYjE0TTUyCi9Vb2R4TzJPMHZFTFJoK0ZqZnR1Mjc4dmh5OWR5R0dPN2VnU2J5TkJJeG5TSTVOeG5hYTkrTXR0SGNzbmlQbjAKaHQ4b3lpWi9RS3FSYmZ6Mkh6QitPQlpXREVLK2lsMjNCbktkUEd1UERmVzNTTEs1MndKNitkTVV3bEp6cWhkdgpGMUtMaUlPTgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
      namespace: a3ViZS1zeXN0ZW0=
      token: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUpyZFdKbExYTjVjM1JsYlNJc0ltdDFZbVZ5Ym1WMFpYTXVhVzh2YzJWeWRtbGpaV0ZqWTI5MWJuUXZjMlZqY21WMExtNWhiV1VpT2lKcmRXSmxjbTVsZEdWekxXUmhjMmhpYjJGeVpDMTBiMnRsYmkwMWJuQnNjU0lzSW10MVltVnlibVYwWlhNdWFXOHZjMlZ5ZG1salpXRmpZMjkxYm5RdmMyVnlkbWxqWlMxaFkyTnZkVzUwTG01aGJXVWlPaUpyZFdKbGNtNWxkR1Z6TFdSaGMyaGliMkZ5WkNJc0ltdDFZbVZ5Ym1WMFpYTXVhVzh2YzJWeWRtbGpaV0ZqWTI5MWJuUXZjMlZ5ZG1salpTMWhZMk52ZFc1MExuVnBaQ0k2SWpCbE5HVXhaak0zTFRVMk9XVXRNVEZsT0MxaE1EZG1MVEE0TURBeU4yVmhabVJoTUNJc0luTjFZaUk2SW5ONWMzUmxiVHB6WlhKMmFXTmxZV05qYjNWdWREcHJkV0psTFhONWMzUmxiVHByZFdKbGNtNWxkR1Z6TFdSaGMyaGliMkZ5WkNKOS50TmdKS0trRmcyYVl0alNiSjNFSlFuLV9URW5vNHpVak85b0VmNFhTWE1DYnVYX1FFVk9zajZ6MmlIZXNjSmhuZ3pKS2lRQ2JmNVNvTGR2Qm9CS1FjUHJTdUc4NXVTX1VZWVBUeW5iUXhLZzFoQU1WdEhZMXE5ZlNrQU8yOS1GeUkwM2ZydlBJT2phakRYWjgtX1QwSFBiNUNjaGl2aWpBUXZFN2tPaFVFQnM1N1l3T0tQYl9pR0hqcnY3ZFBkR1k1MDhmZ0QwdG9vY0ljczh5XzRGV0VtOXF0Y3lDX09xNzRfTGxaaTA5cjVYRmRIbHBHVUR6SFp2QUJHUGo4OUw4LTRUUGdTNzgzNVdwSVJodmFRU1k1cGJ0MWUtcktjamE0S2VvNi1Ia0RWa2VFRlFDclBZU0FGS1BPTVJDMmc5UGpqbXJ3Z3NZM2s0RmVudDBYa2hrRUE=
    kind: Secret
    metadata:
      annotations:
        kubernetes.io/service-account.name: kubernetes-dashboard
        kubernetes.io/service-account.uid: 0e4e1f37-569e-11e8-a07f-080027eafda0
      creationTimestamp: 2018-05-13T11:09:07Z
      name: kubernetes-dashboard-token-5nplq
      namespace: kube-system
      resourceVersion: "159723"
      selfLink: /api/v1/namespaces/kube-system/secrets/kubernetes-dashboard-token-5nplq
      uid: 0e48cd4a-569e-11e8-b69f-080027ab6e55
    type: kubernetes.io/service-account-token

然后就可以进入dashboard  
![dash01][]  
这个问题 有待解决  
未完待续

参考  
[kubernetes-1.9.1][]

[https_github.com_coreos_etcd_releases]: https://github.com/coreos/etcd/releases
[k8s]: /images/20220525/192af83a3fda440aa24a0fd293f94585.png
[https_coredns.io]: https://coredns.io
[Accessing-Dashboard]: https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above
[https_192.168.1.50_30746_login]: https://192.168.1.50:30746/#!/login
[dash]: /images/20220525/aeda47c1661243dfa0dcd3cdaf8a3563.png
[dash01]: /images/20220525/941f4667c3dd493fbaf6990ebf3ed905.png
[kubernetes-1.9.1]: https://jicki.me/kubernetes/2018/01/23/kubernetes-1.9.1.html