二进制手动搭建K8S集群--01-etcd集群、kubectl、kube-apiserver部署

$ kubectl delete node --all      //卸载所有节点
  node "wqdcsrv032.cn.infra" deleted
  node "wqdcsrv033" deleted
  node "wqdcsrv033.cn.infra" deleted


$ kubeadm reset -f            
$ modprobe -r ipip
$ lsmod


$ yum remove -y kubelet kubeadm kubectl     //卸载以上软件
Loaded plugins: langpacks, ulninfo
Resolving Dependencies
--> Running transaction check
---> Package kubeadm.x86_64 0:1.15.1-0 will be erased
---> Package kubectl.x86_64 0:1.15.1-0 will be erased
---> Package kubelet.x86_64 0:1.15.1-0 will be erased
----省略部分信息
Removed:
  kubeadm.x86_64 0:1.15.1-0                                                       kubectl.x86_64 0:1.15.1-0                                                       kubelet.x86_64 0:1.15.1-0


$ rm -rf ~/.kube/        //删除所有组建的系统文件
  rm -rf /etc/kubernetes/
  rm -rf /etc/systemd/system/kubelet.service.d
  rm -rf /etc/systemd/system/kubelet.service
  rm -rf /usr/bin/kube*
  rm -rf etc/cni
  rm -rf opt/cni
  rm -rf var/lib/etcd
  rm -rf /var/etcd

# ll
total 548584
-rw-r--r-- 1 root root  10423953 Jul 30 17:25 etcd-v3.3.13-linux-amd64.tar.gz
-rw-r--r-- 1 root root  13337585 Jul 30 17:18 kubernetes-client-linux-amd64.tar.gz
-rw-r--r-- 1 root root  94223529 Jul 30 17:18 kubernetes-node-linux-amd64.tar.gz
-rw-r--r-- 1 root root 443761836 Jul 30 17:24 kubernetes-server-linux-amd64.tar.gz

mkdir var/log/journal # 持久化保存日志的目录
mkdir etc/systemd/journald.conf.d
cat > etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
# 持久化保存到磁盘
Storage=persistent


# 压缩历史日志
Compress=yes


SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000


# 最大占用空间 10G
SystemMaxUse=10G


# 单日志文件最大 200M
SystemMaxFileSize=200M


# 日志保存时间 2 周
MaxRetentionSec=2week


# 不将日志转发到 syslog
ForwardToSyslog=no
EOF
systemctl restart systemd-journald

vim  environment.sh


#!/usr/bin/bash


# 生成 EncryptionConfig 所需的加密 key
export ENCRYPTION_KEY=$(head -c 32 dev/urandom | base64)


# 集群各机器 IP 数组
export NODE_IPS=(10.67.194.43 10.67.194.42 10.67.194.41)


# 集群各 IP 对应的主机名数组
export NODE_NAMES=(master.kube node1.kube node2.kube)


# etcd 集群服务地址列表
export ETCD_ENDPOINTS="https://10.67.194.43:2379,https://10.67.194.42,https://10.67.194.41:2379"


# etcd 集群间通信的 IP 和端口
export ETCD_NODES="master.kube=https://10.67.194.43:2380,node1.kube=https://10.67.194.42:2380,node2.kube=https://10.67.194.41:2380"


# kube-apiserver 的反向代理(kube-nginx)地址端口
export KUBE_APISERVER="https://127.0.0.1:8443"


# 节点间互联网络接口名称
export IFACE="eth0"


# etcd 数据目录
export ETCD_DATA_DIR="/data/k8s/etcd/data"


# etcd WAL 目录，建议是 SSD 磁盘分区，或者和 ETCD_DATA_DIR 不同的磁盘分区
export ETCD_WAL_DIR="/data/k8s/etcd/wal"


# k8s 各组件数据目录
export K8S_DIR="/data/k8s/k8s"


# docker 数据目录
export DOCKER_DIR="/data/k8s/docker"


## 以下参数一般不需要修改


# TLS Bootstrapping 使用的 Token，可以使用命令 head -c 16 dev/urandom | od -An -t x | tr -d ' ' 生成
BOOTSTRAP_TOKEN="41f7e4ba8b7be874fcff18bf5cf41a7c"


# 最好使用 当前未用的网段 来定义服务网段和 Pod 网段


# 服务网段，部署前路由不可达，部署后集群内路由可达(kube-proxy 保证)
SERVICE_CIDR="10.254.0.0/16"


# Pod 网段，建议 16 段地址，部署前路由不可达，部署后集群内路由可达(flanneld 保证)
CLUSTER_CIDR="172.30.0.0/16"


# 服务端口范围 (NodePort Range)
export NODE_PORT_RANGE="30000-32767"


# flanneld 网络配置前缀
export FLANNEL_ETCD_PREFIX="/kubernetes/network"


# kubernetes 服务 IP (一般是 SERVICE_CIDR 中第一个IP)
export CLUSTER_KUBERNETES_SVC_IP="10.254.0.1"


# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配)
export CLUSTER_DNS_SVC_IP="10.254.0.2"


# 集群 DNS 域名（末尾不带点号）
export CLUSTER_DNS_DOMAIN="cluster.local"


# 将二进制目录 opt/k8s/bin 加到 PATH 中
# export PATH=/opt/k8s/bin:$PATH

mkdir -p opt/k8s/cert && cd opt/k8s/
 
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
ll
total 567386
drwxr-xr-x 2 root root      1024 Jul 31 17:15 cert
-rw-r--r-- 1 root root   6595195 Jul 31 16:57 cfssl-certinfo_linux-amd64
-rw-r--r-- 1 root root   2277873 Jul 31 16:57 cfssljson_linux-amd64
-rw-r--r-- 1 root root  10376657 Jul 31 16:57 cfssl_linux-amd64


mkdir opt/k8s/bin/


mv cfssl_linux-amd64 opt/k8s/bin/cfssl
mv cfssljson_linux-amd64 opt/k8s/bin/cfssljson
mv cfssl-certinfo_linux-amd64 opt/k8s/bin/cfssl-certinfo


chmod +x opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH

mkdir opt/k8s/work
cd opt/k8s/work


vim ca-config.json
****************************************
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
****************************************
-- signing：表示该证书可用于签名其它证书，生成的 ca.pem 证书中 CA=TRUE；
-- server auth：表示 client 可以用该证书对 server 提供的证书进行验证；
-- client auth：表示 server 可以用该证书对 client 提供的证书进行验证；

cd opt/k8s/work
cat > ca-csr.json <<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ],
  "ca": {
    "expiry": "876000h"
 }
}
EOF
**************************************************************
-- CN：Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名 (User Name)，浏览器使用该字段验证网站是否合法；
-- O：Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)；
-- kube-apiserver 将提取的 User、Group 作为 RBAC 授权的用户标识；

cd /opt/k8s/work
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2019/07/31 17:34:06 [INFO] generating a new CA key and certificate from CSR
2019/07/31 17:34:06 [INFO] generate received request
2019/07/31 17:34:06 [INFO] received CSR
2019/07/31 17:34:06 [INFO] generating key: rsa-2048
2019/07/31 17:34:06 [INFO] encoded CSR
2019/07/31 17:34:06 [INFO] signed certificate with serial number 586104341755097112569226212982884644575651160362


ll ca*
-rw-r--r-- 1 root root  292 Jul 31 17:27 ca-config.json
-rw-r--r-- 1 root root 1005 Jul 31 17:34 ca.csr
-rw-r--r-- 1 root root  249 Jul 31 17:29 ca-csr.json
-rw------- 1 root root 1675 Jul 31 17:34 ca-key.pem
-rw-r--r-- 1 root root 1371 Jul 31 17:34 ca.pem

#在master、node1、2创建文件
 mkdir /etc/kubernetes/cert -p


#将主机的文件传过来
scp etc/kubernetes/ca/ca.*  root@10.67.194.4*:/etc/kubernetes/cert


#我这有跳板机......lrzsz吧，也记录一下
sz ca*   
rz -e 五次...x2


#最终把所有文件放到/etc/kubernetes/cert/

$ kubectl get pods
The connection to the server localhost:8080 was refused - did you specify the right host or port?

cd opt/k8s/work
wget https://dl.k8s.io/v1.15.1/kubernetes-client-linux-amd64.tar.g
tar -xzvf kubernetes-client-linux-amd64.tar.gz


分发到所有使用 kubectl 的节点的/opt/ks/bin/目录下
 scp kubernetes/client/bin/kubectl root@${node_ip}:/opt/k8s/bin/
 chmod +x /opt/k8s/bin/*
 export PATH=/opt/k8s/bin:$PATH

cd opt/k8s/work
cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:masters",
      "OU": "4Paradigm"
    }
  ]
}
EOF
*****************************************
-- O 为 system:masters，kube-apiserver 收到该证书后将请求的 Group 设置为 system:masters；
-- 预定义的 ClusterRoleBinding cluster-admin 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予所有 API的权限；
-- 该证书只会被 kubectl 当做 client 证书使用，所以 hosts 字段为空；

cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \
  -ca-key=/opt/k8s/work/ca-key.pem \
  -config=/opt/k8s/work/ca-config.json \
  -profile=kubernetes admin-csr.json | cfssljson -bare admin
ls admin*

cd opt/k8s/work
# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/k8s/work/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kubectl.kubeconfig


# 设置客户端认证参数
kubectl config set-credentials admin \
  --client-certificate=/opt/k8s/work/admin.pem \
  --client-key=/opt/k8s/work/admin-key.pem \
  --embed-certs=true \
  --kubeconfig=kubectl.kubeconfig


# 设置上下文参数
kubectl config set-context kubernetes \
  --cluster=kubernetes \
  --user=admin \
  --kubeconfig=kubectl.kubeconfig


# 设置默认上下文
kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig
************************************************
-- certificate-authority：验证 kube-apiserver 证书的根证书；
-- client-certificate、--client-key：刚生成的 admin 证书和私钥，连接 kube-apiserver 时使用；
-- embed-certs=true：将 ca.pem 和 admin.pem 证书内容嵌入到生成的 kubectl.kubeconfig 文件中(不加时，写入的是证书文件路径，后续拷贝 kubeconfig 到其它机器时，还需要单独拷贝证书文件，不方便。)；

cd opt/k8s/work


#在所有节点创建~/.kube
mkdir ~/.kube


#然后把文件加入到 .kube
scp kubectl.kubeconfig root@${node_ip}:~/.kube/config       

cd /opt/k8s/work
 wget https://github.com/etcd-io/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz 


 ll
-rw-r--r-- 1 root root 10423953 Jul 29 17:49 etcd-v3.3.13-linux-amd64.tar.gz


tar -zxvf etcd-v3.3.13-linux-amd64.tar.gz 


cd etcd-v3.3.13-linux-amd64/


ll
total 29776
drwxr-xr-x 10 1000 1000     4096 May  3 01:55 Documentation
-rwxr-xr-x  1 1000 1000 16927136 May  3 01:55 etcd
-rwxr-xr-x  1 1000 1000 13498880 May  3 01:55 etcdctl
-rw-r--r--  1 1000 1000    38864 May  3 01:55 README-etcdctl.md
-rw-r--r--  1 1000 1000     7262 May  3 01:55 README.md
-rw-r--r--  1 1000 1000     7855 May  3 01:55 READMEv2-etcdctl.md


cp etc* opt/k8s/bin/ 


不要忘记传到所有node，我是lrzsz...
#Master上：sz etc*
#所有Node上： mkdir /etc/k8s/bin/ -p 
             rz -e
         chmod -R +x etc/k8s/bin/

cd opt/k8s/work 


cat > etcd-csr.json <<EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "10.67.194.43",
    "10.67.194.42",
    "10.67.194.41"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF
*********************************************
-- hosts 字段指定授权使用该证书的 etcd 节点 IP 或域名列表，需要将 etcd 集群的三个节点 IP 都列在其中；

cd opt/k8s/work
cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
>     -ca-key=/etc/kubernetes/cert/ca-key.pem \
>     -config=/etc/kubernetes/certc/ca-config.json \
>     -profile=kubernetes etcd-csr.json | cfssljson -bare etcd


2019/07/31 18:12:55 [INFO] generate received request
2019/07/31 18:12:55 [INFO] received CSR
2019/07/31 18:12:55 [INFO] generating key: rsa-2048
2019/07/31 18:12:56 [INFO] encoded CSR
2019/07/31 18:12:56 [INFO] signed certificate with serial number 279848959295734072541807189313815280743075425460
2019/07/31 18:12:56 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").


ll etcd*pem
-rw------- 1 root root 1675 Jul 31 18:12 etcd-key.pem
-rw-r--r-- 1 root root 1444 Jul 31 18:12 etcd.pem

#Master上操作：    sz etc* 
# mkdir etc/etcd/cert -p
#所有Node上操作：   rz -e  到//etc/etcd/cert/

cd opt/k8s/work
source /opt/k8s/bin/environment.sh


cat > etcd.service.template <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos


[Service]
Type=notify
WorkingDirectory=${ETCD_DATA_DIR}
ExecStart=/opt/k8s/bin/etcd \\
  --data-dir=${ETCD_DATA_DIR} \\
  --wal-dir=${ETCD_WAL_DIR} \\
  --name=##NODE_NAME## \\
  --cert-file=/etc/etcd/cert/etcd.pem \\
  --key-file=/etc/etcd/cert/etcd-key.pem \\
  --trusted-ca-file=/etc/kubernetes/cert/ca.pem \\
  --peer-cert-file=/etc/etcd/cert/etcd.pem \\
  --peer-key-file=/etc/etcd/cert/etcd-key.pem \\
  --peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \\
  --peer-client-cert-auth \\
  --client-cert-auth \\
  --listen-peer-urls=https://##NODE_IP##:2380 \\
  --initial-advertise-peer-urls=https://##NODE_IP##:2380 \\
  --listen-client-urls=https://##NODE_IP##:2379,http://127.0.0.1:2379 \\
  --advertise-client-urls=https://##NODE_IP##:2379 \\
  --initial-cluster-token=etcd-cluster-0 \\
  --initial-cluster=${ETCD_NODES} \\
  --initial-cluster-state=new \\
  --auto-compaction-mode=periodic \\
  --auto-compaction-retention=1 \\
  --max-request-bytes=33554432 \\
  --quota-backend-bytes=6442450944 \\
  --heartbeat-interval=250 \\
  --election-timeout=2000
Restart=on-failure
RestartSec=5
LimitNOFILE=65536


[Install]
WantedBy=multi-user.target
EOF
******************************************
-- WorkingDirectory、--data-dir：指定工作目录和数据目录为 ${ETCD_DATA_DIR}，需在启动服务前创建这个目录；
-- wal-dir：指定 wal 目录，为了提高性能，一般使用 SSD 或者和 --data-dir 不同的磁盘；
-- name：指定节点名称，当 --initial-cluster-state 值为 new 时，--name 的参数值必须位于 --initial-cluster 列表中；
-- cert-file、--key-file：etcd server 与 client 通信时使用的证书和私钥；
-- trusted-ca-file：签名 client 证书的 CA 证书，用于验证 client 证书；
-- peer-cert-file、--peer-key-file：etcd 与 peer 通信使用的证书和私钥；
-- peer-trusted-ca-file：签名 peer 证书的 CA 证书，用于验证 peer 证书；

cd opt/k8s/work


source opt/k8s/bin/environment.sh
for (( i=0; i < 3; i++ ))
  do
    sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" etcd.service.template > etcd-${NODE_IPS[i]}.service 
  done


ls *.service


****************************************************************
-- NODE_NAMES 和 NODE_IPS 为相同长度的 bash 数组，分别为节点名称和对应的 IP；

文件重命名为 etcd.service！！！
 scp etcd-${node_ip}.service root@${node_ip}:/etc/systemd/system/etcd.service
 ************************
 --方法很多，自行尝试

source opt/k8s/bin/environment.sh
mkdir -p ${ETCD_DATA_DIR} ${ETCD_WAL_DIR}
chmod 755 opt/k8s/bin/* -R


chmod +x -R etc/etcd/cert/


chmod -R +x etc/kubernetes/cert


systemctl daemon-reload 
systemctl enable etcd.service 
systemctl start etcd.service

etcdctl cluster-health
failed to check the health of member 512545bc7e1e6eed on https://10.67.194.42:2379: Get https://10.67.194.42:2379/health: x509: certificate signed by unknown authority
member 512545bc7e1e6eed is unreachable: [https://10.67.194.42:2379] are all unreachable
failed to check the health of member d7069edb0fde59b2 on https://10.67.194.41:2379: Get https://10.67.194.41:2379/health: x509: certificate signed by unknown authority
member d7069edb0fde59b2 is unreachable: [https://10.67.194.41:2379] are all unreachable
failed to check the health of member efdef5ee8872f5ac on https://10.67.194.43:2379: Get https://10.67.194.43:2379/health: x509: certificate signed by unknown authority
member efdef5ee8872f5ac is unreachable: [https://10.67.194.43:2379] are all unreachable
cluster is unavailable
**************************有错误********************************
这个报错应该是证书的问题了，不带证书测试就是报这个错误，带证书后，测试正常，见下：






# 指定证书检查
 etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem --endpoints=https://10.67.194.43:2379,https://10.67.194.42:2379,https://10.67.194.41:2379 cluster-health
member 512545bc7e1e6eed is healthy: got healthy result from https://10.67.194.42:2379
member d7069edb0fde59b2 is healthy: got healthy result from https://10.67.194.41:2379
member efdef5ee8872f5ac is healthy: got healthy result from https://10.67.194.43:2379
cluster is healthy


#验证服务状态
for node_ip in ${NODE_IPS[@]}
>   do
>     echo ">>> ${node_ip}"
>     ETCDCTL_API=3 opt/k8s/bin/etcdctl \
>     --endpoints=https://${node_ip}:2379 \
>     --cacert=/etc/kubernetes/cert/ca.pem \
>     --cert=/etc/etcd/cert/etcd.pem \
>     --key=/etc/etcd/cert/etcd-key.pem endpoint health
>   done
>>> 10.67.194.43
https://10.67.194.43:2379 is healthy: successfully committed proposal: took = 1.459379ms
>>> 10.67.194.42
https://10.67.194.42:2379 is healthy: successfully committed proposal: took = 1.69417ms
>>> 10.67.194.41
https://10.67.194.41:2379 is healthy: successfully committed proposal: took = 1.42945ms


#查看集群状态
ETCDCTL_API=3 opt/k8s/bin/etcdctl \
  -w table --cacert=/etc/kubernetes/cert/ca.pem \
  --cert=/etc/etcd/cert/etcd.pem \
  --key=/etc/etcd/cert/etcd-key.pem \
  --endpoints=${ETCD_ENDPOINTS} endpoint status 
Failed to get the status of endpoint https://10.67.194.42 (context deadline exceeded)
+---------------------------+------------------+---------+---------+-----------+-----------+------------+
|         ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+---------------------------+------------------+---------+---------+-----------+-----------+------------+
| https://10.67.194.43:2379 | efdef5ee8872f5ac |  3.3.13 |   20 kB |      true |      1697 |         28 |
| https://10.67.194.41:2379 | d7069edb0fde59b2 |  3.3.13 |   20 kB |     false |      1697 |         28 |
+---------------------------+------------------+---------+---------+-----------+-----------+------------+




#查看主etcd
etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem --endpoints=https://10.67.194.43:2379,https://10.67.194.42:2379,https://10.67.194.41:2379? member list


512545bc7e1e6eed: name=node1.kube peerURLs=https://10.67.194.42:2380 clientURLs=https://10.67.194.42:2379 isLeader=false
d7069edb0fde59b2: name=node2.kube peerURLs=https://10.67.194.41:2380 clientURLs=https://10.67.194.41:2379 isLeader=false
efdef5ee8872f5ac: name=master.kube peerURLs=https://10.67.194.43:2380 clientURLs=https://10.67.194.43:2379 isLeader=true


#测试数据master创建
etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem mkdir test
etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem mkdir ls
etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem ls
/ls
/test
#在节点查看数据已经同步

cd opt/k8s/work
mkdir flannel
tar -xzvf flannel-v0.11.0-linux-amd64.tar.gz -C flannel

cd opt/k8s/work
source opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
  do
    echo ">>> ${node_ip}"
    scp flannel/{flanneld,mk-docker-opts.sh} root@${node_ip}:/opt/k8s/bin/
    ssh root@${node_ip} "chmod +x opt/k8s/bin/*"
  done
  
  chmod +x opt/k8s/bin/*

cd opt/k8s/work
cat > flanneld-csr.json <<EOF
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF
**************************************************
-- 该证书只会被 kubectl 当做 client 证书使用，所以 hosts 字段为空

cfssl gencert -ca=/opt/k8s/work/ca.pem \
  -ca-key=/opt/k8s/work/ca-key.pem \
  -config=/opt/k8s/work/ca-config.json \
  -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
  
  ll flanneld*pem

mkdir -p etc/flanneld/cert
将flanel的证书分发到三个节点
chmod +x *

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
etcdctl \
  --endpoints=${ETCD_ENDPOINTS} \
  --ca-file=/opt/k8s/work/ca.pem \
  --cert-file=/opt/k8s/work/flanneld.pem \
  --key-file=/opt/k8s/work/flanneld-key.pem \
  mk ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 21, "Backend": {"Type": "vxlan"}}'
------------------------------  
{"Network":"172.30.0.0/16", "SubnetLen": 21, "Backend": {"Type": "vxlan"}}    


-- flanneld 当前版本 (v0.11.0) 不支持 etcd v3，故使用 etcd v2 API 写入配置 key 和网段数据；
-- 写入的 Pod 网段 ${CLUSTER_CIDR} 地址段（如 16）必须小于 SubnetLen，必须与 kube-controller-manager 的 --cluster-cidr 参数值一致；

cd opt/k8s/work
source opt/k8s/bin/environment.sh
cat > flanneld.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service


[Service]
Type=notify
ExecStart=/opt/k8s/bin/flanneld \\
  -etcd-cafile=/etc/kubernetes/cert/ca.pem \\
  -etcd-certfile=/etc/flanneld/cert/flanneld.pem \\
  -etcd-keyfile=/etc/flanneld/cert/flanneld-key.pem \\
  -etcd-endpoints=${ETCD_ENDPOINTS} \\
  -etcd-prefix=${FLANNEL_ETCD_PREFIX} \\
  -iface=${IFACE} \\
  -ip-masq
ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d run/flannel/docker
Restart=always
RestartSec=5
StartLimitInterval=0


[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF
***************************************************************
-- mk-docker-opts.sh 脚本将分配给 flanneld 的 Pod 子网段信息写入 run/flannel/docker 文件，后续 docker 启动时使用这个文件中的环境变量配置 docker0 网桥；
-- flanneld 使用系统缺省路由所在的接口与其它节点通信，对于有多个网络接口（如内网和公网）的节点，可以用 -iface 参数指定通信接口;
-- flanneld 运行时需要 root 权限；
-- -ip-masq: flanneld 为访问 Pod 网络外的流量设置 SNAT 规则，同时将传递给 Docker 的变量 --ip-masq（/run/flannel/docker 文件中）设置为 false，这样 Docker 将不再创建 SNAT 规则；Docker 的 --ip-masq 为 true 时，创建的 SNAT 规则比较“暴力”：将所有本节点 Pod 发起的、访问非 docker0 接口的请求做 SNAT，这样访问其他节点 Pod 的请求来源 IP 会被设置为 flannel.1 接口的 IP，导致目的 Pod 看不到真实的来源 Pod IP。flanneld 创建的 SNAT 规则比较温和，只对访问非 Pod 网段的请求做 SNAT。

#下载flannled.service文件
scp flanneld.service root@${node_ip}:/etc/systemd/system/

systemctl daemon-reload 
systemctl enable flanneld 
systemctl restart flanneld
systemctl status flanneld
INFRA [root@wqdcsrv034 system]# systemctl restart flanneld
INFRA [root@wqdcsrv034 system]# systemctl status flanneld
● flanneld.service - Flanneld overlay address etcd agent
   Loaded: loaded (/etc/systemd/system/flanneld.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2019-08-02 10:42:37 CST; 21s ago
  Process: 10399 ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d run/flannel/docker (code=exited, status=0/SUCCESS)
 Main PID: 10371 (flanneld)
   Memory: 11.4M
   CGroup: system.slice/flanneld.service
           └─10371 opt/k8s/bin/flanneld -etcd-cafile=/etc/kubernetes/cert/ca.pem -etcd-certfile=/etc/flanneld/cer...




Aug 02 10:42:37 wqdcsrv034.cn.infra flanneld[10371]: I0802 10:42:37.274645   10371 iptables.go:155] Adding ipt...TURN
Aug 02 10:42:37 wqdcsrv034.cn.infra flanneld[10371]: I0802 10:42:37.276682   10371 iptables.go:155] Adding ipt...RADE
Aug 02 10:42:37 wqdcsrv034.cn.infra systemd[1]: Started Flanneld overlay address etcd agent.
Hint: Some lines were ellipsized, use -l to show in full.

source opt/k8s/bin/environment.sh


etcdctl \
  --endpoints=${ETCD_ENDPOINTS} \
  --ca-file=/etc/kubernetes/cert/ca.pem \
  --cert-file=/etc/flanneld/cert/flanneld.pem \
  --key-file=/etc/flanneld/cert/flanneld-key.pem \
  get ${FLANNEL_ETCD_PREFIX}/config
  
  #输出如下
 {"Network":"172.30.0.0/16", "SubnetLen": 21, "Backend": {"Type": "vxlan"}}

etcdctl \
  --endpoints=${ETCD_ENDPOINTS} \
  --ca-file=/etc/kubernetes/cert/ca.pem \
  --cert-file=/etc/flanneld/cert/flanneld.pem \
  --key-file=/etc/flanneld/cert/flanneld-key.pem \
  ls ${FLANNEL_ETCD_PREFIX}/subnets
  
  #输出如下
/kubernetes/network/subnets/172.30.120.0-21
/kubernetes/network/subnets/172.30.136.0-21
/kubernetes/network/subnets/172.30.176.0-21

etcdctl \
  --endpoints=${ETCD_ENDPOINTS} \
  --ca-file=/etc/kubernetes/cert/ca.pem \
  --cert-file=/etc/flanneld/cert/flanneld.pem \
  --key-file=/etc/flanneld/cert/flanneld-key.pem \
  get ${FLANNEL_ETCD_PREFIX}/subnets/172.30.120.0-21
  
  #输出如下
  {"PublicIP":"10.67.194.43","BackendType":"vxlan","BackendData":{"VtepMAC":"e6:7f:4b:16:c5:04"}}
-- 172.30.120.0/21 被分配给节点 master.kube（10.67.194.43）；
-- VtepMAC 为 master.kube 节点的 flannel.1 网卡 MAC 地址；

ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:50:56:ae:2d:cf brd ff:ff:ff:ff:ff:ff
    inet 10.67.194.42/20 brd 10.67.207.255 scope global eth0
       valid_lft forever preferred_lft forever 
21: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default 
    link/ether 5a:c5:72:1c:f7:89 brd ff:ff:ff:ff:ff:ff
    inet 172.30.136.0/32 scope global flannel.1
       valid_lft forever preferred_lft forever


*******************************************、
-- flannel.1 网卡的地址为分配的 Pod 子网段的第一个 IP（.0），且是 32 的地址；

ip route show |grep flannel.1
172.30.120.0/21 via 172.30.120.0 dev flannel.1 onlink 
172.30.176.0/21 via 172.30.176.0 dev flannel.1 onlink 
************************************************
-- 到其它节点 Pod 网段请求都被转发到 flannel.1 网卡；
-- flanneld 根据 etcd 中子网段的信息，如 ${FLANNEL_ETCD_PREFIX}/subnets/172.30.120.0-21 ，来决定进请求发送给哪个节点的互联 IP；

source opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "/usr/sbin/ip addr show flannel.1|grep -w inet"
  done
  
  #显示如下
  >>> 10.67.194.43
The authenticity of host '10.67.194.43 (10.67.194.43)' can't be establishe
ECDSA key fingerprint is SHA256:RYPS0lG5lc7p1joMPJI9rRrxPBoHWtrWIFBHsCY+1w
ECDSA key fingerprint is MD5:54:de:f4:1b:53:95:ad:d8:ab:50:ad:9e:4a:2f:03:
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '10.67.194.43' (ECDSA) to the list of known hos
root@10.67.194.43's password:   

for node_ip in ${NODE_IPS[@]}
  do
    echo ">>> ${node_ip}"
    ssh ${node_ip} "ping -c 1 172.30.120.0"
    ssh ${node_ip} "ping -c 1 172.30.136.0"
    ssh ${node_ip} "ping -c 1 172.30.176.0"
  done

wget https://dl.k8s.io/v1.15.1/kubernetes-server-linux-amd64.tar.gz 


ll
total 535
-rw-r--r-- 1 root root 443761836 Jul 30 10:43 kubernetes-server-linux-amd64.tar.gz


tar -zxvf kubernetes-server-linux-amd64.tar.gz -C /opt/k8s/work


cd /opt/k8s/work/kubernetes/
ll
total 27179
drwxr-xr-x 2 root root     1024 Aug  2 11:22 addons
drwxr-xr-x 3 root root     1024 Jul 18 17:53 client
-rw-r--r-- 1 root root 26620736 Aug  2 11:22 kubernetes-src.tar.gz
-rw-r--r-- 1 root root  1205293 Aug  2 11:22 LICENSES
drwxr-xr-x 3 root root     1024 Aug  2 11:22 server


tar -xzvf  kubernetes-src.tar.gz
cp kubernetes/server/bin/{apiextensions-apiserver,cloud-controller-manager,kube-apiserver,kube-controller-manager,kube-proxy,kube-scheduler,kubeadm,kubectl,kubelet,mounter} /opt/k8s/bin/
chmod +x /opt/k8s/bin/*

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat > kubernetes-csr.json <<EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "10.67.194.43",
    "10.67.194.42",
    "10.67.194.41",
    "${CLUSTER_KUBERNETES_SVC_IP}",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local."
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF

cfssl gencert -ca=/opt/k8s/work/ca.pem \
  -ca-key=/opt/k8s/work/ca-key.pem \
  -config=/opt/k8s/work/ca-config.json \
  -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
ls kubernetes*pem

mkdir -p /etc/kubernetes/cert
cp kubernetes*.pem   /etc/kubernetes/cert

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat > encryption-config.yaml <<EOF
kind: EncryptionConfig
apiVersion: v1
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: ${ENCRYPTION_KEY}
      - identity: {}
EOF

cp encryption-config.yaml /etc/kubernetes/

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
cat > audit-policy.yaml <<EOF
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
  # The following requests were manually identified as high-volume and low-risk, so drop them.
  - level: None
    resources:
      - group: ""
        resources:
          - endpoints
          - services
          - services/status
    users:
      - 'system:kube-proxy'
    verbs:
      - watch


  - level: None
    resources:
      - group: ""
        resources:
          - nodes
          - nodes/status
    userGroups:
      - 'system:nodes'
    verbs:
      - get


  - level: None
    namespaces:
      - kube-system
    resources:
      - group: ""
        resources:
          - endpoints
    users:
      - 'system:kube-controller-manager'
      - 'system:kube-scheduler'
      - 'system:serviceaccount:kube-system:endpoint-controller'
    verbs:
      - get
      - update


  - level: None
    resources:
      - group: ""
        resources:
          - namespaces
          - namespaces/status
          - namespaces/finalize
    users:
      - 'system:apiserver'
    verbs:
      - get


  # Don't log HPA fetching metrics.
  - level: None
    resources:
      - group: metrics.k8s.io
    users:
      - 'system:kube-controller-manager'
    verbs:
      - get
      - list


  # Don't log these read-only URLs.
  - level: None
    nonResourceURLs:
      - '/healthz*'
      - /version
      - '/swagger*'


  # Don't log events requests.
  - level: None
    resources:
      - group: ""
        resources:
          - events


  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - nodes/status
          - pods/status
    users:
      - kubelet
      - 'system:node-problem-detector'
      - 'system:serviceaccount:kube-system:node-problem-detector'
    verbs:
      - update
      - patch


  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - nodes/status
          - pods/status
    userGroups:
      - 'system:nodes'
    verbs:
      - update
      - patch


  # deletecollection calls can be large, don't log responses for expected namespace deletions
  - level: Request
    omitStages:
      - RequestReceived
    users:
      - 'system:serviceaccount:kube-system:namespace-controller'
    verbs:
      - deletecollection


  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
  # so only log at the Metadata level.
  - level: Metadata
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - secrets
          - configmaps
      - group: authentication.k8s.io
        resources:
          - tokenreviews
  # Get repsonses can be large; skip them.
  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
      - group: admissionregistration.k8s.io
      - group: apiextensions.k8s.io
      - group: apiregistration.k8s.io
      - group: apps
      - group: authentication.k8s.io
      - group: authorization.k8s.io
      - group: autoscaling
      - group: batch
      - group: certificates.k8s.io
      - group: extensions
      - group: metrics.k8s.io
      - group: networking.k8s.io
      - group: policy
      - group: rbac.authorization.k8s.io
      - group: scheduling.k8s.io
      - group: settings.k8s.io
      - group: storage.k8s.io
    verbs:
      - get
      - list
      - watch


  # Default level for known APIs
  - level: RequestResponse
    omitStages:
      - RequestReceived
    resources:
      - group: ""
      - group: admissionregistration.k8s.io
      - group: apiextensions.k8s.io
      - group: apiregistration.k8s.io
      - group: apps
      - group: authentication.k8s.io
      - group: authorization.k8s.io
      - group: autoscaling
      - group: batch
      - group: certificates.k8s.io
      - group: extensions
      - group: metrics.k8s.io
      - group: networking.k8s.io
      - group: policy
      - group: rbac.authorization.k8s.io
      - group: scheduling.k8s.io
      - group: settings.k8s.io
      - group: storage.k8s.io
      
  # Default level for all other requests.
  - level: Metadata
    omitStages:
      - RequestReceived
EOF

 cp audit-policy.yaml /etc/kubernetes/audit-policy.yaml

cat > proxy-client-csr.json <<EOF
{
  "CN": "aggregator",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "4Paradigm"
    }
  ]
}
EOF

cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \
  -ca-key=/etc/kubernetes/cert/ca-key.pem  \
  -config=/etc/kubernetes/cert/ca-config.json  \
  -profile=kubernetes proxy-client-csr.json | cfssljson -bare proxy-client


ls proxy-client*.pem
proxy-client-key.pem  proxy-client.pem

cp proxy-client*.pem /etc/kubernetes/cert/

cd /opt/k8s/work
source /opt/k8s/bin/environment.sh


cat > kube-apiserver.service.template <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target


[Service]
WorkingDirectory=${K8S_DIR}/kube-apiserver
ExecStart=/opt/k8s/bin/kube-apiserver \\
  --advertise-address=##NODE_IP## \\
  --default-not-ready-toleration-seconds=360 \\
  --default-unreachable-toleration-seconds=360 \\
  --feature-gates=DynamicAuditing=true \\
  --max-mutating-requests-inflight=2000 \\
  --max-requests-inflight=4000 \\
  --default-watch-cache-size=200 \\
  --delete-collection-workers=2 \\
  --encryption-provider-config=/etc/kubernetes/encryption-config.yaml \\
  --etcd-cafile=/etc/kubernetes/cert/ca.pem \\
  --etcd-certfile=/etc/kubernetes/cert/kubernetes.pem \\
  --etcd-keyfile=/etc/kubernetes/cert/kubernetes-key.pem \\
  --etcd-servers=${ETCD_ENDPOINTS} \\
  --bind-address=##NODE_IP## \\
  --secure-port=6443 \\
  --tls-cert-file=/etc/kubernetes/cert/kubernetes.pem \\
  --tls-private-key-file=/etc/kubernetes/cert/kubernetes-key.pem \\
  --insecure-port=0 \\
  --audit-dynamic-configuration \\
  --audit-log-maxage=15 \\
  --audit-log-maxbackup=3 \\
  --audit-log-maxsize=100 \\
  --audit-log-truncate-enabled \\
  --audit-log-path=${K8S_DIR}/kube-apiserver/audit.log \\
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
  --profiling \\
  --anonymous-auth=false \\
  --client-ca-file=/etc/kubernetes/cert/ca.pem \\
  --enable-bootstrap-token-auth \\
  --requestheader-allowed-names="aggregator" \\
  --requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\
  --requestheader-extra-headers-prefix="X-Remote-Extra-" \\
  --requestheader-group-headers=X-Remote-Group \\
  --requestheader-username-headers=X-Remote-User \\
  --service-account-key-file=/etc/kubernetes/cert/ca.pem \\
  --authorization-mode=Node,RBAC \\
  --runtime-config=api/all=true \\
  --enable-admission-plugins=NodeRestriction \\
  --enable-aggregator-routing=true \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --event-ttl=168h \\
  --kubelet-certificate-authority=/etc/kubernetes/cert/ca.pem \\
  --kubelet-client-certificate=/etc/kubernetes/cert/kubernetes.pem \\
  --kubelet-client-key=/etc/kubernetes/cert/kubernetes-key.pem \\
  --kubelet-https=true \\
  --kubelet-timeout=10s \\
  --proxy-client-cert-file=/etc/kubernetes/cert/proxy-client.pem \\
  --proxy-client-key-file=/etc/kubernetes/cert/proxy-client-key.pem \\
  --service-cluster-ip-range=${SERVICE_CIDR} \\
  --service-node-port-range=${NODE_PORT_RANGE} \\
  --logtostderr=true \\
  --v=2
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536


[Install]
WantedBy=multi-user.target
EOF

kubectl top nodes
Error from server (Forbidden): nodes.metrics.k8s.io is forbidden: User "aggregator" cannot list resource "nodes" in API group "metrics.k8s.io" at the cluster scope
kube-apiserver systemd unit 文件

替换模板文件中的变量，为各节点生成 systemd unit 文件：
cd /opt/k8s/work
source /opt/k8s/bin/environment.sh
for (( i=0; i < 3; i++ ))
  do
    sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-apiserver.service.template > kube-apiserver-${NODE_IPS[i]}.service 
  done
ls kube-apiserver*.service
//这里是为3master创建的模板，只要自己需要的就行

cp kube-apiserver-10.67.194.43.service /etc/systemd/system/kube-apiserver.service

mkdir -p ${K8S_DIR}/kube-apiserver


systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver


systemctl status kube-apiserver.service 
● kube-apiserver.service - Kubernetes API Server
   Loaded: loaded (/etc/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2019-08-02 12:17:48 CST; 9s ago
     Docs: https://github.com/GoogleCloudPlatform/kubernetes
 Main PID: 8742 (kube-apiserver)
   Memory: 208.6M
   CGroup: /system.slice/kube-apiserver.service
           └─8742 /opt/k8s/bin/kube-apiserver --advertise-address=10.67.194.43 --default-not-ready-toleration-seco...


Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [+]poststarthook/generic-apiserver-start-informers ok
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [+]poststarthook/start-apiextensions-informers ok
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [+]poststarthook/start-apiextensions-controllers ok
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [-]poststarthook/crd-informer-synced failed: reason w...eld
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [+]poststarthook/bootstrap-controller ok
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [-]poststarthook/rbac/bootstrap-roles failed: reason ...eld
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [-]poststarthook/scheduling/bootstrap-system-priority...eld
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [-]poststarthook/ca-registration failed: reason withheld
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [+]poststarthook/start-kube-apiserver-admission-initi... ok
Aug 02 12:17:48 wqdcsrv034.cn.infra kube-apiserver[8742]: [+]poststarthook/start-kube-aggregator-

ETCDCTL_API=3 etcdctl \
    --endpoints=${ETCD_ENDPOINTS} \
    --cacert=/opt/k8s/work/ca.pem \
    --cert=/opt/k8s/work/etcd.pem \
    --key=/opt/k8s/work/etcd-key.pem \
    get /registry/ --prefix --keys-only

7.0.0.1:8443 was refused - did you specify the right host or port?
INFRA [root@wqdcsrv034 .kube]# netstat -lnpt|grep kube
tcp        0      0 10.67.194.43:6443       0.0.0.0:*               LISTEN      30710/kube-apiserve