etcd集群 - tycoon3 - 博客园 (2023)

配置各个节点/etc/hosts

[root@host-10-10-18-42 etcd]# cat /etc/hosts127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4::1 localhost localhost.localdomain localhost6 localhost6.localdomain6etcd1 10.10.18.42etcd2 10.10.18.43etcd3 10.10.18.44
etcd1
10.10.18.42

etcd2

10.10.18.43

etcd3

10.10.18.44

配置目录

 mkdir /data/k8s/etcd/{data,wal} -p chown -R etcd.etcd /data/k8s/etcd

etcd1

[root@host-10-10-18-42 etcd]# cat etcd.confETCD_DATA_DIR="/data/k8s/etcd/data"ETCD_WAL_DIR="/data/k8s/etcd/wal"ETCD_LISTEN_PEER_URLS="http://10.10.18.42:2380"ETCD_LISTEN_CLIENT_URLS="http://10.10.18.42:2379"ETCD_MAX_SNAPSHOTS="5"ETCD_MAX_WALS="5"ETCD_NAME="etcd1"ETCD_SNAPSHOT_COUNT="100000"ETCD_HEARTBEAT_INTERVAL="100"ETCD_ELECTION_TIMEOUT="1000"ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.18.42:2380"ETCD_ADVERTISE_CLIENT_URLS="http://10.10.18.42:2379"ETCD_INITIAL_CLUSTER="etcd1=http://10.10.18.42:2380,etcd2=http://10.10.18.43:2380,etcd3=http://10.10.18.44:2380"ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"ETCD_INITIAL_CLUSTER_STATE="new"

etcd2

[root@host-10-10-18-43 etcd]# cat etcd.conf ETCD_DATA_DIR="/data/k8s/etcd/data"ETCD_WAL_DIR="/data/k8s/etcd/wal"ETCD_LISTEN_PEER_URLS="http://10.10.18.43:2380"ETCD_LISTEN_CLIENT_URLS="http://10.10.18.43:2379"ETCD_MAX_SNAPSHOTS="5"ETCD_MAX_WALS="5"ETCD_NAME="etcd2"ETCD_SNAPSHOT_COUNT="100000"ETCD_HEARTBEAT_INTERVAL="100"ETCD_ELECTION_TIMEOUT="1000"ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.18.43:2380"ETCD_ADVERTISE_CLIENT_URLS="http://10.10.18.43:2379"ETCD_INITIAL_CLUSTER="etcd1=http://10.10.18.42:2380,etcd2=http://10.10.18.43:2380,etcd3=http://10.10.18.44:2380"ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"ETCD_INITIAL_CLUSTER_STATE="new"

etcd3

[root@host-10-10-18-44 etcd]# cat etcd.confETCD_DATA_DIR="/data/k8s/etcd/data"ETCD_WAL_DIR="/data/k8s/etcd/wal"ETCD_LISTEN_PEER_URLS="http://10.10.18.44:2380"ETCD_LISTEN_CLIENT_URLS="http://10.10.18.44:2379"ETCD_MAX_SNAPSHOTS="5"ETCD_MAX_WALS="5"ETCD_NAME="etcd3"ETCD_SNAPSHOT_COUNT="100000"ETCD_HEARTBEAT_INTERVAL="100"ETCD_ELECTION_TIMEOUT="1000"ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.10.18.44:2380"ETCD_ADVERTISE_CLIENT_URLS="http://10.10.18.44:2379"ETCD_INITIAL_CLUSTER="etcd1=http://10.10.18.42:2380,etcd2=http://10.10.18.43:2380,etcd3=http://10.10.18.44:2380"ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"ETCD_INITIAL_CLUSTER_STATE="new"
Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer 1829ea2c82ecd13e could not connect: dial tcp 10.10.18.42:2380: i/o timeout (prober "ROUND_TRIPPER_RAFT_MESSAGE")Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer 1829ea2c82ecd13e could not connect: dial tcp 10.10.18.42:2380: i/o timeout (prober "ROUND_TRIPPER_SNAPSHOT")Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer fe3b541533812c5d could not connect: dial tcp 10.10.18.44:2380: i/o timeout (prober "ROUND_TRIPPER_RAFT_MESSAGE")Jun 29 11:28:03 host-10-10-18-43 etcd[4059]: health check for peer fe3b541533812c5d could not connect: dial tcp 10.10.18.44:2380: i/o timeout (prober "ROUND_TRIPPER_SNAPSHOT")
root@ubuntu:~/bibili# telnet 10.10.18.44 2380Trying 10.10.18.44...telnet: Unable to connect to remote host: No route to hostroot@ubuntu:~/bibili# 

关闭CentOS7防火墙

etcd集群 - tycoon3 - 博客园 (1)

# 查看防火墙状态firewall-cmd --state# 停止firewallsystemctl stop firewalld.service# 禁止firewall开机启动systemctl disable firewalld.service

etcd集群 - tycoon3 - 博客园 (2)

关闭SELINUX

# 编辑SELINUX文件vim /etc/selinux/config# 将SELINUX=enforcing改为SELINUX=disabled
root@ubuntu:~/bibili# telnet 10.10.18.44 2380Trying 10.10.18.44...Connected to 10.10.18.44.Escape character is '^]'.^C^C^CConnection closed by foreign host.
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 etcdctl --endpoints=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 endpoint healthhttp://10.10.18.43:2379 is healthy: successfully committed proposal: took = 2.311413mshttp://10.10.18.42:2379 is healthy: successfully committed proposal: took = 4.239303mshttp://10.10.18.44:2379 is healthy: successfully committed proposal: took = 4.742326msroot@ubuntu:~/etcd-v3.5.0-linux-arm64# 
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 member list1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, falsefe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, false
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 endpoint status+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+| http://10.10.18.42:2379 | 1829ea2c82ecd13e | 3.3.11 | 328 kB | true | false | 339 | 17 | 0 | || http://10.10.18.43:2379 | 19ddebfcb3e299fd | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | || http://10.10.18.44:2379 | fe3b541533812c5d | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | |+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+root@ubuntu:~/etcd-v3.5.0-linux-arm64# 
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ENDPOINTS=http://10.10.18.42:2379,http://10.10.18.43:2379,http://10.10.18.44:2379 root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS endpoint status+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+| http://10.10.18.42:2379 | 1829ea2c82ecd13e | 3.3.11 | 328 kB | true | false | 339 | 17 | 0 | || http://10.10.18.43:2379 | 19ddebfcb3e299fd | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | || http://10.10.18.44:2379 | fe3b541533812c5d | 3.3.11 | 328 kB | false | false | 339 | 17 | 0 | |+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+root@ubuntu:~/etcd-v3.5.0-linux-arm64# 
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS put test "helloworld"OKroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS get testtesthelloworldroot@ubuntu:~/etcd-v3.5.0-linux-arm64# 
[root@host-10-10-18-42 etcd]# tree /data/k8s/etcd//data/k8s/etcd/|-- data| `-- member| `-- snap| `-- db`-- wal |-- 0000000000000000-0000000000000000.wal `-- 0.tmp4 directories, 3 files
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS snapshot save mysnapshot.dbError: snapshot must be requested to one selected node, not multiple [http://10.10.18.42:2379 http://10.10.18.43:2379 http://10.10.18.44:2379]root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=$ENDPOINTS snapshot status mysnapshot.db -w jsonDeprecated: Use `etcdutl snapshot status` instead.Error: stat mysnapshot.db: no such file or directoryroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=http://10.10.18.43:2379 snapshot status mysnapshot.db -w jsonDeprecated: Use `etcdutl snapshot status` instead.Error: stat mysnapshot.db: no such file or directoryroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --endpoints=http://10.10.18.43:2379 snapshot save mysnapshot.db{"level":"info","ts":1624938894.0369105,"caller":"snapshot/v3_snapshot.go:68","msg":"created temporary db file","path":"mysnapshot.db.part"}{"level":"info","ts":1624938894.0386374,"logger":"client","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"}{"level":"info","ts":1624938894.0386932,"caller":"snapshot/v3_snapshot.go:76","msg":"fetching snapshot","endpoint":"http://10.10.18.43:2379"}{"level":"info","ts":1624938894.0599878,"logger":"client","caller":"v3/maintenance.go:219","msg":"completed snapshot read; closing"}{"level":"info","ts":1624938894.0606616,"caller":"snapshot/v3_snapshot.go:91","msg":"fetched snapshot","endpoint":"http://10.10.18.43:2379","size":"328 kB","took":"now"}{"level":"info","ts":1624938894.0607412,"caller":"snapshot/v3_snapshot.go:100","msg":"saved","path":"mysnapshot.db"}Snapshot saved at mysnapshot.dbroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl snapshot status mysnapshot.db -w jsonDeprecated: Use `etcdutl snapshot status` instead.{"hash":3787458990,"revision":2,"totalKey":7,"totalSize":327680}root@ubuntu:~/etcd-v3.5.0-linux-arm64# 
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \ --cacert=<trusted-ca-file> --cert=<cert-file> --key=<key-file> \
 --cacert="" verify certificates of TLS-enabled secure servers using this CA bundle --cert="" identify secure client using this TLS certificate file --key="" identify secure client using this TLS key file --endpoints=[127.0.0.1:2379] gRPC endpoints
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ls /etc/kubernetes/pki/etcd/ca.crt ca.key healthcheck-client.crt healthcheck-client.key peer.crt peer.key server.crt server.keyroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=$ENDPOINTS member list1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, falsefe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, falseroot@ubuntu:~/etcd-v3.5.0-linux-arm64# 

TLS 认证文件

需要为 etcd 集群创建加密通信的 TLS 证书,这里复用以前创建的 kubernetes 证书

root@ubuntu:/etc# ps -elf | grep etcd4 S root 7969 7939 2 80 0 - 2672731 futex_ Jun18 ? 05:35:58 etcd --advertise-client-urls=https://10.10.16.82:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://10.10.16.82:2380 --initial-cluster=ubuntu=https://10.10.16.82:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.10.16.82:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.10.16.82:2380 --name=ubuntu --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
[root@host-10-10-18-42 system]# curl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.10.16.82:2379/version{"etcdserver":"3.4.3","etcdcluster":"3.4.0"}
[root@host-10-10-18-42 system]# ls /etc/kubernetes/pki/etcd/ca.crt ca.key healthcheck-client.crt healthcheck-client.key peer.crt peer.key server.crt server.key[root@host-10-10-18-42 system]# 
[root@host-10-10-18-42 etc]# systemctl status etcd.service -l● etcd.service - Etcd Server Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled) Active: active (running) since Tue 2021-06-29 11:36:30 CST; 3h 11min ago Main PID: 4306 (etcd) CGroup: /system.slice/etcd.service └─4306 /usr/bin/etcd --name=etcd1 --data-dir=/data/k8s/etcd/data --listen-client-urls=http://10.10.18.42:2379Jun 29 12:10:56 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 31.49688ms)Jun 29 12:10:56 host-10-10-18-42 etcd[4306]: server is likely overloadedJun 29 12:38:51 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 36.45658ms)Jun 29 12:38:51 host-10-10-18-42 etcd[4306]: server is likely overloadedJun 29 12:38:51 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 217.69444ms)Jun 29 12:38:51 host-10-10-18-42 etcd[4306]: server is likely overloadedJun 29 14:00:07 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 5.25766ms)Jun 29 14:00:07 host-10-10-18-42 etcd[4306]: server is likely overloadedJun 29 14:00:07 host-10-10-18-42 etcd[4306]: failed to send out heartbeat on time (exceeded the 100ms timeout for 33.283ms)Jun 29 14:00:07 host-10-10-18-42 etcd[4306]: server is likely overloaded

the server is already initialized as member before, starting as etcd member

Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: the server is already initialized as member before, starting as etcd member...Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: peerTLS: cert = /etc/kubernetes/pki/etcd/peer.crt, key = /etc/kubernetes/pki/etcd/peer.key, ca = , trusted-ca = /etc/kubernetes/pki/etcd/ca.crt, client-cert-auth = true, crl-file = Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of peer url http://10.10.18.42:2380 is HTTP while peer key/cert files are presented. Ignored peer key/cert files.Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of peer url http://10.10.18.42:2380 is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: listening for peers on http://10.10.18.42:2380Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of client url http://10.10.18.42:2379 is HTTP while peer key/cert files are presented. Ignored key/cert files.Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: The scheme of client url http://10.10.18.42:2379 is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: listening for client requests on 10.10.18.42:2379Jun 29 15:46:01 host-10-10-18-42 etcd[18666]: open /etc/kubernetes/pki/etcd/peer.key: permission deniedJun 29 15:46:01 host-10-10-18-42 systemd[1]: etcd.service: main process exited, code=exited, status=1/FAILUREJun 29 15:46:01 host-10-10-18-42 systemd[1]: Failed to start Etcd Server.
[root@host-10-10-18-42 system]# chown -R etcd.etcd /etc/kubernetes/pki/etcd

/usr/lib/systemd/system/etcd.service

添加秘钥

[root@host-10-10-18-42 system]# cat etcd.service[Unit]Description=Etcd ServerAfter=network.targetAfter=network-online.targetWants=network-online.target[Service]Type=notifyWorkingDirectory=/var/lib/etcd/EnvironmentFile=-/etc/etcd/etcd.confUser=etcd# set GOMAXPROCS to number of processorsExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\" --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --key-file=/etc/kubernetes/pki/etcd/server.key --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true"Restart=on-failureLimitNOFILE=65536[Install]WantedBy=multi-user.target
[root@host-10-10-18-42 system]# systemctl daemon-reload[root@host-10-10-18-42 system]# systemctl restart etcd[root@host-10-10-18-42 system]# 
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS endpoint status+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+| http://10.10.18.42:2379 | 1829ea2c82ecd13e | 3.3.11 | 262 kB | false | false | 270 | 13 | 0 | || http://10.10.18.43:2379 | 19ddebfcb3e299fd | 3.3.11 | 262 kB | false | false | 270 | 13 | 0 | || http://10.10.18.44:2379 | fe3b541533812c5d | 3.3.11 | 262 kB | true | false | 270 | 13 | 0 | |+-------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+root@ubuntu:~/etcd-v3.5.0-linux-arm64# 
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS endpoint health+-------------------------+--------+-------------+-------+| ENDPOINT | HEALTH | TOOK | ERROR |+-------------------------+--------+-------------+-------+| http://10.10.18.44:2379 | true | 14.214315ms | || http://10.10.18.43:2379 | true | 17.301696ms | || http://10.10.18.42:2379 | true | 14.207596ms | |+-------------------------+--------+-------------+-------+root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl -w table --endpoints=$ENDPOINTS member list+------------------+---------+-------+-------------------------+-------------------------+------------+| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER |+------------------+---------+-------+-------------------------+-------------------------+------------+| 1829ea2c82ecd13e | started | etcd1 | http://10.10.18.42:2380 | http://10.10.18.42:2379 | false || 19ddebfcb3e299fd | started | etcd2 | http://10.10.18.43:2380 | http://10.10.18.43:2379 | false || fe3b541533812c5d | started | etcd3 | http://10.10.18.44:2380 | http://10.10.18.44:2379 | false |+------------------+---------+-------+-------------------------+-------------------------+------------+root@ubuntu:~/etcd-v3.5.0-linux-arm64# 
To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/configYou should now deploy a pod network to the cluster.Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/Then you can join any number of worker nodes by running the following on each as root:kubeadm join 10.10.18.46:6443 --token pbje64.ffl4ms0ymvjhwu52 \ --discovery-token-ca-cert-hash sha256:037f81a4c3dab193f50af44af460032172f7b8a700109c9ebebcc731728b165f [root@host-10-10-18-46 ~]# mkdir -p $HOME/.kube[root@host-10-10-18-46 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config[root@host-10-10-18-46 ~]# chown $(id -u):$(id -g) $HOME/.kube/config[root@host-10-10-18-46 ~]# kubeadm config print init-defaults > kubeadm-init.yaml.yamlW0629 17:29:28.721072 20178 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io][root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# lsanaconda-ks.cfg k8s.init kubeadm-init.yaml.yaml[root@host-10-10-18-46 ~]# kubeadm init --config=kubeadm-init.yaml.yamlW0629 17:46:59.845088 14974 strict.go:54] error unmarshaling configuration schema.GroupVersionKind{Group:"kubeadm.k8s.io", Version:"v1beta2", Kind:"ClusterConfiguration"}: error converting YAML to JSON: yaml: unmarshal errors: line 15: key "imageRepository" already set in map line 18: key "apiServer" already set in map line 24: key "etcd" already set in mapW0629 17:46:59.847076 14974 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io][init] Using Kubernetes version: v1.18.0[preflight] Running pre-flight checks [WARNING Service-Docker]: docker service is not enabled, please run 'systemctl enable docker.service' [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/ [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.7. Latest validated version: 19.03error execution phase preflight: [preflight] Some fatal errors occurred: [ERROR Port-6443]: Port 6443 is in use [ERROR Port-10259]: Port 10259 is in use [ERROR Port-10257]: Port 10257 is in use [ERROR FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml]: /etc/kubernetes/manifests/kube-apiserver.yaml already exists [ERROR FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml]: /etc/kubernetes/manifests/kube-controller-manager.yaml already exists [ERROR FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml]: /etc/kubernetes/manifests/kube-scheduler.yaml already exists [ERROR FileAvailable--etc-kubernetes-manifests-etcd.yaml]: /etc/kubernetes/manifests/etcd.yaml already exists [ERROR Port-10250]: Port 10250 is in use [ERROR ExternalEtcdVersion]: Get https://10.10.18.42:2379/version: EOF [ERROR ExternalEtcdVersion]: Get https://10.10.18.43:2379/version: EOF [ERROR ExternalEtcdVersion]: Get https://10.10.18.44:2379/version: EOF[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`To see the stack trace of this error execute with --v=5 or higher
[root@host-10-10-18-46 ~]# vi kubeadm-init.yaml.yaml apiVersion: kubeadm.k8s.io/v1beta2bootstrapTokens:- groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authenticationkind: InitConfigurationlocalAPIEndpoint: advertiseAddress: 1.2.3.4 bindPort: 6443nodeRegistration: criSocket: /var/run/dockershim.sock name: host-10-10-18-46 taints: - effect: NoSchedule key: node-role.kubernetes.io/master---apiServer: timeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta2certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrollerManager: {}dns: type: CoreDNSetcd: local: dataDir: /var/lib/etcdimageRepository: k8s.gcr.iokind: ClusterConfigurationkubernetesVersion: v1.18.0imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containerscontrolPlaneEndpoint: 10.103.22.236:8443apiServer: certSANs: - 10.10.18.45 - 10.10.18.46 - 10.10.16.249 - 127.0.0.1etcd: external: endpoints: - https://10.10.18.42:2379 - https://10.10.18.43:2379 - https://10.10.18.44:2379 caFile: /etc/kubernetes/pki/etcd_bak/ca.crt certFile: /etc/kubernetes/pki/etcd_bak/server.crt keyFile: /etc/kubernetes/pki/etcd_bak/server.keynetworking: dnsDomain: cluster.local serviceSubnet: 10.96.0.0/12scheduler: {}
[root@host-10-10-18-46 ~]# curl --cacert /etc/kubernetes/pki/etcd_bak/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key -L https://10.10.18.44:2379/versioncurl: (35) Encountered end of file
[root@host-10-10-18-46 ~]# curl --cacert /etc/kubernetes/pki/etcd_bak/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key -L https://10.10.18.44:2379/version -v* About to connect() to 10.10.18.44 port 2379 (#0)* Trying 10.10.18.44...* Connected to 10.10.18.44 (10.10.18.44) port 2379 (#0)* Initializing NSS with certpath: sql:/etc/pki/nssdb* CAfile: /etc/kubernetes/pki/etcd_bak/ca.crt CApath: none* NSS error -5961 (PR_CONNECT_RESET_ERROR)* TCP connection reset by peer* Closing connection 0curl: (35) TCP connection reset by peer

http没问题

[root@host-10-10-18-46 ~]# curl --cacert /etc/kubernetes/pki/etcd_bak/ca.crt --cert /etc/kubernetes/pki/etcd/peer.crt --key /etc/kubernetes/pki/etcd/peer.key -L http://10.10.18.44:2379/version {"etcdserver":"3.3.11","etcdcluster":"3.3.0"}
[root@host-10-10-18-46 ~]#
curl http://10.10.18.44:2379/version {"etcdserver":"3.3.11","etcdcluster":"3.3.0"}
[root@host-10-10-18-46 ~]#
root@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=http://10.10.18.42:2379 member list1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, falsefe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, falseroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=http://10.10.18.43:2379 member list1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, falsefe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, falseroot@ubuntu:~/etcd-v3.5.0-linux-arm64# ETCDCTL_API=3 ./etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --endpoints=http://10.10.18.44:2379 member list1829ea2c82ecd13e, started, etcd1, http://10.10.18.42:2380, http://10.10.18.42:2379, false19ddebfcb3e299fd, started, etcd2, http://10.10.18.43:2380, http://10.10.18.43:2379, falsefe3b541533812c5d, started, etcd3, http://10.10.18.44:2380, http://10.10.18.44:2379, falseroot@ubuntu:~/etcd-v3.5.0-linux-arm64# 

可以访问10.10.16.82:2379

[root@host-10-10-18-42 system]# curl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.10.18.42:2379/versioncurl: (35) Encountered end of file[root@host-10-10-18-42 system]# curl --cacert /etc/kubernetes/pki/etcd/ca.crt --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.10.16.82:2379/version{"etcdserver":"3.4.3","etcdcluster":"3.4.0"}
[root@host-10-10-18-42 system]#
root@ubuntu:/etc# ps -elf | grep etcd | grep client-cert-auth4 S root 7969 7939 2 80 0 - 2672731 futex_ Jun18 ? 05:44:28 etcd --advertise-client-urls=https://10.10.16.82:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://10.10.16.82:2380 --initial-cluster=ubuntu=https://10.10.16.82:2380 --key-file=/etc/kubernetes/pki/etcd/server.key --listen-client-urls=https://127.0.0.1:2379,https://10.10.16.82:2379 --listen-metrics-urls=http://127.0.0.1:2381 --listen-peer-urls=https://10.10.16.82:2380 --name=ubuntu --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt --peer-client-cert-auth=true --peer-key-file=/etc/kubernetes/pki/etcd/peer.key --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt --snapshot-count=10000 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

原来是没有开放https

创建基于根证书的config配置文件

#ca办法证书机构cat > ca-config.json <<EOF{ "signing": { "default": { "expiry": "87600h" }, "profiles": { "www": { "expiry": "87600h", "usages": [ "signing", "key encipherment", "server auth", "client auth" ] } } }}EOF#ca机构请求cat > ca-csr.json <<EOF{ "CN": "etcd CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "Beijing", "ST": "Beijing" } ]}EOF#生成证书:读取上边两个文件生成证书cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
root@ubuntu:~/cfssl/etcd# cat > ca-config.json <<EOF> {> "signing": { "default": { "expiry": "87600h" #证书过期时间h单位 }, "profiles": { "www": { "expiry": "87600h", "usages": [ "signing", "key encipherment", "server auth", "client auth" ] } }> }> }> EOFroot@ubuntu:~/cfssl/etcd# cat > ca-csr.json <<EOF> { "CN": "etcd CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "Beijing", "ST": "Beijing" } ]> }> EOF root@ubuntu:~/cfssl/etcd# ./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca -2021/06/30 10:43:36 [INFO] generating a new CA key and certificate from CSR2021/06/30 10:43:36 [INFO] generate received request2021/06/30 10:43:36 [INFO] received CSR2021/06/30 10:43:36 [INFO] generating key: rsa-20482021/06/30 10:43:37 [INFO] encoded CSR2021/06/30 10:43:37 [INFO] signed certificate with serial number 53627328402430641884101375169327098053785759268root@ubuntu:~/cfssl/etcd# 

创建生成etcd自签证书peer的csr的json配置文件

cat > server-csr.json <<EOF{ "CN": "etcd", "hosts": [ "10.10.18.42", "10.10.18.43", "10.10.18.44" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "L": "BeiJing", "ST": "BeiJing" } ]}EOF
root@ubuntu:~/cfssl/etcd# ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | ./cfssljson -bare server2021/06/30 10:46:48 [INFO] generate received request2021/06/30 10:46:48 [INFO] received CSR2021/06/30 10:46:48 [INFO] generating key: rsa-20482021/06/30 10:46:49 [INFO] encoded CSR2021/06/30 10:46:49 [INFO] signed certificate with serial number 277831989248432604565440323258702823212559696597
cat <<EOF >/usr/lib/systemd/system/etcd.service[Unit]Description=Etcd ServerAfter=network.targetAfter=network-online.targetWants=network-online.target[Service]Type=notifyEnvironmentFile=/etc/etcd/etcd.confExecStart=/usr/bin/etcd \--name=etcd01 \--data-dir=\${ETCD_DATA_DIR} \--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \--initial-cluster=\${ETCD_INITIAL_CLUSTER} \--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \--initial-cluster-state=new \--cert-file=/opt/etcd/ssl/server.pem \--key-file=/opt/etcd/ssl/server-key.pem \--peer-cert-file=/opt/etcd/ssl/server.pem \--peer-key-file=/opt/etcd/ssl/server-key.pem \--trusted-ca-file=/opt/etcd/ssl/ca.pem \--peer-trusted-ca-file=/opt/etcd/ssl/ca.pemRestart=on-failureLimitNOFILE=65536[Install]WantedBy=multi-user.targetEOF
[root@host-10-10-18-43 ~]# systemctl restart etcdJob for etcd.service failed because a timeout was exceeded. See "systemctl status etcd.service" and "journalctl -xe" for details.[root@host-10-10-18-43 ~]# journalctl -xeJun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57514" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44110" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44112" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd is starting a new election at term 33312Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd became candidate at term 33313Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd received MsgVoteResp from 19ddebfcb3e299fd at term 33313Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd [logterm: 275, index: 25] sent MsgVote request to 1829ea2c82ecd13e at term 33313Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: 19ddebfcb3e299fd [logterm: 275, index: 25] sent MsgVote request to fe3b541533812c5d at term 33313Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57526" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44120" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44122" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57524" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57536" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57538" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44130" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:58 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44128" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44140" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57548" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44138" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57546" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57558" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44148" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44150" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57556" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44158" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44156" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57566" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57568" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44166" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44164" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57580" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:05:59 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57578" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57620" (error "tls: oversized record received with length 21536", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44172" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44174" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57590" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57592" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57646" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44208" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44210" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57644" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57658" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44218" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.44:44220" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57656" (error "tls: first record does not look like a TLS handshake", ServerName "")Jun 30 11:06:00 host-10-10-18-43 etcd[20044]: rejected connection from "10.10.18.42:57666" (error "tls: first record does not look like a TLS handshake", ServerName "")

删除旧的

[root@host-10-10-18-43 ~]# rm /data/k8s/etcd/wal/* -rf[root@host-10-10-18-43 ~]# rm /data/k8s/etcd/data/* -rf
[root@host-10-10-18-43 ~]# systemctl status etcd● etcd.service - Etcd Server Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled) Active: active (running) since Wed 2021-06-30 11:14:58 CST; 1min 13s ago Main PID: 20226 (etcd) CGroup: /system.slice/etcd.service └─20226 /usr/bin/etcd --name=etcd2 --data-dir=/data/k8s/etcd/data

访问成功

[root@host-10-10-18-46 ~]# curl --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/server.pem --key /opt/etcd/ssl/server-key.pem https://10.10.18.42:2379/version{"etcdserver":"3.3.11","etcdcluster":"3.3.0"}[root@host-10-10-18-46 ~]# 

init失败查看kubelet

Jun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.112133 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.212346 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.312579 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.412767 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.512983 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.613160 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.713375 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.813574 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:14 host-10-10-18-46 kubelet[25210]: E0630 11:32:14.913774 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.013968 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.114144 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.214331 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.314539 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.414737 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.514889 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.615078 25210 kubelet.go:2267] node "host-10-10-18-46" not foundJun 30 11:32:15 host-10-10-18-46 kubelet[25210]: E0630 11:32:15.715240 25210 kubelet.go:2267] node "host-10-10-18-46" not found
[root@host-10-10-18-46 ~]# cat /etc/kubernetes/kubelet.confapiVersion: v1clusters:- cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EWXpNREF6TWpNek5Gb1hEVE14TURZeU9EQXpNak16TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTUt0CndHbXB2bTBPaFFPMjVBeFpJM295Z2FRS0VGSk81c0JSMmVoem1yN2ZVNlBaWWhrb1BoTXZEZ3RCNmdnNlBjQkcKNFB3M2JlQnNBZXZGaThkNEJ0bFVLeTdJVTFrZHdtcldMTHZBT3lRVnJveExSQ0V0QUVMNWlyUENYQmFjZVBTbwpRV3lnRUFYTEQvTkNOb0NndDF1a3RYSEVHNTlWdG1RbmtiSitnVGNpK1FrWnl5MGRQOWUyOE83SjRIcUhUNHo5CkVRNTlUamprdWVid2VaUmF6WVFYQTV1TWZHY2tJK05VQytCank0NHhQYnNTL2FRSnJPM1c2NzQydTJtdXFXblEKUmZBRHJLOGhMODRVWW4vL1ZReWM4cjFNWENEVXRBd0gyU3dROE1EZTJFM3VURGNyU29HSWx4RXJvelR3Y3ZCNgoweDQwVXAwSEhXZ0NQOVF4Ulk4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFKQS9NVVhVVVU0K0ZmVFExaHROQ2JHeUF2SjMKZjZLOHAzUHZEdnQvR2xwWXFMZkRhaFltWHh3ZEsyaFNVMGliRFZkMW1vem0xL3dmenYzbTl2Z1dKR09rTFhVOQpoSlZkNWVGM0IyNWRkWGdhOGVvVVFJdWNMV2t3QklrSmtITnpiRUJ5UmVlTEV4WUZKN205bXFKa1Z4SVN6Rm1FClN6MG96cXRMQUtHaWZSTnhUbXQvbjQ3RjJma2psNmlYRDlpOGx5WmNyOUxVZklIcTVldFYvYmNRbWdOQ01yZXcKeGZ5R3h1YVgxZ2NQT2JITmVQMUUxcXljOHI5dWU3RWFzSFlhaTY4REFTVWxFalJJMXRxUDgwYkZRSHQzRU5xaAp0ckFKdzIzVzhEcTRibHlDdld1YTBCanB0SG1pWFY0UENVZ1dvK3VsUEVWWXVibzlXbC9jUnZxOENMWT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= server: https://10.103.22.236:8443 name: kubernetes
[root@host-10-10-18-46 ~]# cat kubeadm-init.yaml.yamlapiVersion: kubeadm.k8s.io/v1beta2bootstrapTokens:- groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authenticationkind: InitConfigurationlocalAPIEndpoint: advertiseAddress: 1.2.3.4 bindPort: 6443nodeRegistration: criSocket: /var/run/dockershim.sock name: host-10-10-18-46 taints: - effect: NoSchedule key: node-role.kubernetes.io/master---apiServer: timeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta2certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrollerManager: {}dns: type: CoreDNSetcd: local: dataDir: /var/lib/etcdimageRepository: k8s.gcr.iokind: ClusterConfigurationkubernetesVersion: v1.18.0imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containerscontrolPlaneEndpoint: 10.103.22.236:8443
[root@host-10-10-18-46 ~]# netstat -pan | grep 6443tcp 0 1 10.10.18.46:45042 1.2.3.4:6443 SYN_SENT 25210/kubelet tcp 0 1 10.10.18.46:45024 1.2.3.4:6443 SYN_SENT 25210/kubelet tcp6 0 0 :::6443 :::* LISTEN 27229/kube-apiserve tcp6 0 0 ::1:6443 ::1:55698 ESTABLISHED 27229/kube-apiserve tcp6 0 0 ::1:55698 ::1:6443 ESTABLISHED 27229/kube-apiserve unix 3 [ ] STREAM CONNECTED 36443 1/systemd /run/systemd/journal/stdout[root@host-10-10-18-46 ~]# 

另外一个节点

[root@host-10-10-18-45 ~]# kubeadm versionkubeadm version: &version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.1", GitCommit:"7879fc12a63337efff607952a323df90cdc7a335", GitTreeState:"clean", BuildDate:"2020-04-08T17:36:32Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/arm64"}[root@host-10-10-18-45 ~]# netstat -pan | grep 6443[root@host-10-10-18-45 ~]# 

配置keepalived + haproxy

#---------------------------------------------------------------------# Example configuration for a possible web application. See the# full configuration options online.## https://www.haproxy.org/download/2.1/doc/configuration.txt# https://cbonte.github.io/haproxy-dconv/2.1/configuration.html##---------------------------------------------------------------------#---------------------------------------------------------------------# Global settings#---------------------------------------------------------------------global # to have these messages end up in /var/log/haproxy.log you will # need to: # # 1) configure syslog to accept network log events. This is done # by adding the '-r' option to the SYSLOGD_OPTIONS in # /etc/sysconfig/syslog # # 2) configure local2 events to go to the /var/log/haproxy.log # file. A line like the following can be added to # /etc/sysconfig/syslog # # local2.* /var/log/haproxy.log # log 127.0.0.1 local2# chroot /var/lib/haproxy pidfile /var/run/haproxy.pid maxconn 4000# user haproxy# group haproxy # daemon # turn on stats unix socket stats socket /var/lib/haproxy/stats#---------------------------------------------------------------------# common defaults that all the 'listen' and 'backend' sections will# use if not designated in their block#---------------------------------------------------------------------defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 1m timeout server 1m timeout http-keep-alive 10s timeout check 10s maxconn 3000#---------------------------------------------------------------------# main frontend which proxys to the backends#---------------------------------------------------------------------frontend frr mode tcp bind *:9443 ## 监听9443端口 # bind *:443 ssl # To be completed .... acl url_static path_beg -i /static /images /javascript /stylesheets acl url_static path_end -i .jpg .gif .png .css .js default_backend kube-apiserver #---------------------------------------------------------------------# round robin balancing between the various backends#---------------------------------------------------------------------backend kube-apiserver mode tcp option tcplog option tcp-check balance roundrobin default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100 server host-10-10-18-46 10.10.18.46:6443 check

暂时只配置一个server

keepavlivede vip

root@ubuntu:/etc/haproxy# cat ../keepalived/keepalived.confglobal_defs { script_user root enable_script_security } vrrp_script chk_haproxy { script "/bin/bash -c 'if [[ $(netstat -nlp | grep 9443 | wc -l) ]]; then exit 0; else exit 1; fi'" # haproxy 检测 interval 2 # 每2秒执行一次检测 #weight -10 # 权重变化} vrrp_instance VI_1 { interface enahisic2i0 ###宿主机网卡名 state BACKUP virtual_router_id 61 # id设为相同,表示是同一个虚拟路由组 priority 80 #初始权重 nopreempt #不抢占 unicast_peer { 10.10.16.47 10.10.16.251 } virtual_ipaddress { 10.10.16.249 # vip } authentication { auth_type PASS auth_pass password } track_script { chk_haproxy } #notify "/container/service/keepalived/assets/"}

访问keepalived vip

root@ubuntu:/etc/haproxy# telnet 10.10.16.249 9443Trying 10.10.16.249...Connected to 10.10.16.249.Escape character is '^]'.^C^CConnection closed by foreign host.
[root@host-10-10-18-46 ~]# kubeadm init --config kubeadm-init.yaml.yaml W0630 12:02:37.304175 1295 strict.go:54] error unmarshaling configuration schema.GroupVersionKind{Group:"kubeadm.k8s.io", Version:"v1beta2", Kind:"ClusterConfiguration"}: error converting YAML to JSON: yaml: unmarshal errors: line 15: key "imageRepository" already set in map line 18: key "apiServer" already set in map line 24: key "etcd" already set in map
Your Kubernetes control-plane has initialized successfully!To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/configYou should now deploy a pod network to the cluster.Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/You can now join any number of control-plane nodes by copying certificate authoritiesand service account keys on each node and then running the following as root: kubeadm join 10.10.16.249:9443 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:9bc0bcddb2b97791717943b714ffa410cb5963061889086f04 \ --control-plane Then you can join any number of worker nodes by running the following on each as root:kubeadm join 10.10.16.249:9443 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:9bc0bcddb2b97791717943b714ffa410cb5963061889 

kubeadm-init.yaml

[root@host-10-10-18-46 ~]# cat kubeadm-init.yaml.yamlapiVersion: kubeadm.k8s.io/v1beta2bootstrapTokens:- groups: - system:bootstrappers:kubeadm:default-node-token token: abcdef.0123456789abcdef ttl: 24h0m0s usages: - signing - authenticationkind: InitConfigurationlocalAPIEndpoint: advertiseAddress: 1.2.3.4 --默认 bindPort: 6443nodeRegistration: criSocket: /var/run/dockershim.sock name: host-10-10-18-46 taints: - effect: NoSchedule key: node-role.kubernetes.io/master---apiServer: timeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta2certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrollerManager: {}dns: type: CoreDNSetcd: local: dataDir: /var/lib/etcdimageRepository: k8s.gcr.iokind: ClusterConfigurationkubernetesVersion: v1.18.0imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containerscontrolPlaneEndpoint: 10.10.16.249:9443 apiServer: certSANs: - 10.10.18.45 - 10.10.18.46  - 10.10.16.249 - 127.0.0.1etcd: external: endpoints: - https://10.10.18.42:2379 - https://10.10.18.43:2379 - https://10.10.18.44:2379 caFile: /opt/etcd/ssl/ca.pem certFile: /opt/etcd/ssl/server.pem keyFile: /opt/etcd/ssl/server-key.pem networking: dnsDomain: cluster.local serviceSubnet: 10.96.0.0/12scheduler: {}
[root@host-10-10-18-46 ~]# ps -elf | grep apiserver0 S root 31133 21547 0 80 0 - 1724 pipe_w 14:18 pts/0 00:00:00 grep --color=auto apiserver[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wideThe connection to the server 10.10.18.46:6443 was refused - did you specify the right host or port?[root@host-10-10-18-46 ~]# netstat -pan | grep 6443unix 3 [ ] STREAM CONNECTED 36443 1/systemd /run/systemd/journal/stdout[root@host-10-10-18-46 ~]# ps -elf | grep apiserver0 S root 31196 21547 0 80 0 - 1724 pipe_w 14:18 pts/0 00:00:00 grep --color=auto apiserver[root@host-10-10-18-46 ~]# 

重启kubelet

[root@host-10-10-18-46 ~]# ps -elf | grep apiserver4 S root 31884 31863 29 80 0 - 7681 futex_ 14:19 ? 00:00:13 kube-apiserver --advertise-address=1.2.3.4 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/opt/etcd/ssl/ca.pem --etcd-certfile=/opt/etcd/ssl/server.pem --etcd-keyfile=/opt/etcd/ssl/server-key.pem --etcd-servers=https://10.10.18.42:2379,https://10.10.18.43:2379,https://10.10.18.44:2379 --insecure-port=0 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key0 S root 32191 21547 0 80 0 - 1724 pipe_w 14:20 pts/0 00:00:00 grep --color=auto apiserver[root@host-10-10-18-46 ~]# netstat -pan | grep 6443tcp 0 1 10.10.18.46:48926 1.2.3.4:6443 SYN_SENT 31315/kubelet tcp 0 1 10.10.18.46:48936 1.2.3.4:6443 SYN_SENT 31315/kubelet tcp6 0 0 :::6443 :::* LISTEN 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42914 ESTABLISHED 31884/kube-apiserve tcp6 0 0 ::1:6443 ::1:59596 ESTABLISHED 31884/kube-apiserve tcp6 0 0 ::1:59596 ::1:6443 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42906 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42930 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42966 ESTABLISHED 31884/kube-apiserve tcp6 0 0 10.10.18.46:6443 10.10.16.82:42900 ESTABLISHED 31884/kube-apiserve 

配置执行 kubectl 命令用户

[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wideUnable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes")[root@host-10-10-18-46 ~]# mkdir -p $HOME/.kube[root@host-10-10-18-46 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/configcp: overwrite ‘/root/.kube/config’? y [root@host-10-10-18-46 ~]# chown $(id -u):$(id -g) $HOME/.kube/config[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATEScoredns-546565776c-ch9n7 0/1 Pending 0 135m <none> <none> <none> <none>coredns-546565776c-dddl9 0/1 Pending 0 135m <none> <none> <none> <none>kube-apiserver-host-10-10-18-46 1/1 Running 34 139m 10.10.18.46 host-10-10-18-46 <none> <none>kube-controller-manager-host-10-10-18-46 1/1 Running 25 139m 10.10.18.46 host-10-10-18-46 <none> <none>kube-proxy-zl8fw 1/1 Running 0 135m 10.10.18.46 host-10-10-18-46 <none> <none>kube-scheduler-host-10-10-18-46 1/1 Running 25 139m 10.10.18.46 host-10-10-18-46 <none> <none>[root@host-10-10-18-46 ~]# 

查看集群状态

[root@host-10-10-18-46 ~]# kubectl get csNAME STATUS MESSAGE ERRORscheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused controller-manager Healthy ok etcd-1 Healthy {"health":"true"} etcd-2 Healthy {"health":"true"} etcd-0 Healthy {"health":"true"} [root@host-10-10-18-46 ~]# 
root@ubuntu:~/cfssl/etcd# ETCDCTL_API=3 ./etcdctl --cacert ./etcd/ca.pem --cert ./etcd/server.pem --key ./etcd/server-key.pem --endpoints=https://10.10.18.42:2379 get / --prefix --keys-only | more/registry/apiregistration.k8s.io/apiservices/v1./registry/apiregistration.k8s.io/apiservices/v1.admissionregistration.k8s.io/registry/apiregistration.k8s.io/apiservices/v1.apiextensions.k8s.io/registry/apiregistration.k8s.io/apiservices/v1.apps/registry/apiregistration.k8s.io/apiservices/v1.authentication.k8s.io/registry/apiregistration.k8s.io/apiservices/v1.authorization.k8s.io/registry/apiregistration.k8s.io/apiservices/v1.autoscaling/registry/apiregistration.k8s.io/apiservices/v1.batch
root@ubuntu:~/cfssl/etcd# ETCDCTL_API=3 ./etcdctl --cacert ./etcd/ca.pem --cert ./etcd/server.pem --key ./etcd/server-key.pem --endpoints=https://10.10.18.42:2379 get /registry/clusterrolebindings/kubeadm:get-nodes/registry/clusterrolebindings/kubeadm:get-nodesk8s2rbac.authorization.k8s.io/v1ClusterRoleBindingkubeadm:get-nodes"*$a0766228-3694-4906-9787-b2ca2b181b7b2z kubeadmUpdaterbac.authorization.k8s.io/vFieldsV1:IG{"f:roleRef":{"f:apiGroup":{},"f:kind":{},"f:name":{}},"f:subjects":{}}UGrouprbac.authorization.k8s.io/system:bootstrappers:kubeadm:default-node-token";rbac.authorization.k8s.io ClusterRolekubeadm:get-nodes"
[root@host-10-10-18-46 ~]# kubectl -n kube-system get nodesNAME STATUS ROLES AGE VERSIONhost-10-10-18-46 NotReady master 157m v1.18.1[root@host-10-10-18-46 ~]# 
[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATEScoredns-546565776c-ch9n7 0/1 Pending 0 157m <none> <none> <none> <none>coredns-546565776c-dddl9 0/1 Pending 0 157m <none> <none> <none> <none>kube-apiserver-host-10-10-18-46 1/1 Running 41 160m 10.10.18.46 host-10-10-18-46 <none> <none>kube-controller-manager-host-10-10-18-46 1/1 Running 31 160m 10.10.18.46 host-10-10-18-46 <none> <none>kube-proxy-zl8fw 1/1 Running 0 157m 10.10.18.46 host-10-10-18-46 <none> <none>kube-scheduler-host-10-10-18-46 1/1 Running 31 160m 10.10.18.46 host-10-10-18-46 <none> <none>[root@host-10-10-18-46 ~]# kubectl -n kube-system describe coredns-546565776c-ch9n7error: the server doesn't have a resource type "coredns-546565776c-ch9n7"[root@host-10-10-18-46 ~]# kubectl -n kube-system describe pods coredns-546565776c-ch9n7Name: coredns-546565776c-ch9n7Namespace: kube-systemPriority: 2000000000Priority Class Name: system-cluster-criticalNode: <none>Labels: k8s-app=kube-dns pod-template-hash=546565776cAnnotations: <none>Status: PendingIP: IPs: <none>Controlled By: ReplicaSet/coredns-546565776cContainers: coredns: Image: registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.6.7 Ports: 53/UDP, 53/TCP, 9153/TCP Host Ports: 0/UDP, 0/TCP, 0/TCP Args: -conf /etc/coredns/Corefile Limits: memory: 170Mi Requests: cpu: 100m memory: 70Mi Liveness: http-get http://:8080/health delay=60s timeout=5s period=10s #success=1 #failure=5 Readiness: http-get http://:8181/ready delay=0s timeout=1s period=10s #success=1 #failure=3 Environment: <none> Mounts: /etc/coredns from config-volume (ro) /var/run/secrets/kubernetes.io/serviceaccount from coredns-token-gl9fl (ro)Conditions: Type Status PodScheduled False Volumes: config-volume: Type: ConfigMap (a volume populated by a ConfigMap) Name: coredns Optional: false coredns-token-gl9fl: Type: Secret (a volume populated by a Secret) SecretName: coredns-token-gl9fl Optional: falseQoS Class: BurstableNode-Selectors: kubernetes.io/os=linuxTolerations: CriticalAddonsOnly node-role.kubernetes.io/master:NoSchedule node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300sEvents: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 120m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 110m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 99m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 89m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 80m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 70m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 57m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 43m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 33m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 23m default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling 6m19s default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate.[root@host-10-10-18-46 ~]# 

允许master节点部署pod,使用命令如下:

[root@host-10-10-18-46 ~]# kubectl taint nodes --all node-role.kubernetes.io/master-node/host-10-10-18-46 untainted[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATEScoredns-546565776c-ch9n7 0/1 Pending 0 159m <none> <none> <none> <none>coredns-546565776c-dddl9 0/1 Pending 0 159m <none> <none> <none> <none>kube-apiserver-host-10-10-18-46 1/1 Running 42 162m 10.10.18.46 host-10-10-18-46 <none> <none>kube-controller-manager-host-10-10-18-46 1/1 Running 32 162m 10.10.18.46 host-10-10-18-46 <none> <none>kube-proxy-zl8fw 1/1 Running 0 159m 10.10.18.46 host-10-10-18-46 <none> <none>kube-scheduler-host-10-10-18-46 1/1 Running 32 162m 10.10.18.46 host-10-10-18-46 <none> <none>
[root@host-10-10-18-46 ~]# kubectl -n kube-system delete pods coredns-546565776c-ch9n7 coredns-546565776c-dddl9 pod "coredns-546565776c-ch9n7" deletedpod "coredns-546565776c-dddl9" deleted[root@host-10-10-18-46 ~]# kubectl -n kube-system get pods -o wideNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATEScoredns-546565776c-v49kt 0/1 Pending 0 3s <none> <none> <none> <none>coredns-546565776c-z5pq6 0/1 Pending 0 4s <none> <none> <none> <none>kube-apiserver-host-10-10-18-46 1/1 Running 42 163m 10.10.18.46 host-10-10-18-46 <none> <none>kube-controller-manager-host-10-10-18-46 1/1 Running 32 163m 10.10.18.46 host-10-10-18-46 <none> <none>kube-proxy-zl8fw 1/1 Running 0 160m 10.10.18.46 host-10-10-18-46 <none> <none>kube-scheduler-host-10-10-18-46 1/1 Running 32 163m 10.10.18.46 host-10-10-18-46 <none> <none>

没有paused

首先,我们看看需要安装哪些镜像,使用如下命令:

[root@host-10-10-18-46 ~]# kubeadm config images listI0630 15:03:59.166843 13472 version.go:252] remote version is much newer: v1.21.2; falling back to: stable-1.18W0630 15:03:59.835027 13472 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]k8s.gcr.io/kube-apiserver:v1.18.20k8s.gcr.io/kube-controller-manager:v1.18.20k8s.gcr.io/kube-scheduler:v1.18.20k8s.gcr.io/kube-proxy:v1.18.20k8s.gcr.io/pause:3.2k8s.gcr.io/etcd:3.4.3-0k8s.gcr.io/coredns:1.6.7
[root@host-10-10-18-46 ~]# kubectl -n kube-system get nodesNAME STATUS ROLES AGE VERSIONhost-10-10-18-46 NotReady master 166m v1.18.1[root@host-10-10-18-46 ~]# 
QoS Class: BurstableNode-Selectors: kubernetes.io/os=linuxTolerations: CriticalAddonsOnly node-role.kubernetes.io/master:NoSchedule node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300sEvents: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate. Warning FailedScheduling <unknown> default-scheduler 0/1 nodes are available: 1 node(s) had taint {node.kubernetes.io/not-ready: }, that the pod didn't tolerate.[root@host-10-10-18-46 ~]# 
[root@host-10-10-18-46 ~]# kubectl describe node host-10-10-18-46Name: host-10-10-18-46Roles: masterLabels: beta.kubernetes.io/arch=arm64 beta.kubernetes.io/os=linux kubernetes.io/arch=arm64 kubernetes.io/hostname=host-10-10-18-46 kubernetes.io/os=linux node-role.kubernetes.io/master=Annotations: kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock node.alpha.kubernetes.io/ttl: 0 volumes.kubernetes.io/controller-managed-attach-detach: trueCreationTimestamp: Wed, 30 Jun 2021 12:03:35 +0800Taints: node.kubernetes.io/not-ready:NoScheduleUnschedulable: falseLease: HolderIdentity: host-10-10-18-46 AcquireTime: <unset> RenewTime: Wed, 30 Jun 2021 14:54:05 +0800Conditions: Type Status LastHeartbeatTime LastTransitionTime Reason Message ---- ------ ----------------- ------------------ ------ ------- MemoryPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientMemory kubelet has sufficient memory available DiskPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasNoDiskPressure kubelet has no disk pressure PIDPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientPID kubelet has sufficient PID available Ready False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletNotReady runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitializedAddresses: InternalIP: 10.10.18.46 Hostname: host-10-10-18-46Capacity: cpu: 4 ephemeral-storage: 7978Mi hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7756672Ki pods: 110Allocatable: cpu: 4 ephemeral-storage: 7528985383 hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7654272Ki pods: 110System Info: Machine ID: 30689d599b59462f9fee88051771bea5 System UUID: B80706BA-B199-4ED2-927B-66A6EC045417 Boot ID: 3205f1fc-6015-4fcd-a9c1-c9c24e2d8d80 Kernel Version: 4.14.0-115.el7a.0.1.aarch64 OS Image: CentOS Linux 7 (AltArch) Operating System: linux Architecture: arm64 Container Runtime Version: docker://20.10.7 Kubelet Version: v1.18.1 Kube-Proxy Version: v1.18.1Non-terminated Pods: (4 in total) Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE --------- ---- ------------ ---------- --------------- ------------- --- kube-system kube-apiserver-host-10-10-18-46 250m (6%) 0 (0%) 0 (0%) 0 (0%) 169m kube-system kube-controller-manager-host-10-10-18-46 200m (5%) 0 (0%) 0 (0%) 0 (0%) 170m kube-system kube-proxy-zl8fw 0 (0%) 0 (0%) 0 (0%) 0 (0%) 166m kube-system kube-scheduler-host-10-10-18-46 100m (2%) 0 (0%) 0 (0%) 0 (0%) 170mAllocated resources: (Total limits may be over 100 percent, i.e., overcommitted.) Resource Requests Limits -------- -------- ------ cpu 550m (13%) 0 (0%) memory 0 (0%) 0 (0%) ephemeral-storage 0 (0%) 0 (0%) hugepages-2Mi 0 (0%) 0 (0%) hugepages-512Mi 0 (0%) 0 (0%)Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Starting 171m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 171m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 171m (x5 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 170m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeHasSufficientMemory 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal NodeAllocatableEnforced 170m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal Starting 166m kube-proxy, host-10-10-18-46 Starting kube-proxy. Normal Starting 34m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 34m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 34m (x7 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 10m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 10m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 10m (x7 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID[root@host-10-10-18-46 ~]# kubectl describe node host-10-10-18-46 | grep TaintsTaints: node.kubernetes.io/not-ready:NoSchedule[root@host-10-10-18-46 ~]# 
[root@host-10-10-18-46 ~]# kubectl taint node host-10-10-18-46 node-role.kubernetes.io/master:NoSchedule-error: taint "node-role.kubernetes.io/master:NoSchedule" not found[root@host-10-10-18-46 ~]# kubectl describe nodes |grep TaintsTaints: node.kubernetes.io/not-ready:NoSchedule[root@host-10-10-18-46 ~]# 

可以看到最后的方式为NoSchedule,可以通过kubectl taint命令进行设定如下三种方式,具体说明如下:

  • NoSchedule: 不调度
  • PreferNoSchedule: 尽量不调度
  • NoExecute: 不调度并且立即驱逐节点上现存pod
[root@host-10-10-18-46 ~]# kubectl describe node host-10-10-18-46Name: host-10-10-18-46Roles: masterLabels: beta.kubernetes.io/arch=arm64 beta.kubernetes.io/os=linux kubernetes.io/arch=arm64 kubernetes.io/hostname=host-10-10-18-46 kubernetes.io/os=linux node-role.kubernetes.io/master=Annotations: kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock node.alpha.kubernetes.io/ttl: 0 volumes.kubernetes.io/controller-managed-attach-detach: trueCreationTimestamp: Wed, 30 Jun 2021 12:03:35 +0800Taints: node.kubernetes.io/not-ready:NoScheduleUnschedulable: falseLease: HolderIdentity: host-10-10-18-46 AcquireTime: <unset> RenewTime: Wed, 30 Jun 2021 14:54:05 +0800Conditions: Type Status LastHeartbeatTime LastTransitionTime Reason Message ---- ------ ----------------- ------------------ ------ ------- MemoryPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientMemory kubelet has sufficient memory available DiskPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasNoDiskPressure kubelet has no disk pressure PIDPressure False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletHasSufficientPID kubelet has sufficient PID available Ready False Wed, 30 Jun 2021 14:49:22 +0800 Wed, 30 Jun 2021 12:03:35 +0800 KubeletNotReady runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitializedAddresses: InternalIP: 10.10.18.46 Hostname: host-10-10-18-46Capacity: cpu: 4 ephemeral-storage: 7978Mi hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7756672Ki pods: 110Allocatable: cpu: 4 ephemeral-storage: 7528985383 hugepages-2Mi: 0 hugepages-512Mi: 0 memory: 7654272Ki pods: 110System Info: Machine ID: 30689d599b59462f9fee88051771bea5 System UUID: B80706BA-B199-4ED2-927B-66A6EC045417 Boot ID: 3205f1fc-6015-4fcd-a9c1-c9c24e2d8d80 Kernel Version: 4.14.0-115.el7a.0.1.aarch64 OS Image: CentOS Linux 7 (AltArch) Operating System: linux Architecture: arm64 Container Runtime Version: docker://20.10.7 Kubelet Version: v1.18.1 Kube-Proxy Version: v1.18.1Non-terminated Pods: (4 in total) Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits AGE --------- ---- ------------ ---------- --------------- ------------- --- kube-system kube-apiserver-host-10-10-18-46 250m (6%) 0 (0%) 0 (0%) 0 (0%) 169m kube-system kube-controller-manager-host-10-10-18-46 200m (5%) 0 (0%) 0 (0%) 0 (0%) 170m kube-system kube-proxy-zl8fw 0 (0%) 0 (0%) 0 (0%) 0 (0%) 166m kube-system kube-scheduler-host-10-10-18-46 100m (2%) 0 (0%) 0 (0%) 0 (0%) 170mAllocated resources: (Total limits may be over 100 percent, i.e., overcommitted.) Resource Requests Limits -------- -------- ------ cpu 550m (13%) 0 (0%) memory 0 (0%) 0 (0%) ephemeral-storage 0 (0%) 0 (0%) hugepages-2Mi 0 (0%) 0 (0%) hugepages-512Mi 0 (0%) 0 (0%)Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Starting 171m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 171m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 171m (x5 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 171m (x4 over 171m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 170m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeHasSufficientMemory 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 170m kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal NodeAllocatableEnforced 170m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal Starting 166m kube-proxy, host-10-10-18-46 Starting kube-proxy. Normal Starting 34m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 34m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 34m (x8 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 34m (x7 over 34m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID Normal Starting 10m kubelet, host-10-10-18-46 Starting kubelet. Normal NodeAllocatableEnforced 10m kubelet, host-10-10-18-46 Updated Node Allocatable limit across pods Normal NodeHasSufficientMemory 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientMemory Normal NodeHasNoDiskPressure 10m (x8 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasNoDiskPressure Normal NodeHasSufficientPID 10m (x7 over 10m) kubelet, host-10-10-18-46 Node host-10-10-18-46 status is now: NodeHasSufficientPID
 runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized

安装flannel后解决

[root@host-10-10-18-46 pki]# kubectl get nodesNAME STATUS ROLES AGE VERSIONhost-10-10-18-46 Ready master 3h54m v1.18.1
[root@host-10-10-18-46 pki]# kubectl get pods -o wide -n kube-systemNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATEScoredns-546565776c-v49kt 0/1 ContainerCreating 0 75m <none> host-10-10-18-46 <none> <none>coredns-546565776c-z5pq6 0/1 ContainerCreating 0 75m <none> host-10-10-18-46 <none> <none>kube-apiserver-host-10-10-18-46 1/1 Running 64 3h59m 10.10.18.46 host-10-10-18-46 <none> <none>kube-controller-manager-host-10-10-18-46 1/1 Running 51 3h59m 10.10.18.46 host-10-10-18-46 <none> <none>kube-flannel-ds-arm64-x7mnq 1/1 Running 13 43m 10.10.18.46 host-10-10-18-46 <none> <none>kube-proxy-zl8fw 1/1 Running 0 3h55m 10.10.18.46 host-10-10-18-46 <none> <none>kube-scheduler-host-10-10-18-46 1/1 Running 52 3h59m 10.10.18.46 host-10-10-18-46 <none> <none>

token没有过期

 如果没有--discovery-token-ca-cert-hash值,也可以通过以下命令获取openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
  • 如果是过期了,需要重新生成

1. 执行kubeadm token create --print-join-command,重新生成,重新生成基础的 join 命令(对于添加 master 节点还需要重新生成certificate-key,见下一步)# 如果是添加 worker 节点,不需要执行这一步,直接使用上面返回的 join 命令加入集群。2. 使用 kubeadm init phase upload-certs --experimental-upload-certs 重新生成certificate-key# 添加 master 节点:用上面第1步生成的 join 命令和第2步生成的--certificate-key 值拼接起来执行

新增Master节点额外需要certificate-key参数,使用以下命令生成:

# 生成certificate-keykubeadm init phase upload-certs --upload-certs# 使用Node节点的join命令并且拼上--control-plane --certificate-key参数kubeadm join kubernetes-vip:9443 --token bayqt8.eaafmfthasquy4yn --discovery-token-ca-cert-hash sha256:250115fad0a4b6852a919dbba4222ac65bc64843c660363ab119606ff8819d0a --control-plane --certificate-key bfd5bc7ff4aa54e1cba9a5979210c06ae087ae6fb9979af8f851554638889d7b
[root@host-10-10-18-46 ~]# kubeadm token listTOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPSabcdef.0123456789abcdef 20h 2021-07-01T12:03:42+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token[root@host-10-10-18-46 ~]# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'9bc0bcddb2b97791717943b714ffa410cb5963061889086f04eda6150cb590fc[root@host-10-10-18-46 ~]# kubeadm init phase upload-certs --upload-certsI0630 15:34:33.032985 8128 version.go:252] remote version is much newer: v1.21.2; falling back to: stable-1.18W0630 15:34:34.097393 8128 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io][upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace[upload-certs] Using certificate key:8c94eb58dfdfc88b2f949d59f7f4348984dc0b155e37488a2f95df7048ca7374
[root@host-10-10-18-46 ~]# kubeadm token listTOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPSabcdef.0123456789abcdef 20h 2021-07-01T12:03:42+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-tokens7li41.5u78f4i2oqfg4t1c 1h 2021-06-30T17:34:43+08:00 <none> Proxy for managing TTL for the kubeadm-certs secret <none>[root@host-10-10-18-46 ~]#
[root@host-10-10-18-46 ~]# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'9bc0bcddb2b97791717943b714ffa410cb5963061889086f04eda6150cb590fc[root@host-10-10-18-46 ~]#

拷贝秘钥

证书失效管理

部署高可用集群

KunPeng平台 Cfssl 1.4.1版本移植安装指南

Top Articles
Latest Posts
Article information

Author: Zonia Mosciski DO

Last Updated: 03/03/2023

Views: 5991

Rating: 4 / 5 (51 voted)

Reviews: 90% of readers found this page helpful

Author information

Name: Zonia Mosciski DO

Birthday: 1996-05-16

Address: Suite 228 919 Deana Ford, Lake Meridithberg, NE 60017-4257

Phone: +2613987384138

Job: Chief Retail Officer

Hobby: Tai chi, Dowsing, Poi, Letterboxing, Watching movies, Video gaming, Singing

Introduction: My name is Zonia Mosciski DO, I am a enchanting, joyous, lovely, successful, hilarious, tender, outstanding person who loves writing and wants to share my knowledge and understanding with you.