1.3
环境说明
Ubuntu18.04安装Kubernetes 1.24
| IP地址 |
系统环境 |
规格 |
角色 |
| 192.168.71.97 |
ubuntu-18.04.6 |
4C*4G-50GB |
master |
| 192.168.71.98 |
ubuntu-18.04.6 |
4C*4G-50GB |
node1 |
| 192.168.71.99 |
ubuntu-18.04.6 |
4C*4G-50GB |
node2 |
安装
容器安装（master和node）
apt-get update
apt-get -y install apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
apt-get -y update
apt-get -y install docker-ce
mkdir /etc/docker
# Setup daemon.
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"registry-mirrors": ["https://ndqnsmic.mirror.aliyuncs.com"]
}
EOF
mkdir -p /etc/systemd/system/docker.service.d
sed -i '/^disabled_plugins/d' /etc/containerd/config.toml
cat <<EOF> /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
# 安装crictl: https://blog.csdn.net/xixihahalelehehe/article/details/116591151
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.1/crictl-v1.26.1-linux-amd64.tar.gz
tar zxvf crictl-v1.26.1-linux-amd64.tar.gz -C /usr/local/bin
systemctl daemon-reload && service docker start && systemctl enable docker
systemctl restart containerd
chmod 777 /var/run/docker.sock
# 验证 containerd 功能是否正常,会显示 IMAGE 等字样，如果没有，请执行 systemctl restart containerd 再次重启一下。
docker images
配置源并安装（master和node）
apt-get update && apt-get install -y apt-transport-https
sudo curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
apt-get update
apt-get install -y kubelet=1.24.0-00 kubeadm=1.24.0-00 kubectl=1.24.0-00
systemctl enable kubelet && systemctl start kubelet && systemctl status kubelet
# journalctl -x kubelet 查看报错信息，kubeadm 工具比较喜欢更改 pause 的地址，看到pause 拉取错误了，tag命令替换成对应的地址。
ctr -n k8s.io i pull registry.aliyuncs.com/google_containers/pause:3.6
ctr -n k8s.io i tag registry.aliyuncs.com/google_containers/pause:3.6 registry.k8s.io/pause:3.6
ctr -n k8s.io i tag registry.aliyuncs.com/google_containers/pause:3.6 k8s.gcr.io/pause:3.6
（master和node）
# 默认使用本机hostname，不设置，会与node重名，最终只能显示一个node.
# kubeadm初始化k8s集群# 每台都拉取镜像包
crictl pull docker.io/calico/cni:v3.25.0
crictl pull docker.io/calico/kube-controllers:v3.25.0
crictl pull docker.io/calico/node:v3.25.0
crictl pull docker.io/library/nginx:latest
crictl pull k8s.gcr.io/pause:3.6
crictl pull registry.aliyuncs.com/google_containers/pause:3.6
crictl pull registry.k8s.io/pause:3.6
crictl pull registry.aliyuncs.com/google_containers/coredns:v1.8.6
crictl pull registry.aliyuncs.com/google_containers/etcd:3.5.3-0
crictl pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.24.0
crictl pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.24.0
crictl pull registry.aliyuncs.com/google_containers/kube-proxy:v1.24.0
crictl pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.24.0
crictl pull registry.aliyuncs.com/google_containers/pause:3.7
ctr -n k8s.io i tag registry.aliyuncs.com/google_containers/pause:3.6 k8s.gcr.io/pause:3.6
sudo kubeadm init --image-repository=registry.aliyuncs.com/google_containers --pod-network-cidr=10.244.0.0/16 --kubernetes-version=v1.24.0 --ignore-preflight-errors=all -v5
sudo mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
node执行
sudo hostnamectl set-hostname node
# 再master上执行
sudo kubeadm token create --print-join-command
# 粘贴执行输出的命令。
sudo kubeadm join xxx --token xxxxx --discovery-token-ca-cert-hash sha256:xxxx -v 5
安装网络插件
# 安装k8s网络插件
sudo curl https://docs.tigera.io/archive/v3.25/manifests/calico.yaml -O && kubectl apply -f calico.yaml
应用安装测试
kubectl get node显示Ready时再执行下面语句。
kubectl taint node master node-role.kubernetes.io/master:NoSchedule-
kubectl taint node master node-role.kubernetes.io/control-plane:NoSchedule-
#创建一个tomcat应用并访问
kubectl create deployment tomcat --image=tomcat
kubectl expose deployment tomcat --port=8080 --target-port=8080 --type=NodePort
#确认 tomcat 部署成功。
kubectl get pods
#确认端口
kubectl get service
#浏览器访问应用。
http://xxxxx:xxx
查看启动状态
root@k8s-master:~# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 19d v1.24.0
k8s-node1 Ready <none> 19d v1.24.0
k8s-node2 Ready <none> 19d v1.24.0
root@k8s-master:~# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
default tomcat-84c98b5bb-zn5bv 1/1 Running 2 (22h ago) 19d
kube-system calico-kube-controllers-55fc758c88-8dz8l 1/1 Running 0 19d
kube-system calico-node-mcl2j 1/1 Running 0 19d
kube-system calico-node-ss8cd 1/1 Running 2 (22h ago) 19d
kube-system calico-node-vlzdd 1/1 Running 2 (22h ago) 19d
kube-system coredns-74586cf9b6-d8vs2 1/1 Running 0 19d
kube-system coredns-74586cf9b6-hf2qm 1/1 Running 0 19d
kube-system etcd-k8s-master 1/1 Running 4 (7h30m ago) 19d
kube-system kube-apiserver-k8s-master 1/1 Running 3 (7h32m ago) 19d
kube-system kube-controller-manager-k8s-master 1/1 Running 4 (7h30m ago) 19d
kube-system kube-proxy-p2px4 1/1 Running 2 (22h ago) 19d
kube-system kube-proxy-s6f65 1/1 Running 0 19d
kube-system kube-proxy-w5p9q 1/1 Running 2 (22h ago) 19d
kube-system kube-scheduler-k8s-master 1/1 Running 4 (7h30m ago) 19d
root@k8s-master:~# kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Apr 30, 2025 06:04 UTC 345d ca no
apiserver Apr 30, 2025 06:04 UTC 345d ca no
apiserver-etcd-client Apr 30, 2025 06:04 UTC 345d etcd-ca no
apiserver-kubelet-client Apr 30, 2025 06:04 UTC 345d ca no
controller-manager.conf Apr 30, 2025 06:04 UTC 345d ca no
etcd-healthcheck-client Apr 30, 2025 06:04 UTC 345d etcd-ca no
etcd-peer Apr 30, 2025 06:04 UTC 345d etcd-ca no
etcd-server Apr 30, 2025 06:04 UTC 345d etcd-ca no
front-proxy-client Apr 30, 2025 06:04 UTC 345d front-proxy-ca no
scheduler.conf Apr 30, 2025 06:04 UTC 345d ca no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca Apr 28, 2034 06:04 UTC 9y no
etcd-ca Apr 28, 2034 06:04 UTC 9y no
front-proxy-ca Apr 28, 2034 06:04 UTC 9y no
root@k8s-master:~# crictl image ls
IMAGE TAG IMAGE ID SIZE
docker.io/calico/cni v3.25.0 d70a5947d57e5 88MB
docker.io/calico/kube-controllers v3.25.0 5e785d005ccc1 31.3MB
docker.io/calico/node v3.25.0 08616d26b8e74 87.2MB
docker.io/library/nginx latest 7383c266ef252 71MB
k8s.gcr.io/pause 3.6 6270bb605e12e 302kB
registry.aliyuncs.com/google_containers/pause 3.6 6270bb605e12e 302kB
registry.k8s.io/pause 3.6 6270bb605e12e 302kB
registry.aliyuncs.com/google_containers/coredns v1.8.6 a4ca41631cc7a 13.6MB
registry.aliyuncs.com/google_containers/etcd 3.5.3-0 aebe758cef4cd 102MB
registry.aliyuncs.com/google_containers/kube-apiserver v1.24.0 529072250ccc6 33.8MB
registry.aliyuncs.com/google_containers/kube-controller-manager v1.24.0 88784fb4ac2f6 31MB
registry.aliyuncs.com/google_containers/kube-proxy v1.24.0 77b49675beae1 39.5MB
registry.aliyuncs.com/google_containers/kube-scheduler v1.24.0 e3ed7dee73e93 15.5MB
registry.aliyuncs.com/google_containers/pause 3.7 221177c6082a8 311kB
root@k8s-master:~# crictl ps
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
c2fdf7df867b5 5e785d005ccc1 8 hours ago Running calico-kube-controllers 0 b12c3e5b5736c calico-kube-controllers-55fc758c88-8dz8l
4cc62631f3cf4 a4ca41631cc7a 8 hours ago Running coredns 0 916049f0b430d coredns-74586cf9b6-d8vs2
8a0f57eb2b189 a4ca41631cc7a 8 hours ago Running coredns 0 4ef2ac28d1b5f coredns-74586cf9b6-hf2qm
db0325f275566 08616d26b8e74 8 hours ago Running calico-node 0 e642ac46dec8a calico-node-mcl2j
a378dceb07b1c 77b49675beae1 8 hours ago Running kube-proxy 0 c802ab54bc10c kube-proxy-s6f65
881bb24033a4b e3ed7dee73e93 8 hours ago Running kube-scheduler 4 a3eafeb99d96d kube-scheduler-k8s-master
d951b63c70c26 88784fb4ac2f6 8 hours ago Running kube-controller-manager 4 6bd44f3c44c98 kube-controller-manager-k8s-master
e3506775e2db3 aebe758cef4cd 8 hours ago Running etcd 4 800f2c8b6c1b3 etcd-k8s-master
ca280b8dd36a8 529072250ccc6 8 hours ago Running kube-apiserver 3 de480d62a3e4f kube-apiserver-k8s-master
root@k8s-master:~#
异常排错
# kubectl describe pod -n kube-system coredns-5bbd96d687-9dv66
……
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedScheduling 2m6s default-scheduler 0/1 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling..
// 可以尝试解决
# kubectl taint nodes master node.kubernetes.io/not-ready:NoSchedule-
外部安装etcdctl
检查当前安装版本
root@k8s-master:/home/wnh# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-55fc758c88-8dz8l 1/1 Running 2 (14h ago) 18d
calico-node-mcl2j 1/1 Running 2 (14h ago) 18d
calico-node-ss8cd 1/1 Running 2 (14h ago) 18d
calico-node-vlzdd 1/1 Running 2 (14h ago) 18d
coredns-74586cf9b6-d8vs2 1/1 Running 2 (14h ago) 18d
coredns-74586cf9b6-hf2qm 1/1 Running 2 (14h ago) 18d
etcd-k8s-master 1/1 Running 2 (14h ago) 18d
kube-apiserver-k8s-master 1/1 Running 2 (14h ago) 18d
kube-controller-manager-k8s-master 1/1 Running 2 (14h ago) 18d
kube-proxy-p2px4 1/1 Running 2 (14h ago) 18d
kube-proxy-s6f65 1/1 Running 2 (14h ago) 18d
kube-proxy-w5p9q 1/1 Running 2 (14h ago) 18d
kube-scheduler-k8s-master 1/1 Running 2 (14h ago) 18d
root@k8s-master:/home/wnh# kubeckubec^C
root@k8s-master:/home/wnh# kubectl exec -it -n kube-system etcd-k8s-master /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
sh-5.1# etcdctl
etcdctl etcdctl-3.0.17 etcdctl-3.1.20 etcdctl-3.2.32 etcdctl-3.3.17 etcdctl-3.4.18 etcdctl-3.5.3
sh-5.1# etcdctl version
etcdctl version: 3.5.3
API version: 3.5
sh-5.1#
将对应版本的etcdctl安装到本地
https://github.com/etcd-io/etcd/releases
https://github.com/etcd-io/etcd/releases?page=3
root@k8s-master:/home/wnh/k8s# wget https://github.com/etcd-io/etcd/releases/download/v3.5.3/etcd-v3.5.3-linux-amd64.tar.gz
root@k8s-master:/home/wnh/k8s# tar xvf etcd-v3.5.3-linux-amd64.tar.gz
root@k8s-master:/home/wnh/k8s# cp etcd-v3.5.3-linux-amd64/etcdctl /usr/bin/
测试快照备份是否能够正常执行
ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt endpoint health -w table
ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt snapshot save /home/wnh/k8s/etcdsnapshot/97etcdback-240519.db
快照恢复测试
ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt snapshot restore /home/wnh/k8s/etcdsnapshot/97etcdback-240519.db --data-dir=/var/lib/etcd
// 其他参数参考
etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints 10.20.30.31:2379 snapshot restore /var/backups/kube_etcd/etcd-2024-0206-snapshot.db \
--name=etcd01 \
--initial-cluster=etcd01=https://10.20.30.31:2380 \
--initial-advertise-peer-urls=https://10.20.30.31:2380 \
--data-dir=/var/lib/etcd
// 单节点和集群对应的参考文档
https://zhuanlan.zhihu.com/p/696459722
// 通过 etcdctl 客户端查看当前新安装 Etcd 集群存储的 key
ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt get --prefix "" --keys-only=true
ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt endpoint health -w table
ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt endpoint status -w table
root@k8s-master:/var/lib# ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt endpoint health -w table
+----------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+----------------------------+--------+-------------+-------+
| https://192.168.71.97:2379 | true | 19.762009ms | |
+----------------------------+--------+-------------+-------+
root@k8s-master:/var/lib# ETCDCTL_API=3 etcdctl --endpoints=https://192.168.71.97:2379 --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt endpoint status -w table
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.71.97:2379 | 8e9e05c52164694d | 3.5.3 | 3.4 MB | true | false | 2 | 3336 | 3336 | |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
root@k8s-master:/var/lib#