1. 简介

IP地址 操作系统 k8s版本 runtime 部署方式
192.168.10.31 centos7 1.24.2 containerd kubeadm
192.168.10.32 centos7 1.24.2 containerd kubeadm
192.168.10.33 centos7 1.24.2 containerd kubeadm

2. 初始化环境

更新系统

1
2
3
[root@node001 ~]# yum update -y
[root@node002 ~]# yum update -y
[root@node003 ~]# yum update -y

关闭防火墙

1
2
3
[root@node001 ~]# firewall-cmd --set-default-zone=trusted
[root@node002 ~]# firewall-cmd --set-default-zone=trusted
[root@node003 ~]# firewall-cmd --set-default-zone=trusted

关闭selinux

1
2
3
[root@node001 ~]# setenforce 0
[root@node002 ~]# setenforce 0
[root@node003 ~]# setenforce 0

关闭swap

1
2
3
4
5
[root@node001 ~]# swapoff -a ; sed -i '/swap/d' /etc/fstab
[root@node002 ~]# swapoff -a ; sed -i '/swap/d' /etc/fstab
[root@node003 ~]# swapoff -a ; sed -i '/swap/d' /etc/fstab

[root@node002 ~]# swapon -s

配置yum源

1
2
3
[root@node001 ~]# rm -rf /etc/yum.repos.d/* ; wget ftp://ftp.rhce.cc/k8s/* -P /etc/yum.repos.d/
[root@node002 ~]# rm -rf /etc/yum.repos.d/* ; wget ftp://ftp.rhce.cc/k8s/* -P /etc/yum.repos.d/
[root@node003 ~]# rm -rf /etc/yum.repos.d/* ; wget ftp://ftp.rhce.cc/k8s/* -P /etc/yum.repos.d/

添加hosts

1
2
3
4
5
cat > /etc/hosts <<EOF
192.168.10.31 node001
192.168.10.32 node002
192.168.10.33 node003
EOF

加载内核模块

1
2
3
4
5
6
7
cat > /etc/modules-load.d/containerd.conf <<EOF
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

修改内核参数

1
2
3
4
5
6
7
cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

3. 安装containerd

1
2
3
4
[root@node001 ~]# yum install containerd.io cri-tools -y
[root@node002 ~]# yum install containerd.io cri-tools -y
[root@node003 ~]# yum install containerd.io cri-tools -y

生成配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
containerd config default > /etc/containerd/config.toml

第一:搜索mirrors,把
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
改成
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."nerdctl.io"]
endpoint = ["https://frz7i079.mirror.aliyuncs.com"]

第二:搜索sandbox,把
sandbox_image = "k8s.gcr.io/pause:3.6"
改为
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.7"

第三:搜索SystemdCgroup,把
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = ""
CriuImagePath = ""
CriuPath = ""
CriuWorkPath = ""
IoGid = 0
IoUid = 0
NoNewKeyring = false
NoPivotRoot = false
Root = ""
ShimCgroup = ""
SystemdCgroup = false
改成
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true

config.toml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/var/lib/containerd"
state = "/run/containerd"
temp = ""
version = 2

[cgroup]
path = ""

[debug]
address = ""
format = ""
gid = 0
level = ""
uid = 0

[grpc]
address = "/run/containerd/containerd.sock"
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
tcp_address = ""
tcp_tls_ca = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0

[metrics]
address = ""
grpc_histogram = false

[plugins]

[plugins."io.containerd.gc.v1.scheduler"]
deletion_threshold = 0
mutation_threshold = 100
pause_threshold = 0.02
schedule_delay = "0s"
startup_delay = "100ms"

[plugins."io.containerd.grpc.v1.cri"]
device_ownership_from_security_context = false
disable_apparmor = false
disable_cgroup = false
disable_hugetlb_controller = true
disable_proc_mount = false
disable_tcp_service = true
enable_selinux = false
enable_tls_streaming = false
enable_unprivileged_icmp = false
enable_unprivileged_ports = false
ignore_image_defined_volumes = false
max_concurrent_downloads = 3
max_container_log_line_size = 16384
netns_mounts_under_state_dir = false
restrict_oom_score_adj = false
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.7"
selinux_category_range = 1024
stats_collect_period = 10
stream_idle_timeout = "4h0m0s"
stream_server_address = "127.0.0.1"
stream_server_port = "0"
systemd_cgroup = false
tolerate_missing_hugetlb_controller = true
unset_seccomp_profile = ""

[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = ""
ip_pref = ""
max_conf_num = 1

[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
disable_snapshot_annotations = true
discard_unpacked_layers = false
ignore_rdt_not_enabled_errors = false
no_pivot = false
snapshotter = "overlayfs"

[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""

[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""

[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = "node"

[plugins."io.containerd.grpc.v1.cri".registry]
config_path = ""

[plugins."io.containerd.grpc.v1.cri".registry.auths]

[plugins."io.containerd.grpc.v1.cri".registry.configs]

[plugins."io.containerd.grpc.v1.cri".registry.headers]

[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://frz7i079.mirror.aliyuncs.com"]

[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""

[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"

[plugins."io.containerd.internal.v1.restart"]
interval = "10s"

[plugins."io.containerd.internal.v1.tracing"]
sampling_ratio = 1.0
service_name = "containerd"

[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"

[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false

[plugins."io.containerd.runtime.v1.linux"]
no_shim = false
runtime = "runc"
runtime_root = ""
shim = "containerd-shim"
shim_debug = false

[plugins."io.containerd.runtime.v2.task"]
platforms = ["linux/amd64"]
sched_core = false

[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]

[plugins."io.containerd.service.v1.tasks-service"]
rdt_config_file = ""

[plugins."io.containerd.snapshotter.v1.aufs"]
root_path = ""

[plugins."io.containerd.snapshotter.v1.btrfs"]
root_path = ""

[plugins."io.containerd.snapshotter.v1.devmapper"]
async_remove = false
base_image_size = ""
discard_blocks = false
fs_options = ""
fs_type = ""
pool_name = ""
root_path = ""

[plugins."io.containerd.snapshotter.v1.native"]
root_path = ""

[plugins."io.containerd.snapshotter.v1.overlayfs"]
root_path = ""
upperdir_label = false

[plugins."io.containerd.snapshotter.v1.zfs"]
root_path = ""

[plugins."io.containerd.tracing.processor.v1.otlp"]
endpoint = ""
insecure = false
protocol = ""

[proxy_plugins]

[stream_processors]

[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar"

[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
"io.containerd.timeout.bolt.open" = "0s"
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"

[ttrpc]
address = ""
gid = 0
uid = 0

重启服务

1
2
3
[root@node001 ~]# systemctl enable containerd;systemctl restart containerd
[root@node002 ~]# systemctl enable containerd;systemctl restart containerd
[root@node003 ~]# systemctl enable containerd;systemctl restart containerd

4. 配置crictl客户端工具

1
2
3
[root@node001 ~]# crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock
[root@node002 ~]# crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock
[root@node003 ~]# crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock

5. nerdctl安装和配置

1
2
3
4
5
6
curl -OL https://github.com/containerd/nerdctl/releases/download/v0.22.0/nerdctl-0.22.0-linux-amd64.tar.gz
tar xf nerdctl-0.22.0-linux-amd64.tar.gz -C /usr/bin/

curl -OL https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
mkdir -p /opt/cni/bin
tar xf cni-plugins-linux-amd64-v1.1.1.tgz -C /opt/cni/bin/

创建配置文件

1
2
3
4
5
6
7
8
9
10
11
12
mkdir /etc/nerdctl/

cat > /etc/nerdctl/nerdctl.toml <<EOF
debug = false
debug_full = false
address = "unix:///var/run/containerd/containerd.sock"
namespace = "k8s.io"
#snapshotter = "stargz"
cgroup_manager = "systemd"
hosts_dir = ["/etc/containerd/certs.d"]
insecure_registry = false
EOF

镜像加速

1
2
3
4
5
6
7
8
mkdir -p /etc/containerd/certs.d/docker.io

cat > /etc/containerd/certs.d/docker.io/hosts.toml <<EOF
# server = "https://docker.io"
[host."https://frz7i079.mirror.aliyuncs.com"]
capabilities = ["pull","resolve"]
override_path = true
EOF

切换名称空间

1
export CONTAINERD_NAMESPACE=k8s.io

6. 安装k8s

查看yum源中可用的版本

1
[root@node001 ~]# yum list --showduplicates kubeadm --disableexcludes=kubernetes

安装客户端工具

1
2
3
[root@node001 ~]# yum install -y kubelet-1.24.2-0 kubeadm-1.24.2-0 kubectl-1.24.2-0 --disableexcludes=kubernetes
[root@node002 ~]# yum install -y kubelet-1.24.2-0 kubeadm-1.24.2-0 kubectl-1.24.2-0 --disableexcludes=kubernetes
[root@node003 ~]# yum install -y kubelet-1.24.2-0 kubeadm-1.24.2-0 kubectl-1.24.2-0 --disableexcludes=kubernetes

启动kubelet

1
2
3
[root@node001 ~]# systemctl restart kubelet ; systemctl enable kubelet
[root@node002 ~]# systemctl restart kubelet ; systemctl enable kubelet
[root@node003 ~]# systemctl restart kubelet ; systemctl enable kubelet

初始化安装1.24.2

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@node001 ~]# kubeadm init --image-repository registry.aliyuncs.com/google_containers --kubernetes-version=v1.24.2 --pod-network-cidr=10.244.0.0/16

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 172.17.203.55:6443 --token 58kell.5rbsvxa5pftz54v3 \
--discovery-token-ca-cert-hash sha256:c38177a8a3962a5e50a3d8e58f60de869d315d3c753b915e69d7a3ec70a6faea

其他节点加入集群

1
2
kubeadm join 172.17.203.55:6443 --token 58kell.5rbsvxa5pftz54v3 \
--discovery-token-ca-cert-hash sha256:c38177a8a3962a5e50a3d8e58f60de869d315d3c753b915e69d7a3ec70a6faea

查看node节点状态

1
2
3
4
5
[root@node001 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
node001 NotReady control-plane 3m58s v1.24.2
node002 NotReady <none> 48s v1.24.2
node003 NotReady <none> 45s v1.24.2

安装cni网络插件

1
2
3
4
5
6
7
8
9
10
11
12
[root@node001 ~]# wget https://docs.projectcalico.org/manifests/calico.yaml

修改calico里pod所在网段
- name: CALICO_IPV4POOL_CIDR
value: "10.244.0.0/16"


为了防止多张网卡带来的问题,修改calico.yaml
- name: CALICO_IPV4POOL_VXLAN
value: "Never"
- name: IP_AUTODETECTION_METHOD
value: "interface=eth1"

安装calico

1
[root@node001 ~]# kubectl apply -f calico.yaml

安装命令补全

1
2
[root@node001 ~]# yum install bash-completion -y
source <(kubectl completion bash)

集群状态

1
2
3
4
5
[root@node001 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
node001 Ready control-plane 32m v1.24.2
node002 Ready <none> 29m v1.24.2
node003 Ready <none> 29m v1.24.2

7. 删除新增节点

删除节点

1
2
3
4
5
6
[root@node001 ~]# kubectl drain node003 --ignore-daemonsets
[root@node001 ~]# kubectl delete nodes node003
[root@node001 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
node001 Ready control-plane 36m v1.24.2
node002 Ready <none> 33m v1.24.2

node003重置

1
[root@node003 ~]# kubeadm reset

获取加入集群的命令

1
2
[root@node001 ~]# kubeadm token create --print-join-command
kubeadm join 172.17.203.55:6443 --token yl9g09.34s5glmqlomdjc59 --discovery-token-ca-cert-hash sha256:c38177a8a3962a5e50a3d8e58f60de869d315d3c753b915e69d7a3ec70a6faea

重新加入集群

1
[root@node003 ~]# kubeadm join 172.17.203.55:6443 --token yl9g09.34s5glmqlomdjc59 --discovery-token-ca-cert-hash sha256:c38177a8a3962a5e50a3d8e58f60de869d315d3c753b915e69d7a3ec70a6faea

8. 安装metrics-server

下载最新版本

1
2
3
4
5
6
https://github.com/kubernetes-sigs/metrics-server/releases/tag/metrics-server-helm-chart-3.8.2

导入镜像
[root@node001 metrics]# nerdctl load -i metric-img-v0.6.1.tar
[root@node002 metrics]# nerdctl load -i metric-img-v0.6.1.tar
[root@node003 metrics]# nerdctl load -i metric-img-v0.6.1.tar

修改配置

1
2
3
4
5
6
7
8
9
10
修改metrics-server-deployment.yaml
containers: - name: metrics-server
image: k8s.gcr.io/metrics-server-amd64:v0.3.6
#imagePullPolicy: Always
imagePullPolicy: IfNotPresent
command: - /metrics-server
- --metric-resolution=30s
- --kubelet-insecure-tls
- --kubelet-preferred-address-types=InternalIP
#- --cert-dir=/tmp

安装

1
[root@node001 metrics]# kubectl apply -f components.yaml 

查看负载

1
2
3
4
5
[root@node001 metrics]# kubectl  top node
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
node001 84m 2% 1929Mi 52%
node002 32m 0% 717Mi 19%
node003 36m 0% 910Mi 24%