1. 简介

etcd集群是一个分布式系统,使用Raft协议来维护集群内容各个节点状态的一致性。
主机状态Leader,Follower,Candidate

当集群初始化时候,每个节点都是Follower角色
通过心跳与其他节点同步数据
当Follower在一定时间内没有收到来自其他节点的心跳,会将自己角色改变为Candidate,并发起一次选主投票。

配置etcd集群,建议尽可能是奇数个节点。

目前在用的两个大版本v2,v3
v3已键值对的方式来存储数据,k8s从1.5开始使用v3版本etcd。

2. 单节点etcd

2.1 安装etcd

1
[root@node001 ~]# yum install etcd -y

2.2 修改配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@node001 ~]# vim /etc/etcd/etcd.conf
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="http://192.168.10.41:2380,http://localhost:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.10.41:2379,http://localhost:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="default"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
#ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.10.41:2379,http://localhost:2379"

2.3 启动etcd

1
[root@node001 ~]#  systemctl start etcd

2.4 使用etcdctl连接服务端

etcdctl默认v2版本
添加环境变量,使用v3版本命令行
[root@node001 ~]# export ETCDCTL_API=3

2.5 etcdctl常用命令

创建键值对

1
2
3
4
5
6
7
[root@node001 ~]# etcdctl put aa 111
OK
[root@node001 ~]# etcdctl get aa
aa
111
[root@node001 ~]# etcdctl del aa
1

2.6 创建etcd快照

1
2
[root@node001 ~]# etcdctl snapshot save snap1.data
Snapshot saved at snap1.data

2.7 恢复快照

关闭集群

1
[root@node001 ~]# systemctl stop etcd

删除数据目录

1
[root@node001 ~]# rm -rf /var/lib/etcd/default.etcd

回复快照

1
2
[root@node001 ~]# etcdctl snapshot restore snap1.data --name="default" --data-dir=/var/lib/etcd/default.etcd --initial-cluster="default=http://localhost:2380"
2022-09-06 21:23:03.135401 I | etcdserver/membership: added member 8e9e05c52164694d [http://localhost:2380] to cluster cdf818194e3a8c32

修改属主

1
[root@node001 ~]# chown -R etcd.etcd /var/lib/etcd/

启动服务

1
[root@node001 ~]# systemctl start etcd

3. 多节点etcd集群

3.1 安装etcd

1
2
3
[root@node001 ~]# yum install etcd -y
[root@node002 ~]# yum install etcd -y
[root@node003 ~]# yum install etcd -y

3.2 etcd集群配置文件

第一个节点首次启动需要ETCD_INITIAL_CLUSTER_STATE=“new”

node001

1
2
3
4
5
6
7
8
9
10
[root@node001 ~]# cat /etc/etcd/etcd.conf 
ETCD_DATA_DIR="/var/lib/etcd/cluster.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.10.41:2380,http://localhost:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.10.41:2379,http://localhost:2379"
ETCD_NAME="etcd-41"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.10.41:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379,http://192.168.10.41:2379"
ETCD_INITIAL_CLUSTER="etcd-41=http://192.168.10.41:2380,etcd-42=http://192.168.10.42:2380,etcd-43=http://192.168.10.43:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

node002

1
2
3
4
5
6
7
8
9
10
[root@node002 ~]# cat /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/cluster.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.10.42:2380,http://localhost:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.10.42:2379,http://localhost:2379"
ETCD_NAME="etcd-42"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.10.42:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379,http://192.168.10.42:2379"
ETCD_INITIAL_CLUSTER="etcd-41=http://192.168.10.41:2380,etcd-42=http://192.168.10.42:2380,etcd-43=http://192.168.10.43:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

node003

1
2
3
4
5
6
7
8
9
10
[root@node003 ~]# cat /etc/etcd/etcd.conf
ETCD_DATA_DIR="/var/lib/etcd/cluster.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.10.43:2380,http://localhost:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.10.43:2379,http://localhost:2379"
ETCD_NAME="etcd-43"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.10.43:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379,http://192.168.10.43:2379"
ETCD_INITIAL_CLUSTER="etcd-41=http://192.168.10.41:2380,etcd-42=http://192.168.10.42:2380,etcd-43=http://192.168.10.43:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

参数详解

ETCD_NAME 节点名称,默认为default
ETCD_DATA_DIR 服务运行数据保存的路径
ETCD_LISTEN_PEER_URLS 监听的同伴通信的地址,比如http://ip:2380,如果有多个,使用逗号分隔。需要所有节点都能够访问,所以不要使用 localhost!
ETCD_LISTEN_CLIENT_URLS 监听的客户端服务地址
ETCD_ADVERTISE_CLIENT_URLS 对外公告的该节点客户端监听地址,这个值会告诉集群中其他节点。
ETCD_INITIAL_ADVERTISE_PEER_URLS 对外公告的该节点同伴监听地址,这个值会告诉集群中其他节点
ETCD_INITIAL_CLUSTER 集群中所有节点的信息,格式为
ETCD_INITIAL_CLUSTER_STATE 新建集群的时候,这个值为 new;假如加入已经存在的集群,这个值为existing。
ETCD_INITIAL_CLUSTER_TOKEN 集群的ID,多个集群的时候,每个集群的ID必须保持唯一

3.3 启动服务

1
2
3
[root@node001 ~]# systemctl start etcd
[root@node002 ~]# systemctl start etcd
[root@node003 ~]# systemctl start etcd

3.4 添加环境变量

使用v3版本读写数据

1
export ETCDCTL_API=3

3.5 查看集群

1
2
3
4
[root@node001 ~]# etcdctl member list
1fcc72ff7c9fcff9, started, etcd-42, http://192.168.10.42:2380, http://192.168.10.42:2379,http://localhost:2379
9ea06ab63b510bd0, started, etcd-43, http://192.168.10.43:2380, http://192.168.10.43:2379,http://localhost:2379
a711108a22efa59d, started, etcd-41, http://192.168.10.41:2380, http://192.168.10.41:2379,http://localhost:2379

3.6 创建快照

1
2
[root@node003 ~]# etcdctl snapshot save snap2.data
Snapshot saved at snap2.data

3.7 恢复快照

复制快照到所有节点

1
2
[root@node003 ~]# scp snap2.data 192.168.10.42:/root
[root@node003 ~]# scp snap2.data 192.168.10.41:/root

关闭服务

1
2
3
[root@node001 ~]#  systemctl stop etcd
[root@node002 ~]# systemctl stop etcd
[root@node003 ~]# systemctl stop etcd

删除数据

1
2
3
[root@node003 ~]# rm -rf /var/lib/etcd/cluster.etcd/
[root@node002 ~]# rm -rf /var/lib/etcd/cluster.etcd/
[root@node001 ~]# rm -rf /var/lib/etcd/cluster.etcd/

恢复快照

node003节点

1
2
3
4
[root@node003 ~]# etcdctl snapshot restore snap2.data --name etcd-43 --data-dir=/var/lib/etcd/cluster.etcd --initial-cluster=etcd-41=http://192.168.10.41:2380,etcd-42=http://192.168.10.42:2380,etcd-43=http://192.168.10.43:2380 --initial-advertise-peer-urls=http://192.168.10.43:2380
2022-09-06 22:19:59.686988 I | etcdserver/membership: added member 1fcc72ff7c9fcff9 [http://192.168.10.42:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:19:59.687020 I | etcdserver/membership: added member 9ea06ab63b510bd0 [http://192.168.10.43:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:19:59.687025 I | etcdserver/membership: added member a711108a22efa59d [http://192.168.10.41:2380] to cluster ef8bee9f9d5c4009

node002节点

1
2
3
4
5
[root@node002 ~]# etcdctl snapshot restore snap2.data --name etcd-42 --data-dir=/var/lib/etcd/cluster.etcd --initial-cluster=etcd-41=http://192.168.10.41:2380,etcd-42=http://192.168.10.42:2380,etcd-43=http://192.168.10.43:2380 --initial-advertise-peer-urls=http://192.168.10.42:2380
2022-09-06 22:21:25.372853 I | etcdserver/membership: added member 1fcc72ff7c9fcff9 [http://192.168.10.42:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:21:25.372900 I | etcdserver/membership: added member 9ea06ab63b510bd0 [http://192.168.10.43:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:21:25.372907 I | etcdserver/membership: added member a711108a22efa59d [http://192.168.10.41:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:19:59.687025 I | etcdserver/membership: added member a711108a22efa59d [http://192.168.10.41:2380] to cluster ef8bee9f9d5c4009

node001节点

1
2
3
4
[root@node001 ~]# etcdctl snapshot restore snap2.data --name etcd-41 --data-dir=/var/lib/etcd/cluster.etcd --initial-cluster=etcd-41=http://192.168.10.41:2380,etcd-42=http://192.168.10.42:2380,etcd-43=http://192.168.10.43:2380 --initial-advertise-peer-urls=http://192.168.10.41:2380
2022-09-06 22:22:17.243782 I | etcdserver/membership: added member 1fcc72ff7c9fcff9 [http://192.168.10.42:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:22:17.243815 I | etcdserver/membership: added member 9ea06ab63b510bd0 [http://192.168.10.43:2380] to cluster ef8bee9f9d5c4009
2022-09-06 22:22:17.243820 I | etcdserver/membership: added member a711108a22efa59d [http://192.168.10.41:2380] to cluster ef8bee9f9d5c4009

修改属主

1
2
3
[root@node003 ~]# chown -R etcd.etcd /var/lib/etcd/
[root@node002 ~]# chown -R etcd.etcd /var/lib/etcd/
[root@node001 ~]# chown -R etcd.etcd /var/lib/etcd/

启动服务

1
2
3
[root@node001 ~]# systemctl start etcd
[root@node002 ~]# systemctl start etcd
[root@node003 ~]# systemctl start etcd

4. 访问k8s的etcd

4.1 查看配置文件

1
[root@node001 ~]# cat /etc/kubernetes/manifests/etcd.yaml

4.2 创建数据

1
2
3
4
5
[root@node001 ~]# etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key put aaa 111
OK
[root@node001 ~]# etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get aaa
aaa
111