1. 简介

利用kubernetes自带的一些功能及第三方的工具来提升kubernetes的安全性

2. PSP

pod安全策略

2.1 启用psp

1
- --enable-admission-plugins=NodeRestriction,PodSecurityPolicy

一旦启用了PSP,且没有任何PSP规则的情况下,任何用户都不能再创建pod,包括管理员

2.2 创建规则

2.2.1 不允许创建特权pod

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp1
spec:
privileged: false
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
runAsUser:
rule: RunAsAny
fsGroup:
rule: RunAsAny
volumes:
- "*"

此时普通用户无法访问到psp,因此仍然无法创建pod

2.2.2 创建权限

1
kubectl create clusterrole crole2 --verb use --resource=psp --resource-name psp1
1
kubectl create clusterrolebinding crolebind2 --clusterrole crole2 --user ren
1
kubectl create clusterrole psp-allow-kube-system --verb=use --resource=psp --resource-name=psp-allow-all
1
kubectl create rolebinding rolebind-allow-psp-kube-system --clusterrole=psp-allow-kube-system --group=system:serviceaccounts -n kube-system

2.2.2 允许使用宿主机网络空间

默认没写,不允许使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp1
spec:
privileged: false
hostNetwork: true
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
runAsUser:
rule: RunAsAny
fsGroup:
rule: RunAsAny
volumes:
- "*"

2.2.3 只允许使用特定类型的卷

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp1
spec:
privileged: false
hostNetwork: true
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
runAsUser:
rule: RunAsAny
fsGroup:
rule: RunAsAny
volumes:
- emptyDir
- projected
- hostPath

2.2.4 hostPath限制宿主机目录

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp1
spec:
privileged: false
hostNetwork: true
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
runAsUser:
rule: RunAsAny
fsGroup:
rule: RunAsAny
volumes:
- emptyDir
- projected
- hostPath
allowedHostPath:
- pathPrefix: "/tmp"

2.2.5 以特定用户运行pod

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp1
spec:
privileged: false
hostNetwork: true
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
runAsUser:
rule: 'MustRunAs'
ranges:
- min: 1000
- max: 1500
fsGroup:
rule: RunAsAny
volumes:
- emptyDir
- projected
- hostPath
allowedHostPath:
- pathPrefix: "/tmp"

以普通用户运行pod容器进程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
run: pod3
name: pod3
spec:
nodeName: vms41.rhce.cc
terminationGracePeriodSeconds: 0
containers:
- image: hub.c.163.com/library/centos
imagePullPolicy: IfNotPresent
name: pod3
resources: {}
securityContext:
runAsUser: 1000
command: ["/bin/sh","-c","sleep 1d"]
dnsPolicy: ClusterFirst
restartPolicy: Always
status: {}

2.2.6 hostPorts范围限制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp1
spec:
privileged: false
hostNetwork: true
hostPorts:
- min: 0
- max: 8880
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
runAsUser:
rule: 'MustRunAs'
ranges:
- min: 1000
- max: 1500
fsGroup:
rule: RunAsAny
volumes:
- emptyDir
- projected
- hostPath
allowedHostPath:
- pathPrefix: "/tmp"

3. OPA

Open Policy Agent
也是基于准入控制器–开发出来的一个功能,不是kubernetes自带的,而是第三方的,OPA并非只能用于kubernetes,可以用于很多地方,比如ssh,sudo,docker等

3.1 OPA和docker整合

3.1.1 创建策略目录

1
mkdir -p /etc/docker/policies

3.1.2 安装策略插件

1
docker plugin install openpolicyagent/opa-docker-authz-v2:0.4 opa-args="-policy-file /opa/policies/authz.rego"

3.1.3 查看安装的插件

1
2
3
root@node-0:~#  docker plugin list
ID NAME DESCRIPTION ENABLED
a45c554aca67 openpolicyagent/opa-docker-authz-v2:0.4 A policy-enabled authorization plugin for Do… true

3.1.4 启用opa

1
2
3
4
5
6
7
root@node-0:~# cat /etc/docker/daemon.json
{
"registry-mirrors": [
"https://frz7i079.mirror.aliyuncs.com"
],
"authorization-plugins": ["openpolicyagent/opa-docker-authz-v2:0.4"]
}

3.1.5 创建rego规则

官方文档:https://www.openpolicyagent.org/docs/latest/docker-authorization/

3.1.5.1 创建策略

拒绝开启所有系统调用
拒绝创建特权容器

cat /etc/docker/policies/authz.rego

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
package docker.authz

default allow = false

allow {
not deny
}

deny {
seccomp_unconfined
}

seccomp_unconfined {
input.Body.HostConfig.SecurityOpt[_] == "seccomp=unconfined"
input.Body.HostConfig.Privileged == true

}

input.Body是固定的,后面的是docker inspect 查看到的

3.1.5.1 登录用户策略

限制harbor登录用户

bob用户只读权限

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
root@node-0:~# cat /etc/docker/policies/authz.rego
package docker.authz
default allow = false

allow {
user_id := input.Headers["Authz-User"]
user := users[user_id]
not user.readOnly
}

allow {
user_id := input.Headers["Authz-User"]
users[user_id].readOnly
input.Method == "GET"
}

users = {
"bob": {"readOnly": true},
"alice": {"readOnly": false},
}

3.1 kubernetes中使用OPA

gatekeeper项目是Kubernetes基于OPA的实现。gatekeeper允许我们以Kubernetes本机方式使用OPA来实施所需的策略
安装gatekeeper之后,会生成constrainttemplates.templates.gatekeeper.sh 这个CRD
然后利用这个CRD 可以创建template
然后创建constraint

下载:https://github.com/open-policy-agent/gatekeeper/tree/master/deploy

3.1.1 安装gatekeeper

1
kubectl apply -f gatekeeper.yaml

3.1.2 查看crd

1
2
3
4
5
6
7
root@vms40:~/gatekeeper# kubectl get crd|grep gatekeeper
configs.config.gatekeeper.sh 2022-09-26T05:38:20Z
constraintpodstatuses.status.gatekeeper.sh 2022-09-26T05:38:20Z
constrainttemplatepodstatuses.status.gatekeeper.sh 2022-09-26T05:38:20Z
constrainttemplates.templates.gatekeeper.sh 2022-09-26T05:38:20Z
providers.externaldata.gatekeeper.sh 2022-09-26T05:38:20Z

3.1.3 自定义资源类型

3.1.3.1 拒绝特定镜像

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
name: blacklistimages
spec:
crd:
spec:
names:
kind: BlacklistImages
targets:
- rego: |
package k8strustedimages

images {
image := input.review.object.spec.containers[_].image
not startswith(image, "docker-fake.io/")
not startswith(image, "google-gcr-fake.com/")
}

violation[{"msg": msg}] {
not images
msg := "not trusted image!"
}
target: admission.k8s.gatekeeper.sh

3.1.3.2 创建资源

1
kubectl apply -f gatekeeper-tmp.yaml 

3.1.3.3 查看crd

1
2
root@vms40:~/gatekeeper# kubectl get crd|grep blacklistimages
blacklistimages.constraints.gatekeeper.sh 2022-09-26T05:51:28

3.1.3.4 创建资源类型对象

应用到pod上

1
2
3
4
5
6
7
8
9
10
11
12
13
14
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: BlacklistImages
metadata:
generation: 1
managedFields:
name: pod-trusted-images
resourceVersion: "14449"
spec:
match:
kinds:
- apiGroups:
- ""
kinds:
- Pod

4. 沙箱

容器化部署,即使宿主机里没有安装mysql、启动mysql,但是在宿主机里仍然能够看到mysqld的进程,为了安全性,在宿主机里能否隔离容器里运行的容器

4.1 gvisor运行时

下载: https://gvisor.dev/docs/user_guide/install

gvisor运行时沙箱运行容器,宿主机看不到相关进程

4.1.1 安装

1
2
root@node-0:/usr/local/bin# ./runsc  install
2022/09/26 15:24:07 Added runtime "runsc" with arguments [] to "/etc/docker/daemon.json".

4.1.2 重启docker

1
systemctl restart docker

4.1.3 创建容器

指定运行时

1
docker run -itd --name=web1 --restart=always --runtime=runsc nginx

4.1.4 宿主机查看进程

1
2
root@node-0:/usr/local/bin# ps -ef |grep nginx
root 14442 115329 0 15:29 pts/2 00:00:00 grep --color=auto nginx

4.1.5 containerd中使用gvisor运行时

4.1.5.1 安装运行时

1
chmod +x /usr/local/bin/containerd-shim-runsc-v1

4.1.5.2 添加配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
runtime_type = "io.containerd.runsc.v1"

4.1.5.3 重启containerd

1
systemctl restart containerd

4.1.5.4 查看info

crictl info 查看已经有2个运行时

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
"runtimes": {
"runc": {
"runtimeType": "io.containerd.runc.v2",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": [],
"ContainerAnnotations": [],
"runtimeRoot": "",
"options": {
"SystemdCgroup": true
},
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0
},
"runsc": {
"runtimeType": "io.containerd.runsc.v1",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": null,
"ContainerAnnotations": null,
"runtimeRoot": "",
"options": null,
"privileged_without_host_devices": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0
}
},

4.1.5.5 创建容器

1
nerdctl run -d --name=web1 --restart=always --runtime=runsc nginx

4.1.5.6 宿主机查看

宿主机是没有nginx进程的

1
ps -ef |grep nginx

4.1 kata运行时

使用kata创建出来的已经不是容器了,而是一个传统上虚拟机–kvm 胖pod
kvm–硬件辅助虚拟化,必须开启cpu的虚拟化功能

4.1 在k8s里配置gvisor

4.1.1 修改containerd配置

修改 /etc/containerd/config.toml
[plugins.“io.containerd.grpc.v1.cri”.containerd.runtimes]
[plugins.“io.containerd.grpc.v1.cri”.containerd.runtimes.runc]
runtime_type = “io.containerd.runc.v2”
在此条目下已经有了runc的配置,添加runsc的配置
[plugins.“io.containerd.grpc.v1.cri”.containerd.runtimes.runsc]
runtime_type = “io.containerd.runsc.v1”
此时containerd上支持了两种runtime,分别是runc和run

4.1.2 修改kubelet参数,让其支持runsc作为runtime

cat > /etc/systemd/system/kubelet.service.d/0-cri-containerd.conf <<EOF
[Service]
Environment=“KUBELET_EXTRA_ARGS=–container-runtime=remote --runtime-request-timeout=15m
–container-runtime-endpoint=unix:///run/containerd/containerd.sock”
EOF

4.1.3 创建runtimeClass

1
2
3
4
5
6
cat rc.yaml
apiVersion: node.k8s.io/v1beta1
kind: RuntimeClass
metadata:
name: myclass
handler: runsc # 对应的 CRI 配置的名称

4.1.4 创建pod

指定runc

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
cat pod1.yaml
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
run: pod1
name: pod1
spec:
terminationGracePeriodSeconds: 0
runtimeClassName: myclass
nodeName: vms73.rhce.cc
containers:
- image: nginx
imagePullPolicy: IfNotPresent
name: pod1
resources: {}
dnsPolicy: ClusterFirst
restartPolicy: Always
status: {}