k8s接入ceph块存储
使用ceph-csi 实现
ceph服务端
1.创建一个池,用以为
k8s
提供块存储服务
[root@ceph01 ~]# ceph osd pool create rbd-k8s-pool 256 256 SSD_rule
pool 'rbd-k8s-pool' created
2.设置配额
[root@ceph01 ~]# ceph osd pool set-quota rbd-k8s-pool max_bytes 100G
set-quota max_bytes = 107374182400 for pool rbd-k8s-pool
3.关联应用
[root@ceph01 ~]# ceph osd pool application enable rbd-k8s-pool rbd
enabled application 'rbd' on pool 'rbd-k8s-pool'
4.初始化
rbd pool init rbd-k8s-pool
5.创建用户
[root@ceph01 ~]# ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=rbd-k8s-pool' mgr 'profile rbd pool=rbd-k8s-pool'
[client.kubernetes]
key = AQCS6kFg0NRDIBAAorr8r5Oxiz1eYH61VvLVYA==
k8s节点
主节点执行以下步骤
1.下载配置文件
2.上传配置文件解压
unzip ceph-csi-3.2.0.zip
3.创建一个命名空间,用于管理
ceph-csi
kubectl create ns ceph-csi
4.更改
ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-config-map.yaml
- 首先获取集群信息(
ceph
管理节点执行)b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294
为集群ID
mon
节点地址:192.168.1.69:6789,192.168.1.70:6789,192.168.1.71:6789
[root@ceph01 ~]# ceph mon dump
dumped monmap epoch 2
epoch 2
fsid b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294
last_changed 2021-02-22 14:36:08.199609
created 2021-02-22 14:27:26.357269
min_mon_release 14 (nautilus)
0: [v2:192.168.1.69:3300/0,v1:192.168.1.69:6789/0] mon.ceph01
1: [v2:192.168.1.70:3300/0,v1:192.168.1.70:6789/0] mon.ceph02
2: [v2:192.168.1.71:3300/0,v1:192.168.1.71:6789/0] mon.ceph03
更改csi-config-map.yaml
内容如下:
vim ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-config-map.yaml
内容参考如下
---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-
[
{
"clusterID": "b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294",
"monitors": [
"192.168.1.69:6789",
"192.168.1.70:6789",
"192.168.1.71:6789"
]
}
]
metadata:
name: ceph-csi-config
5.创建
csi-config-map
kubectl -n ceph-csi apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-config-map.yaml
6.创建
csi-rbd-secret
创建
cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbd-secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: csi-rbd-secret
namespace: ceph-csi
stringData:
userID: kubernetes
userKey: AQCS6kFg0NRDIBAAorr8r5Oxiz1eYH61VvLVYA==
EOF
其中:AQCS6kFg0NRDIBAAorr8r5Oxiz1eYH61VvLVYA==
可通过在ceph
服务端执行ceph auth get client.kubernetes
获取
- 发布
kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbd-secret.yaml
7.配置清单中的
namespace
改成ceph-csi
sed -i "s/namespace: default/namespace: ceph-csi/g" $(grep -rl "namespace: default" ./ceph-csi-3.2.0/deploy/rbd/kubernetes)
sed -i -e "/^kind: ServiceAccount/{N;N;a\ namespace: ceph-csi # 输入到这里的时候需要按一下回车键,在下一行继续输入
}" $(egrep -rl "^kind: ServiceAccount" ./ceph-csi-3.2.0/deploy/rbd/kubernetes)
8.创建
ServiceAccount
和RBAC ClusterRole/ClusterRoleBinding
资源对象
kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
9.创建
PodSecurityPolicy
kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-provisioner-psp.yaml
kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml
10.调整
csi-rbdplugin-provisioner.yaml
和csi-rbdplugin.yaml
- 将
csi-rbdplugin.yaml
中的kms
部分配置注释掉
# vim ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin.yaml
...
- name: ceph-csi-encryption-kms-config
mountPath: /etc/ceph-csi-encryption-kms-config/
...
...
- name: ceph-csi-encryption-kms-config
configMap:
name: ceph-csi-encryption-kms-config
...
11.将
csi-rbdplugin-provisioner.yaml
中的kms
部分配置注释掉
# vim ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
...
- name: ceph-csi-encryption-kms-config
mountPath: /etc/ceph-csi-encryption-kms-config/
...
...
- name: ceph-csi-encryption-kms-config
configMap:
name: ceph-csi-encryption-kms-config
...
12.将
csi-rbdplugin.yaml
中的image
部分调整为可访问镜像地址
k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
quay.io/cephcsi/cephcsi:v3.2.0
13.将
csi-rbdplugin-provisioner.yaml
中的image
部分调整为可访问镜像地址
quay.io/cephcsi/cephcsi:v3.2.0
k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4
k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2
k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
k8s.gcr.io/sig-storage/csi-resizer:v1.0.1
14.发布
csi-rbdplugin-provisioner.yaml
和csi-rbdplugin.yaml
kubectl -n ceph-csi create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
kubectl -n ceph-csi create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin.yaml
15.查看运行状态
[root@ceph01 ~]# kubectl get pod -n ceph-csi
NAME READY STATUS RESTARTS AGE
csi-rbdplugin-ddc42 3/3 Running 0 76s
csi-rbdplugin-fwwfv 3/3 Running 0 76s
csi-rbdplugin-provisioner-76959bd74d-gwd9k 7/7 Running 0 5h32m
csi-rbdplugin-provisioner-76959bd74d-nb574 7/7 Running 0 5h32m
16.创建
StorageClass
b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294
为ceph
集群ID
注意替换
- 生成配置文件
cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/storageclass.yaml
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ceph-csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
clusterID: b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294
pool: rbd-k8s-pool
imageFeatures: layering
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi
csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi
csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi
csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
- discard
EOF
- 创建
kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/storageclass.yaml
- 配置为默认
storage class
kubectl patch storageclass ceph-csi-rbd-sc -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
- 查看
storage class
[root@ceph01 ~]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
ceph-csi-rbd-sc (default) rbd.csi.ceph.com Delete Immediate true 117s
17.创建
pvc
验证可用性
- 生成配置
cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/pvc-demo.yaml
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: ceph-pvc-demo
namespace: default
spec:
storageClassName: ceph-csi-rbd-sc
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
EOF
- 创建
kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/pvc-demo.yaml
- 查看
[root@ceph01 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc-demo Bound pvc-7b7d4d8a-c4f4-40b6-9372-661ece7c385e 1Gi RWO ceph-csi-rbd-sc 13s
18.
pvc
扩容
- 生成配置
cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/nginx-demo.yaml
apiVersion: v1
kind: Pod
metadata:
name: testpv
labels:
role: web-frontend
spec:
containers:
- name: web
image: nginx
ports:
- name: web
containerPort: 80
volumeMounts:
- name: ceph-pvc-demo
mountPath: "/usr/share/nginx/html"
volumes:
- name: ceph-pvc-demo
persistentVolumeClaim:
claimName: ceph-pvc-demo
EOF
- 发布
kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/nginx-demo.yaml
- 查看
pvc
[root@ceph01 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc-demo Bound pvc-360f8c5b-0f82-4f17-957b-a7eb5cf93f7e 1Gi RWO ceph-csi-rbd-sc 2m50s
- 编辑修改
pvc
kubectl edit pvc ceph-pvc-demo
- 修改以下内容,
storage: 1Gi
调整为storage: 10Gi
...
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
...
- 重启
kubectl get pod testpv -o yaml | kubectl replace --force -f -
- 再次查看
pvc
[root@ceph01 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ceph-pvc-demo Bound pvc-360f8c5b-0f82-4f17-957b-a7eb5cf93f7e 10Gi RWO ceph-csi-rbd-sc 9m26s