k8s接入ceph块存储

使用ceph-csi 实现

ceph服务端

1.创建一个池，用以为k8s提供块存储服务

[root@ceph01 ~]# ceph osd pool create rbd-k8s-pool 256 256 SSD_rule
pool 'rbd-k8s-pool' created

2.设置配额

[root@ceph01 ~]# ceph osd pool set-quota rbd-k8s-pool max_bytes 100G
set-quota max_bytes = 107374182400 for pool rbd-k8s-pool

3.关联应用

[root@ceph01 ~]# ceph osd pool application enable rbd-k8s-pool rbd
enabled application 'rbd' on pool 'rbd-k8s-pool'

4.初始化

rbd pool init rbd-k8s-pool

5.创建用户

[root@ceph01 ~]# ceph auth get-or-create client.kubernetes mon 'profile rbd' osd 'profile rbd pool=rbd-k8s-pool' mgr 'profile rbd pool=rbd-k8s-pool'
[client.kubernetes]
    key = AQCS6kFg0NRDIBAAorr8r5Oxiz1eYH61VvLVYA==

k8s节点

主节点执行以下步骤

1.下载配置文件

ceph-csi-3.2.0.zip

2.上传配置文件解压

unzip ceph-csi-3.2.0.zip

3.创建一个命名空间，用于管理ceph-csi

kubectl create ns ceph-csi

4.更改ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-config-map.yaml

首先获取集群信息(ceph管理节点执行)
- b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294为集群ID
- mon节点地址：192.168.1.69:6789,192.168.1.70:6789,192.168.1.71:6789

[root@ceph01 ~]# ceph mon dump
    dumped monmap epoch 2
    epoch 2
    fsid b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294
    last_changed 2021-02-22 14:36:08.199609
    created 2021-02-22 14:27:26.357269
    min_mon_release 14 (nautilus)
    0: [v2:192.168.1.69:3300/0,v1:192.168.1.69:6789/0] mon.ceph01
    1: [v2:192.168.1.70:3300/0,v1:192.168.1.70:6789/0] mon.ceph02
    2: [v2:192.168.1.71:3300/0,v1:192.168.1.71:6789/0] mon.ceph03

更改csi-config-map.yaml内容如下：

vim ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-config-map.yaml

内容参考如下

---
apiVersion: v1
kind: ConfigMap
data:
  config.json: |-
    [
      {
        "clusterID": "b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294",
        "monitors": [
          "192.168.1.69:6789",
          "192.168.1.70:6789",
          "192.168.1.71:6789"
        ]
      }
    ]
metadata:
  name: ceph-csi-config

5.创建csi-config-map

kubectl -n ceph-csi apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-config-map.yaml

6.创建csi-rbd-secret

创建

cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbd-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: csi-rbd-secret
  namespace: ceph-csi
stringData:
  userID: kubernetes
  userKey: AQCS6kFg0NRDIBAAorr8r5Oxiz1eYH61VvLVYA==
EOF

其中：AQCS6kFg0NRDIBAAorr8r5Oxiz1eYH61VvLVYA==可通过在ceph服务端执行ceph auth get client.kubernetes获取

发布

kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbd-secret.yaml

7.配置清单中的namespace改成ceph-csi

sed -i "s/namespace: default/namespace: ceph-csi/g" $(grep -rl "namespace: default" ./ceph-csi-3.2.0/deploy/rbd/kubernetes)
sed -i -e "/^kind: ServiceAccount/{N;N;a\  namespace: ceph-csi  # 输入到这里的时候需要按一下回车键，在下一行继续输入
}" $(egrep -rl "^kind: ServiceAccount" ./ceph-csi-3.2.0/deploy/rbd/kubernetes)

8.创建ServiceAccount和RBAC ClusterRole/ClusterRoleBinding资源对象

kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml

9.创建PodSecurityPolicy

kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-provisioner-psp.yaml
kubectl create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml

10.调整csi-rbdplugin-provisioner.yaml和csi-rbdplugin.yaml

将csi-rbdplugin.yaml中的kms部分配置注释掉

# vim ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin.yaml
...
- name: ceph-csi-encryption-kms-config
  mountPath: /etc/ceph-csi-encryption-kms-config/
...
...
- name: ceph-csi-encryption-kms-config
  configMap:
    name: ceph-csi-encryption-kms-config
...

11.将csi-rbdplugin-provisioner.yaml中的kms部分配置注释掉

# vim ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
...
- name: ceph-csi-encryption-kms-config
  mountPath: /etc/ceph-csi-encryption-kms-config/
...
...
- name: ceph-csi-encryption-kms-config
  configMap:
    name: ceph-csi-encryption-kms-config
...

12.将csi-rbdplugin.yaml中的image部分调整为可访问镜像地址

k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
quay.io/cephcsi/cephcsi:v3.2.0

13.将csi-rbdplugin-provisioner.yaml中的image部分调整为可访问镜像地址

quay.io/cephcsi/cephcsi:v3.2.0
k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4
k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2
k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
k8s.gcr.io/sig-storage/csi-resizer:v1.0.1

14.发布csi-rbdplugin-provisioner.yaml和csi-rbdplugin.yaml

kubectl -n ceph-csi create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
kubectl -n ceph-csi create -f ceph-csi-3.2.0/deploy/rbd/kubernetes/csi-rbdplugin.yaml

15.查看运行状态

[root@ceph01 ~]# kubectl get pod -n ceph-csi
NAME                                         READY   STATUS    RESTARTS   AGE
csi-rbdplugin-ddc42                          3/3     Running   0          76s
csi-rbdplugin-fwwfv                          3/3     Running   0          76s
csi-rbdplugin-provisioner-76959bd74d-gwd9k   7/7     Running   0          5h32m
csi-rbdplugin-provisioner-76959bd74d-nb574   7/7     Running   0          5h32m

16.创建StorageClass

b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294为ceph集群ID注意替换

生成配置文件

cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/storageclass.yaml
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ceph-csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:
  clusterID: b1c2511e-a1a5-4d6d-a4be-0e7f0d6d4294
  pool: rbd-k8s-pool
  imageFeatures: layering
  csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
  csi.storage.k8s.io/provisioner-secret-namespace: ceph-csi
  csi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secret
  csi.storage.k8s.io/controller-expand-secret-namespace: ceph-csi
  csi.storage.k8s.io/node-stage-secret-name: csi-rbd-secret
  csi.storage.k8s.io/node-stage-secret-namespace: ceph-csi
  csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
  - discard
EOF

创建

kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/storageclass.yaml

配置为默认storage class

kubectl patch storageclass ceph-csi-rbd-sc -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

查看storage class

[root@ceph01 ~]# kubectl get sc
NAME                        PROVISIONER        RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
ceph-csi-rbd-sc (default)   rbd.csi.ceph.com   Delete          Immediate              true                   117s

17.创建pvc验证可用性

生成配置

cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/pvc-demo.yaml 
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: ceph-pvc-demo
  namespace: default
spec:
  storageClassName: ceph-csi-rbd-sc
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
EOF

创建

kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/pvc-demo.yaml

查看

[root@ceph01 ~]# kubectl get pvc
    NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
    ceph-pvc-demo   Bound    pvc-7b7d4d8a-c4f4-40b6-9372-661ece7c385e   1Gi        RWO            ceph-csi-rbd-sc   13s

18.pvc扩容

生成配置

cat <<EOF > ceph-csi-3.2.0/deploy/rbd/kubernetes/nginx-demo.yaml 
apiVersion: v1
kind: Pod
metadata:
  name: testpv
  labels:
    role: web-frontend
spec:
  containers:
  - name: web
    image: nginx
    ports:
      - name: web
        containerPort: 80
    volumeMounts:
      - name: ceph-pvc-demo
        mountPath: "/usr/share/nginx/html"
  volumes:
  - name: ceph-pvc-demo
    persistentVolumeClaim:
      claimName: ceph-pvc-demo
EOF

发布

kubectl apply -f ceph-csi-3.2.0/deploy/rbd/kubernetes/nginx-demo.yaml

查看pvc

[root@ceph01 ~]# kubectl get pvc
NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
ceph-pvc-demo   Bound    pvc-360f8c5b-0f82-4f17-957b-a7eb5cf93f7e   1Gi        RWO            ceph-csi-rbd-sc   2m50s

编辑修改pvc

kubectl edit pvc ceph-pvc-demo

修改以下内容，storage: 1Gi调整为storage: 10Gi

...
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
...

重启

kubectl get pod testpv -o yaml | kubectl replace --force -f -

再次查看pvc

[root@ceph01 ~]# kubectl get pvc
NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE
ceph-pvc-demo   Bound    pvc-360f8c5b-0f82-4f17-957b-a7eb5cf93f7e   10Gi       RWO            ceph-csi-rbd-sc   9m26s

02块设备类型存储使用

k8s接入ceph块存储

ceph服务端

k8s节点

results matching ""

No results matching ""