K8S动态卷实现
OS: Ubuntu Server 22.04 LTS
一: nfs-subdir-external-provisioner实现
1) 前提条件
(1) 安装及配置nfs-kernel-server[即:NFS Server]于srv5节点
root@srv5:~# apt install nfs-kernel-server -y
root@srv5:~# mkdir /var/lib/nfs-share/
root@srv5:~# chmod 777 /var/lib/nfs-share/
root@srv5:~# vim /etc/exports
# /etc/exports: the access control list for filesystems which may be exported
# to NFS clients. See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4 gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check)
#
/var/lib/nfs-share *(rw,no_root_squash,no_all_squash,sync)
root@srv5:~# systemctl restart nfs-server
(2) Master及Worker节点安装nfs-common组建[即:NFS Client]
root@srv1:~# apt install nfs-common -y
root@srv2:~# apt install nfs-common -y
root@srv3:~# apt install nfs-common -y
2) 安装NFS Provisioner
root@srv1:~# kubectl get pvc
No resources found in default namespace.
root@srv1:~# kubectl get pv
No resources found
root@srv1:~# helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
"nfs-subdir-external-provisioner" has been added to your repositories
root@srv1:~# helm repo list
NAME URL
nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
################################################## 错误汇总 ##################################################
# 1. 因无法直接downloadnfs-subdir-external-provisioner镜像，所以需要单独先*载下**
# 2. 在k8s的Master节点确认镜像的版本
root@srv1:~# helm pull nfs-subdir-external-provisioner/nfs-subdir-external-provisioner
root@srv1:~# tar xfz nfs-subdir-external-provisioner-4.0.16.tgz
root@srv1:~# cat nfs-subdir-external-provisioner/values.yaml | grep -A 3 ^image:
image:
repository: k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner
tag: v4.0.2
pullPolicy: IfNotPresent
# 3. 对所有k8s节点操作，*载下**镜像---[以srv1举例]
root@srv1:~# crictl pull eipwork/nfs-subdir-external-provisioner:v4.0.2
Image is up to date for sha256:932b0bface75b80e713245d7c2ce8c44b7e127c075bd2d27281a16677c8efef3
root@srv1:~# ctr --namespace=k8s.io image tag \
docker.io/eipwork/nfs-subdir-external-provisioner:v4.0.2 \
k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
################################################## 汇总结束 ##################################################
root@srv1:~# helm install nfs-client-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
--set nfs.server=192.168.1.15 \
--set nfs.path=/var/lib/nfs-share \
--set storageClass.defaultClass=true
NAME: nfs-client-provisioner
LAST DEPLOYED: Sat Jun 25 16:52:41 2022
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
root@srv1:~# helm list
NAME NAMESPACE REVISION UPDATED STATUS
nfs-client-provisioner default 1 2022-06-25 16:52:41.570705978 +0800 CST deployed
CHART APP VERSION
nfs-subdir-external-provisioner-4.0.16 4.0.2
3) 确认
root@srv1:~# kubectl get pods
NAME READY STATUS RESTARTS AGE
nfs-client-provisioner-nfs-subdir-external-provisioner-7854gwn9 1/1 Running 0 41s
snow-nginx-56ccc94d85-x7spq 1/1 Running 1 (81m ago) 3d
二: 基本使用
1) 创建PVC及自动创建PV
root@srv1:~# vim chuai-pvc.yml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: snow-provisioner
spec:
accessModes:
- ReadWriteMany
storageClassName: nfs-client
resources:
requests:
storage: 1Gi
root@srv1:~# kubectl apply -f chuai-pvc.yml
persistentvolumeclaim/snow-provisioner created
2）确认创建情况
root@srv1:~# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
snow-provisioner Bound pvc-11a9fd7a-0259-4e2a-b269-f19a2484652f 1Gi RWX nfs-client 51m
root@srv1:~# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-11a9fd7a-0259-4e2a-b269-f19a2484652f 5Gi RWO Delete Bound default/my-provisioner nfs-client 15s
3) 测试
root@srv1:~# vim chuai-pod.yml
apiVersion: v1
kind: Pod
metadata:
name: snow-nginx
spec:
containers:
- name: snow-nginx
image: nginx
ports:
- containerPort: 80
name: web
volumeMounts:
- mountPath: /usr/share/nginx/html
name: nginx-pvc
volumes:
- name: nginx-pvc
persistentVolumeClaim:
claimName: snow-provisioner
root@srv1:~# kubectl apply -f chuai-pod.yml
pod/snow-nginx created
root@srv1:~# kubectl get pod snow-nginx -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
snow-nginx 1/1 Running 0 8s 10.244.2.12 srv2.1000y.cloud <none> <none>
4) 测试确认
root@srv1:~# kubectl exec snow-nginx -- df /usr/share/nginx/html
Filesystem 1K-blocks Used Available
192.168.1.15:/var/lib/nfs-share/default-snow-provisioner-pvc-11a9fd7a-0259-4e2a-b269-f19a2484652f 78892800 7102080 67737472
Use% Mounted on
10% /usr/share/nginx/html
root@srv1:~# echo "Nginx Index for 1000y.cloud" > index.html
root@srv1:~# kubectl cp index.html snow-nginx:/usr/share/nginx/html/index.html
root@srv1:~# curl 10.244.2.12
Nginx Index for 1000y.cloud
# NFS Server端确认
root@srv5:~# tree /var/lib/nfs-share/
/var/lib/nfs-share/
└── default-snow-provisioner-pvc-11a9fd7a-0259-4e2a-b269-f19a2484652f
└── index.html
1 directory, 1 file
5) 删除
root@srv1:~# kubectl delete pod snow-nginx
pod "snow-nginx" deleted
root@srv1:~# kubectl delete pvc snow-provisioner
persistentvolumeclaim "snow-provisioner" deleted
root@srv1:~# kubectl get pv
No resources found
root@srv1:~# kubectl get pvc
No resources found in default namespace.
6) 使用StatefulSet---volumeClaimTemplates方式
root@srv1:~# kubectl get pv
No resources found
root@srv1:~# kubectl get pvc
No resources found in default namespace.
root@srv1:~# kubectl get storageclass
NAME PROVISIONER RECLAIMPOLICY
nfs-client (default) cluster.local/nfs-client-provisioner-nfs-subdir-external-provisioner Delete
VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
Immediate true 7m27s
root@srv1:~# vim statefulset.yml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: snow-nginx
spec:
serviceName: snow-nginx
replicas: 1
selector:
matchLabels:
app: snow-nginx
template:
metadata:
labels:
app: snow-nginx
spec:
containers:
- name: snow-nginx
image: nginx
volumeMounts:
- name: data
mountPath: /usr/share/nginx/html
volumeClaimTemplates:
- metadata:
name: data
spec:
storageClassName: nfs-client
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi
root@srv1:~# kubectl apply -f statefulset.yml
statefulset.apps/snow-nginx created
root@srv1:~# kubectl get statefulset
NAME READY AGE
snow-nginx 1/1 11s
root@srv1:~# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nfs-client-provisioner-nfs-subdir-external-provisioner-7854gwn9 1/1 Running 0 10m 10.244.2.11 srv2.1000y.cloud <none> <none>
snow-nginx-0 1/1 Running 0 69s 10.244.2.13 srv2.1000y.cloud <none> <none>
snow-nginx-56ccc94d85-x7spq 1/1 Running 1 (91m ago) 3d 10.244.1.7 srv3.1000y.cloud <none> <none>
root@srv1:~# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
data-snow-nginx-0 Bound pvc-8a8db35a-9503-416d-b2db-15f479985f40 1Gi RWO nfs-client 2m38s
root@srv1:~# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-8a8db35a-9503-416d-b2db-15f479985f40 1Gi RWO Delete Bound default/data-snow-nginx-0 nfs-client 2m56s