K8S的PV和PVC介绍

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

apiVersion: v1
kind: Pod
metadata:
  name: configmap-pod
spec:
  containers:
    - name: test
      image: busybox
      volumeMounts:
        - name: config-vol
          mountPath: /etc/config
  volumes:
    - name: config-vol
      configMap:
        name: log-config
        items:
          - key: log_level
            path: log_level

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

[root@node131 k8s_pv_pvc]# kubectl get pvc -A
NAMESPACE   NAME                       STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
default     nfs-pv-provisioning-demo   Pending                                                     7s
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]# kubectl describe  pvc nfs-pv-provisioning-demo
Name:          nfs-pv-provisioning-demo
Namespace:     default
StorageClass:
Status:        Pending
Volume:
Labels:        demo=nfs-pv-provisioning
Annotations:   <none>
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:
Access Modes:
VolumeMode:    Filesystem
Used By:       <none>
Events:
  Type    Reason         Age               From                         Message
  ----    ------         ----              ----                         -------
  Normal  FailedBinding  5s (x3 over 27s)  persistentvolume-controller  no persistent volumes available for this claim and no storage class is set
[root@node131 k8s_pv_pvc]#

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv0003
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: slow
  mountOptions:
    - hard
    - nfsvers=4.1
  nfs:
    path: /tmp
    server: 192.168.182.131

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

[root@node131 ~]#
[root@node131 ~]# kubectl get pv
NAME   CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM         STORAGECLASS   REASON   AGE
nfs    1Mi        RWX            Retain           Bound    default/nfs                           5h58m
[root@node131 ~]
[root@node131 k8s_pv_pvc]# kubectl delete -f nfs/nfs-pvc.yaml
persistentvolumeclaim "nfs" deleted
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]#
[root@node131 ~]#
[root@node131 ~]#
[root@node131 ~]# kubectl get pv
NAME   CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS     CLAIM         STORAGECLASS   REASON   AGE
nfs    1Mi        RWX            Retain           Released   default/nfs                           6h1m
[root@node131 ~]#
[root@node131 k8s_pv_pvc]# kubectl create -f nfs/custom-nfs-pvc.yaml
persistentvolumeclaim/nfs created
[root@node131 k8s_pv_pvc]#
[root@node131 ~]#
[root@node131 ~]# kubectl get pvc
NAME   STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
nfs    Pending                                                     8s
[root@node131 ~]#
[root@node131 ~]#
[root@node131 ~]# kubectl describe pv
Name:            nfs
Labels:          <none>
Annotations:     pv.kubernetes.io/bound-by-controller: yes
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:
Status:          Released
Claim:           default/nfs
Reclaim Policy:  Retain
Access Modes:    RWX
VolumeMode:      Filesystem
Capacity:        1Mi
Node Affinity:   <none>
Message:
Source:
    Type:      NFS (an NFS mount that lasts the lifetime of a pod)
    Server:    10.233.16.102
    Path:      /exports
    ReadOnly:  false
Events:        <none>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: myclaim
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  resources:
    requests:
      storage: 8Gi
  storageClassName: slow
  selector:
    matchLabels:
      release: "stable"
    matchExpressions:
      - {key: environment, operator: In, values: [dev]}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

apiVersion: v1
kind: Pod
metadata:
  name: mypod
spec:
  containers:
    - name: myfrontend
      image: nginx
      volumeMounts:
      - mountPath: "/var/www/html"
        name: mypd
  volumes:
    - name: mypd
      persistentVolumeClaim:
        claimName: myclaim

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

// NewController creates a new PersistentVolume controller
controller := &PersistentVolumeController{
        volumes:                       newPersistentVolumeOrderedIndex(),
        claims:                        cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
        kubeClient:                    p.KubeClient,
        eventRecorder:                 eventRecorder,
        runningOperations:             goroutinemap.NewGoRoutineMap(true /* exponentialBackOffOnError */),
        cloud:                         p.Cloud,
        enableDynamicProvisioning:     p.EnableDynamicProvisioning,
        clusterName:                   p.ClusterName,
        createProvisionedPVRetryCount: createProvisionedPVRetryCount,
        createProvisionedPVInterval:   createProvisionedPVInterval,
        claimQueue:                    workqueue.NewNamed("claims"),
        volumeQueue:                   workqueue.NewNamed("volumes"),
        resyncPeriod:                  p.SyncPeriod,
        operationTimestamps:           metrics.NewOperationStartTimeCache(),
    }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

// persistentVolumeOrderedIndex is a cache.Store that keeps persistent volumes
// indexed by AccessModes and ordered by storage capacity.
type persistentVolumeOrderedIndex struct {
    store cache.Indexer
}

func newPersistentVolumeOrderedIndex() persistentVolumeOrderedIndex {
    return persistentVolumeOrderedIndex{cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{"accessmodes": accessModesIndexFunc})}
}

1
2
3
4
5
6
7

// syncVolume is the main controller method to decide what to do with a volume.
// It's invoked by appropriate cache.Controller callbacks when a volume is
// created, updated or periodically synced. We do not differentiate between
// these events.
func (ctrl *PersistentVolumeController) syncVolume(volume *v1.PersistentVolume) error {
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

// findBestMatchForClaim is a convenience method that finds a volume by the claim's AccessModes and requests for Storage
func (pvIndex *persistentVolumeOrderedIndex) findBestMatchForClaim(claim *v1.PersistentVolumeClaim, delayBinding bool) (*v1.PersistentVolume, error) {
    return pvIndex.findByClaim(claim, delayBinding)
}



// find returns the nearest PV from the ordered list or nil if a match is not found
func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *v1.PersistentVolumeClaim, delayBinding bool) (*v1.PersistentVolume, error) {
    // PVs are indexed by their access modes to allow easier searching.  Each
    // index is the string representation of a set of access modes. There is a
    // finite number of possible sets and PVs will only be indexed in one of
    // them (whichever index matches the PV's modes).
    //
    // A request for resources will always specify its desired access modes.
    // Any matching PV must have at least that number of access modes, but it
    // can have more.  For example, a user asks for ReadWriteOnce but a GCEPD
    // is available, which is ReadWriteOnce+ReadOnlyMany.
    //
    // Searches are performed against a set of access modes, so we can attempt
    // not only the exact matching modes but also potential matches (the GCEPD
    // example above).
    allPossibleModes := pvIndex.allPossibleMatchingAccessModes(claim.Spec.AccessModes)

    for _, modes := range allPossibleModes {
        volumes, err := pvIndex.listByAccessModes(modes)
        if err != nil {
            return nil, err
        }

        bestVol, err := pvutil.FindMatchingVolume(claim, volumes, nil /* node for topology binding*/, nil /* exclusion map */, delayBinding)
        if err != nil {
            return nil, err
        }

        if bestVol != nil {
            return bestVol, nil
        }
    }
    return nil, nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

// FindMatchingVolume goes through the list of volumes to find the best matching volume
// for the claim.
//
// This function is used by both the PV controller and scheduler.
//
// delayBinding is true only in the PV controller path.  When set, prebound PVs are still returned
// as a match for the claim, but unbound PVs are skipped.
//
// node is set only in the scheduler path. When set, the PV node affinity is checked against
// the node's labels.
//
// excludedVolumes is only used in the scheduler path, and is needed for evaluating multiple
// unbound PVCs for a single Pod at one time.  As each PVC finds a matching PV, the chosen
// PV needs to be excluded from future matching.
func FindMatchingVolume(
    claim *v1.PersistentVolumeClaim,
    volumes []*v1.PersistentVolume,
    node *v1.Node,
    excludedVolumes map[string]*v1.PersistentVolume,
    delayBinding bool) (*v1.PersistentVolume, error) {

    }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM centos
RUN yum -y install /usr/bin/ps nfs-utils && yum clean all
RUN mkdir -p /exports
ADD run_nfs.sh /usr/local/bin/
ADD index.html /tmp/index.html
RUN chmod 644 /tmp/index.html

# expose mountd 20048/tcp and nfsd 2049/tcp and rpcbind 111/tcp
EXPOSE 2049/tcp 20048/tcp 111/tcp 111/udp

ENTRYPOINT ["/usr/local/bin/run_nfs.sh", "/exports"]

1
2
3
4

# 制作镜像
docker build -t k8s.gcr.io/volume-nfs:0.8 .

docker save k8s.gcr.io/volume-nfs:0.8 -o volume-nfs-img.tar

1
2

# 在测试环境中导出镜像
docker load -i volume-nfs-img.tar

1
2
3

# If you are on GCE, create a GCE PD-based PVC:
# kubectl create -f examples/staging/volumes/nfs/provisioner/nfs-server-gce-pv.yaml
# kubectl create -f nfs/provisioner/nfs-server-gce-pv.yaml

1
2
3
4
5

# test pv
kubectl create -f nfs/test/pv0001.yaml

# test pvc
kubectl create -f nfs/test/pvc-pv0001.yaml

1
2
3
4

#kubectl create -f examples/staging/volumes/nfs/nfs-server-rc.yaml
#kubectl create -f examples/staging/volumes/nfs/nfs-server-service.yaml
kubectl create -f nfs/custom-nfs-server-rc.yaml
kubectl create -f nfs/nfs-server-service.yaml

1
2
3
4
5
6
7
8

Tolerations:     node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                 node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age   From               Message
  ----     ------            ----  ----               -------
  Warning  FailedScheduling  29s   default-scheduler  0/1 nodes are available: 1 persistentvolumeclaim "nfs-pv-provisioning-demo" not found.
  Warning  FailedScheduling  29s   default-scheduler  0/1 nodes are available: 1 persistentvolumeclaim "nfs-pv-provisioning-demo" not found.

1
2
3

kubectl describe services nfs-server

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

[root@node131 k8s_pv_pvc]# kubectl describe services nfs-server
Name:              nfs-server
Namespace:         default
Labels:            <none>
Annotations:       <none>
Selector:          role=nfs-server
Type:              ClusterIP
IP Families:       <none>
IP:                10.233.63.192
IPs:               10.233.63.192
Port:              nfs  2049/TCP
TargetPort:        2049/TCP
Endpoints:         10.233.124.46:2049
Port:              mountd  20048/TCP
TargetPort:        20048/TCP
Endpoints:         10.233.124.46:20048
Port:              rpcbind  111/TCP
TargetPort:        111/TCP
Endpoints:         10.233.124.46:111
Session Affinity:  None
Events:            <none>

1
2
3
4
5
6

#kubectl create -f examples/staging/volumes/nfs/nfs-pv.yaml
#kubectl create -f examples/staging/volumes/nfs/nfs-pvc.yaml

kubectl create -f nfs/custom-nfs-pv.yaml
kubectl create -f nfs/custom-nfs-pvc.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

[root@node131 k8s_pv_pvc]# kubectl create -f nfs/nfs-pv.yaml
persistentvolume/nfs created
[root@node131 k8s_pv_pvc]# kubectl get pv
NAME   CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
nfs    1Mi        RWX            Retain           Available                                   6s

[root@node131 k8s_pv_pvc]# kubectl describe pv nfs
Name:            nfs
Labels:          <none>
Annotations:     <none>
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:
Status:          Available
Claim:
Reclaim Policy:  Retain
Access Modes:    RWX
VolumeMode:      Filesystem
Capacity:        1Mi
Node Affinity:   <none>
Message:
Source:
    Type:      NFS (an NFS mount that lasts the lifetime of a pod)
    Server:    nfs-server.default.svc.cluster.local
    Path:      /tmp/data
    ReadOnly:  false
Events:        <none>
[root@node131 k8s_pv_pvc]#




[root@node131 k8s_pv_pvc]# kubectl create -f nfs/nfs-pvc.yaml
persistentvolumeclaim/nfs created
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]# kubectl get pv
NAME   CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM         STORAGECLASS   REASON   AGE
nfs    1Mi        RWX            Retain           Bound    default/nfs                           114s
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]# kubectl get pvc
NAME   STATUS   VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS   AGE
nfs    Bound    nfs      1Mi        RWX                           16s
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]#

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

[root@node131 k8s_pv_pvc]# kubectl describe pv nfs
Name:            nfs
Labels:          <none>
Annotations:     pv.kubernetes.io/bound-by-controller: yes
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:
Status:          Bound
Claim:           default/nfs
Reclaim Policy:  Retain
Access Modes:    RWX
VolumeMode:      Filesystem
Capacity:        1Mi
Node Affinity:   <none>
Message:
Source:
    Type:      NFS (an NFS mount that lasts the lifetime of a pod)
    Server:    nfs-server.default.svc.cluster.local
    Path:      /tmp/data
    ReadOnly:  false
Events:        <none>
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]#
[root@node131 k8s_pv_pvc]# kubectl describe pvc nfs
Name:          nfs
Namespace:     default
StorageClass:
Status:        Bound
Volume:        nfs
Labels:        <none>
Annotations:   pv.kubernetes.io/bind-completed: yes
               pv.kubernetes.io/bound-by-controller: yes
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      1Mi
Access Modes:  RWX
VolumeMode:    Filesystem
Used By:       <none>
Events:        <none>
[root@node131 k8s_pv_pvc]#

1
2
3

#kubectl create -f examples/staging/volumes/nfs/nfs-busybox-rc.yaml
kubectl create -f nfs/custom-nfs-busybox-rc.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

[root@node131 nfs]# kubectl get po -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
default       nfs-busybox-mntdb                          1/1     Running   0          35s
default       nfs-busybox-r76ml                          1/1     Running   0          35s
default       nfs-server-p6p4n                           1/1     Running   1          134m
kube-system   calico-kube-controllers-65b86747bd-c4qsp   1/1     Running   10         41d
kube-system   calico-node-lglh4                          1/1     Running   11         41d
kube-system   coredns-8677555d68-flqh4                   1/1     Running   1          90m
kube-system   kube-apiserver-node131                     1/1     Running   10         41d
kube-system   kube-controller-manager-node131            1/1     Running   10         41d
kube-system   kube-proxy-mktp9                           1/1     Running   10         41d
kube-system   kube-scheduler-node131                     1/1     Running   10         41d
kube-system   nodelocaldns-lfjzs                         1/1     Running   10         41d
[root@node131 nfs]#




[root@node131 nfs]# cat /tmp/data/index.html
Mon Feb  1 09:59:49 UTC 2021
nfs-busybox-9td8j
[root@node131 nfs]# cat /tmp/data/index.html
Mon Feb  1 09:59:49 UTC 2021
nfs-busybox-9td8j
[root@node131 nfs]#
[root@node131 nfs]#
[root@node131 nfs]#
[root@node131 nfs]#
[root@node131 nfs]# cat /tmp/data/index.html
Mon Feb  1 09:59:57 UTC 2021
nfs-busybox-9td8j
[root@node131 nfs]#


[root@node131 nfs]# kubectl exec -ndefault       nfs-busybox-r76ml -- cat /mnt/index.html
Mon Feb  1 10:17:58 UTC 2021
nfs-busybox-mntdb
[root@node131 nfs]#

1
2
3
4
5
6

#kubectl create -f examples/staging/volumes/nfs/nfs-web-rc.yaml
#kubectl create -f examples/staging/volumes/nfs/nfs-web-service.yaml

kubectl create -f nfs/nfs-web-rc.yaml
kubectl create -f nfs/nfs-web-service.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

[root@node131 nfs]# kubectl get po -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
default       nfs-busybox-mntdb                          1/1     Running   0          35s
default       nfs-busybox-r76ml                          1/1     Running   0          35s
default       nfs-server-p6p4n                           1/1     Running   1          134m
default       nfs-web-dhmst                              1/1     Running   0          8m19s
default       nfs-web-lxlcr                              1/1     Running   0          8m19s
kube-system   calico-kube-controllers-65b86747bd-c4qsp   1/1     Running   10         41d
kube-system   calico-node-lglh4                          1/1     Running   11         41d
kube-system   coredns-8677555d68-flqh4                   1/1     Running   1          90m
kube-system   kube-apiserver-node131                     1/1     Running   10         41d
kube-system   kube-controller-manager-node131            1/1     Running   10         41d
kube-system   kube-proxy-mktp9                           1/1     Running   10         41d
kube-system   kube-scheduler-node131                     1/1     Running   10         41d
kube-system   nodelocaldns-lfjzs                         1/1     Running   10         41d
[root@node131 nfs]#
[root@node131 nfs]#



[root@node131 nfs]# kubectl get svc
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
kubernetes   ClusterIP   10.233.0.1      <none>        443/TCP                      41d
nfs-server   ClusterIP   10.233.63.192   <none>        2049/TCP,20048/TCP,111/TCP   140m
nfs-web      ClusterIP   10.233.63.218   <none>        80/TCP                       15m
[root@node131 nfs]#
[root@node131 nfs]#

[root@node131 nfs]# kubectl exec nfs-web-dhmst -- cat /usr/share/nginx/html/index.html
Mon Feb  1 10:06:54 UTC 2021
nfs-busybox-9td8j
[root@node131 nfs]#

1

kubectl exec -ndefault       nfs-busybox-mntdb -- wget -qO- http://nfs-web

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

[root@node131 nfs]#
[root@node131 nfs]# kubectl exec -ndefault       nfs-busybox-mntdb -- wget -qO- http://nfs-web
Mon Feb  1 10:24:37 UTC 2021
nfs-busybox-mntdb
[root@node131 nfs]#
[root@node131 nfs]#
[root@node131 nfs]#
[root@node131 nfs]# kubectl exec -ndefault       nfs-busybox-mntdb -- wget -qO- http://nfs-web
Mon Feb  1 10:24:41 UTC 2021
nfs-busybox-r76ml
[root@node131 nfs]#

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

[root@node131 ~]# kubectl describe pv
Name:            nfs
Labels:          <none>
Annotations:     pv.kubernetes.io/bound-by-controller: yes
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:
Status:          Bound
Claim:           default/nfs
Reclaim Policy:  Retain
Access Modes:    RWX
VolumeMode:      Filesystem
Capacity:        1Mi
Node Affinity:   <none>
Message:
Source:
    Type:      NFS (an NFS mount that lasts the lifetime of a pod)
    Server:    10.233.16.102
    Path:      /exports
    ReadOnly:  false
Events:        <none>
[root@node131 ~]#
[root@node131 ~]#
[root@node131 ~]# kubectl describe pvc
Name:          nfs
Namespace:     default
StorageClass:
Status:        Bound
Volume:        nfs
Labels:        <none>
Annotations:   pv.kubernetes.io/bind-completed: yes
               pv.kubernetes.io/bound-by-controller: yes
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      1Mi
Access Modes:  RWX
VolumeMode:    Filesystem
Used By:       nfs-busybox-b8wdw
               nfs-busybox-hvc8w
               nfs-web-7bnhj
               nfs-web-mxpx9
Events:        <none>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# create nfs server
kubectl create -f nfs/custom-nfs-server-rc.yaml
kubectl create -f nfs/nfs-server-service.yaml


# create nfs pv pvc
kubectl create -f nfs/custom-nfs-pv.yaml
kubectl create -f nfs/custom-nfs-pvc.yaml


# create busybox write
kubectl create -f nfs/custom-nfs-busybox-rc.yaml

# create web read
kubectl create -f nfs/nfs-web-rc.yaml
kubectl create -f nfs/nfs-web-service.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# remove web read
kubectl delete -f nfs/nfs-web-service.yaml
kubectl delete -f nfs/nfs-web-rc.yaml

# remove busybox write
kubectl delete -f nfs/custom-nfs-busybox-rc.yaml
# remove nfs pv pvc
kubectl delete -f nfs/nfs-pvc.yaml
kubectl delete -f nfs/custom-nfs-pv.yaml

# remove nfs server
kubectl delete -f nfs/nfs-server-service.yaml
kubectl delete -f nfs/custom-nfs-server-rc.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

Events:
  Type     Reason       Age                From               Message
  ----     ------       ----               ----               -------
  Normal   Scheduled    26s                default-scheduler  Successfully assigned default/nfs-busybox-jwshl to node131
  Warning  FailedMount  11s (x6 over 27s)  kubelet            MountVolume.SetUp failed for volume "nfs" : mount failed: exit status 32
Mounting command: mount
Mounting arguments: -t nfs nfs-server.default.svc.cluster.local:/tmp/data /var/lib/kubelet/pods/d4ca8ca9-e1c8-4610-8a74-f0fdea827fee/volumes/kubernetes.io~nfs/nfs
Output: mount: 文件系统类型错误、选项错误、nfs-server.default.svc.cluster.local:/tmp/data 上有坏超级块、
       缺少代码页或助手程序，或其他错误
       (对某些文件系统(如 nfs、cifs) 您可能需要
       一款 /sbin/mount.<类型> 助手程序)

       有些情况下在 syslog 中可以找到一些有用信息- 请尝试
       dmesg | tail  这样的命令看看。

1

ls /sbin/mount.*

1

yum install -y nfs-utils 

1

apt-get install nfs-common

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

Events:
  Type     Reason       Age                From               Message
  ----     ------       ----               ----               -------
  Normal   Scheduled    28s                default-scheduler  Successfully assigned default/nfs-busybox-k6dpc to node131
  Warning  FailedMount  11s (x6 over 27s)  kubelet            MountVolume.SetUp failed for volume "nfs" : mount failed: exit status 32
Mounting command: mount
Mounting arguments: -t nfs 10.233.124.49:/tmp/data /var/lib/kubelet/pods/e5932fde-fe05-4612-b29d-333f48b03338/volumes/kubernetes.io~nfs/nfs
Output: mount.nfs: Protocol not supported
[root@node131 nfs]#

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

Events:
  Type     Reason       Age               From               Message
  ----     ------       ----              ----               -------
  Normal   Scheduled    33s               default-scheduler  Successfully assigned default/nfs-busybox-4ht5m to node131
  Warning  FailedMount  1s (x7 over 34s)  kubelet            MountVolume.SetUp failed for volume "nfs" : mount failed: exit status 32
Mounting command: mount
Mounting arguments: -t nfs 10.233.124.49:/ /var/lib/kubelet/pods/bd34361c-4883-47a2-9e70-08772437e341/volumes/kubernetes.io~nfs/nfs
Output: mount.nfs: access denied by server while mounting 10.233.124.49:/
[root@node131 nfs]#

1
2
3
4
5
6

nfs:
    #server: nfs-server.default.svc.cluster.local
    server: 10.233.124.49
    path: "/exports"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

apiVersion: v1
kind: ReplicationController
metadata:
  name: nfs-server
spec:
  replicas: 1
  selector:
    role: nfs-server
  template:
    metadata:
      labels:
        role: nfs-server
    spec:
      containers:
      - name: nfs-server
        image: k8s.gcr.io/volume-nfs:0.8
        ports:
          - name: nfs
            containerPort: 2049
          - name: mountd
            containerPort: 20048
          - name: rpcbind
            containerPort: 111
        securityContext:
          privileged: true
        volumeMounts:
          - mountPath: /exports
            # name: mypvc
            name: data-volume
      volumes:
        # - name: mypvc
        #   persistentVolumeClaim:
        #     claimName: nfs-pv-provisioning-demo
        - name: data-volume                   #卷名
          hostPath:
            path: /tmp/data

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs
spec:
  capacity:
    storage: 1Mi
  accessModes:
    - ReadWriteMany
    #- ReadWriteOnce
  nfs:
    # faild: nfs-server svc name
    #server: nfs-server.default.svc.cluster.local
    # success: nfs-server svc ip
    server: 10.233.16.102
    path: "/exports"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs
spec:
  accessModes:
    - ReadWriteMany
    #- ReadWriteOnce
  storageClassName: ""
  resources:
    requests:
      storage: 1Mi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# This mounts the nfs volume claim into /mnt and continuously
# overwrites /mnt/index.html with the time and hostname of the pod.

apiVersion: v1
kind: ReplicationController
metadata:
  name: nfs-busybox
spec:
  replicas: 2
  selector:
    name: nfs-busybox
  template:
    metadata:
      labels:
        name: nfs-busybox
    spec:
      containers:
      - image: busybox
        command:
          - sh
          - -c
          - 'while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done'
        imagePullPolicy: IfNotPresent
        name: busybox
        volumeMounts:
          # name must match the volume name below
          - name: nfs
            mountPath: "/mnt"
      volumes:
      - name: nfs
        persistentVolumeClaim:
          claimName: nfs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# This pod mounts the nfs volume claim into /usr/share/nginx/html and
# serves a simple web page.

apiVersion: v1
kind: ReplicationController
metadata:
  name: nfs-web
spec:
  replicas: 2
  selector:
    role: web-frontend
  template:
    metadata:
      labels:
        role: web-frontend
    spec:
      containers:
      - name: web
        image: nginx:1.19
        ports:
          - name: web
            containerPort: 80
        volumeMounts:
            # name must match the volume name below
            - name: nfs
              mountPath: "/usr/share/nginx/html"
      volumes:
      - name: nfs
        persistentVolumeClaim:
          claimName: nfs

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253

// Volume represents a named volume in a pod that may be accessed by any container in the pod.
type Volume struct {
    // Volume's name.
    // Must be a DNS_LABEL and unique within the pod.
    // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
    Name string `json:"name" protobuf:"bytes,1,opt,name=name"`
    // VolumeSource represents the location and type of the mounted volume.
    // If not specified, the Volume is implied to be an EmptyDir.
    // This implied behavior is deprecated and will be removed in a future version.
    VolumeSource `json:",inline" protobuf:"bytes,2,opt,name=volumeSource"`
}

// Represents the source of a volume to mount.
// Only one of its members may be specified.
type VolumeSource struct {
    // HostPath represents a pre-existing file or directory on the host
    // machine that is directly exposed to the container. This is generally
    // used for system agents or other privileged things that are allowed
    // to see the host machine. Most containers will NOT need this.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath
    // ---
    // TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not
    // mount host directories as read/write.
    // +optional
    HostPath *HostPathVolumeSource `json:"hostPath,omitempty" protobuf:"bytes,1,opt,name=hostPath"`
    // EmptyDir represents a temporary directory that shares a pod's lifetime.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir
    // +optional
    EmptyDir *EmptyDirVolumeSource `json:"emptyDir,omitempty" protobuf:"bytes,2,opt,name=emptyDir"`
    // GCEPersistentDisk represents a GCE Disk resource that is attached to a
    // kubelet's host machine and then exposed to the pod.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk
    // +optional
    GCEPersistentDisk *GCEPersistentDiskVolumeSource `json:"gcePersistentDisk,omitempty" protobuf:"bytes,3,opt,name=gcePersistentDisk"`
    // AWSElasticBlockStore represents an AWS Disk resource that is attached to a
    // kubelet's host machine and then exposed to the pod.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore
    // +optional
    AWSElasticBlockStore *AWSElasticBlockStoreVolumeSource `json:"awsElasticBlockStore,omitempty" protobuf:"bytes,4,opt,name=awsElasticBlockStore"`
    // GitRepo represents a git repository at a particular revision.
    // DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an
    // EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir
    // into the Pod's container.
    // +optional
    GitRepo *GitRepoVolumeSource `json:"gitRepo,omitempty" protobuf:"bytes,5,opt,name=gitRepo"`
    // Secret represents a secret that should populate this volume.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#secret
    // +optional
    Secret *SecretVolumeSource `json:"secret,omitempty" protobuf:"bytes,6,opt,name=secret"`
    // NFS represents an NFS mount on the host that shares a pod's lifetime
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs
    // +optional
    NFS *NFSVolumeSource `json:"nfs,omitempty" protobuf:"bytes,7,opt,name=nfs"`
    // ISCSI represents an ISCSI Disk resource that is attached to a
    // kubelet's host machine and then exposed to the pod.
    // More info: https://examples.k8s.io/volumes/iscsi/README.md
    // +optional
    ISCSI *ISCSIVolumeSource `json:"iscsi,omitempty" protobuf:"bytes,8,opt,name=iscsi"`
    // Glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime.
    // More info: https://examples.k8s.io/volumes/glusterfs/README.md
    // +optional
    Glusterfs *GlusterfsVolumeSource `json:"glusterfs,omitempty" protobuf:"bytes,9,opt,name=glusterfs"`
    // PersistentVolumeClaimVolumeSource represents a reference to a
    // PersistentVolumeClaim in the same namespace.
    // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims
    // +optional
    PersistentVolumeClaim *PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty" protobuf:"bytes,10,opt,name=persistentVolumeClaim"`
    // RBD represents a Rados Block Device mount on the host that shares a pod's lifetime.
    // More info: https://examples.k8s.io/volumes/rbd/README.md
    // +optional
    RBD *RBDVolumeSource `json:"rbd,omitempty" protobuf:"bytes,11,opt,name=rbd"`
    // FlexVolume represents a generic volume resource that is
    // provisioned/attached using an exec based plugin.
    // +optional
    FlexVolume *FlexVolumeSource `json:"flexVolume,omitempty" protobuf:"bytes,12,opt,name=flexVolume"`
    // Cinder represents a cinder volume attached and mounted on kubelets host machine.
    // More info: https://examples.k8s.io/mysql-cinder-pd/README.md
    // +optional
    Cinder *CinderVolumeSource `json:"cinder,omitempty" protobuf:"bytes,13,opt,name=cinder"`
    // CephFS represents a Ceph FS mount on the host that shares a pod's lifetime
    // +optional
    CephFS *CephFSVolumeSource `json:"cephfs,omitempty" protobuf:"bytes,14,opt,name=cephfs"`
    // Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running
    // +optional
    Flocker *FlockerVolumeSource `json:"flocker,omitempty" protobuf:"bytes,15,opt,name=flocker"`
    // DownwardAPI represents downward API about the pod that should populate this volume
    // +optional
    DownwardAPI *DownwardAPIVolumeSource `json:"downwardAPI,omitempty" protobuf:"bytes,16,opt,name=downwardAPI"`
    // FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.
    // +optional
    FC *FCVolumeSource `json:"fc,omitempty" protobuf:"bytes,17,opt,name=fc"`
    // AzureFile represents an Azure File Service mount on the host and bind mount to the pod.
    // +optional
    AzureFile *AzureFileVolumeSource `json:"azureFile,omitempty" protobuf:"bytes,18,opt,name=azureFile"`
    // ConfigMap represents a configMap that should populate this volume
    // +optional
    ConfigMap *ConfigMapVolumeSource `json:"configMap,omitempty" protobuf:"bytes,19,opt,name=configMap"`
    // VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine
    // +optional
    VsphereVolume *VsphereVirtualDiskVolumeSource `json:"vsphereVolume,omitempty" protobuf:"bytes,20,opt,name=vsphereVolume"`
    // Quobyte represents a Quobyte mount on the host that shares a pod's lifetime
    // +optional
    Quobyte *QuobyteVolumeSource `json:"quobyte,omitempty" protobuf:"bytes,21,opt,name=quobyte"`
    // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.
    // +optional
    AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,22,opt,name=azureDisk"`
    // PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine
    PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty" protobuf:"bytes,23,opt,name=photonPersistentDisk"`
    // Items for all in one resources secrets, configmaps, and downward API
    Projected *ProjectedVolumeSource `json:"projected,omitempty" protobuf:"bytes,26,opt,name=projected"`
    // PortworxVolume represents a portworx volume attached and mounted on kubelets host machine
    // +optional
    PortworxVolume *PortworxVolumeSource `json:"portworxVolume,omitempty" protobuf:"bytes,24,opt,name=portworxVolume"`
    // ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.
    // +optional
    ScaleIO *ScaleIOVolumeSource `json:"scaleIO,omitempty" protobuf:"bytes,25,opt,name=scaleIO"`
    // StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.
    // +optional
    StorageOS *StorageOSVolumeSource `json:"storageos,omitempty" protobuf:"bytes,27,opt,name=storageos"`
    // CSI (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature).
    // +optional
    CSI *CSIVolumeSource `json:"csi,omitempty" protobuf:"bytes,28,opt,name=csi"`
    // Ephemeral represents a volume that is handled by a cluster storage driver (Alpha feature).
    // The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts,
    // and deleted when the pod is removed.
    //
    // Use this if:
    // a) the volume is only needed while the pod runs,
    // b) features of normal volumes like restoring from snapshot or capacity
    //    tracking are needed,
    // c) the storage driver is specified through a storage class, and
    // d) the storage driver supports dynamic volume provisioning through
    //    a PersistentVolumeClaim (see EphemeralVolumeSource for more
    //    information on the connection between this volume type
    //    and PersistentVolumeClaim).
    //
    // Use PersistentVolumeClaim or one of the vendor-specific
    // APIs for volumes that persist for longer than the lifecycle
    // of an individual pod.
    //
    // Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to
    // be used that way - see the documentation of the driver for
    // more information.
    //
    // A pod can use both types of ephemeral volumes and
    // persistent volumes at the same time.
    //
    // +optional
    Ephemeral *EphemeralVolumeSource `json:"ephemeral,omitempty" protobuf:"bytes,29,opt,name=ephemeral"`
}

// PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace.
// This volume finds the bound PV and mounts that volume for the pod. A
// PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another
// type of volume that is owned by someone else (the system).
type PersistentVolumeClaimVolumeSource struct {
    // ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume.
    // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims
    ClaimName string `json:"claimName" protobuf:"bytes,1,opt,name=claimName"`
    // Will force the ReadOnly setting in VolumeMounts.
    // Default false.
    // +optional
    ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,2,opt,name=readOnly"`
}

// PersistentVolumeSource is similar to VolumeSource but meant for the
// administrator who creates PVs. Exactly one of its members must be set.
type PersistentVolumeSource struct {
    // GCEPersistentDisk represents a GCE Disk resource that is attached to a
    // kubelet's host machine and then exposed to the pod. Provisioned by an admin.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk
    // +optional
    GCEPersistentDisk *GCEPersistentDiskVolumeSource `json:"gcePersistentDisk,omitempty" protobuf:"bytes,1,opt,name=gcePersistentDisk"`
    // AWSElasticBlockStore represents an AWS Disk resource that is attached to a
    // kubelet's host machine and then exposed to the pod.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore
    // +optional
    AWSElasticBlockStore *AWSElasticBlockStoreVolumeSource `json:"awsElasticBlockStore,omitempty" protobuf:"bytes,2,opt,name=awsElasticBlockStore"`
    // HostPath represents a directory on the host.
    // Provisioned by a developer or tester.
    // This is useful for single-node development and testing only!
    // On-host storage is not supported in any way and WILL NOT WORK in a multi-node cluster.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath
    // +optional
    HostPath *HostPathVolumeSource `json:"hostPath,omitempty" protobuf:"bytes,3,opt,name=hostPath"`
    // Glusterfs represents a Glusterfs volume that is attached to a host and
    // exposed to the pod. Provisioned by an admin.
    // More info: https://examples.k8s.io/volumes/glusterfs/README.md
    // +optional
    Glusterfs *GlusterfsPersistentVolumeSource `json:"glusterfs,omitempty" protobuf:"bytes,4,opt,name=glusterfs"`
    // NFS represents an NFS mount on the host. Provisioned by an admin.
    // More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs
    // +optional
    NFS *NFSVolumeSource `json:"nfs,omitempty" protobuf:"bytes,5,opt,name=nfs"`
    // RBD represents a Rados Block Device mount on the host that shares a pod's lifetime.
    // More info: https://examples.k8s.io/volumes/rbd/README.md
    // +optional
    RBD *RBDPersistentVolumeSource `json:"rbd,omitempty" protobuf:"bytes,6,opt,name=rbd"`
    // ISCSI represents an ISCSI Disk resource that is attached to a
    // kubelet's host machine and then exposed to the pod. Provisioned by an admin.
    // +optional
    ISCSI *ISCSIPersistentVolumeSource `json:"iscsi,omitempty" protobuf:"bytes,7,opt,name=iscsi"`
    // Cinder represents a cinder volume attached and mounted on kubelets host machine.
    // More info: https://examples.k8s.io/mysql-cinder-pd/README.md
    // +optional
    Cinder *CinderPersistentVolumeSource `json:"cinder,omitempty" protobuf:"bytes,8,opt,name=cinder"`
    // CephFS represents a Ceph FS mount on the host that shares a pod's lifetime
    // +optional
    CephFS *CephFSPersistentVolumeSource `json:"cephfs,omitempty" protobuf:"bytes,9,opt,name=cephfs"`
    // FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.
    // +optional
    FC *FCVolumeSource `json:"fc,omitempty" protobuf:"bytes,10,opt,name=fc"`
    // Flocker represents a Flocker volume attached to a kubelet's host machine and exposed to the pod for its usage. This depends on the Flocker control service being running
    // +optional
    Flocker *FlockerVolumeSource `json:"flocker,omitempty" protobuf:"bytes,11,opt,name=flocker"`
    // FlexVolume represents a generic volume resource that is
    // provisioned/attached using an exec based plugin.
    // +optional
    FlexVolume *FlexPersistentVolumeSource `json:"flexVolume,omitempty" protobuf:"bytes,12,opt,name=flexVolume"`
    // AzureFile represents an Azure File Service mount on the host and bind mount to the pod.
    // +optional
    AzureFile *AzureFilePersistentVolumeSource `json:"azureFile,omitempty" protobuf:"bytes,13,opt,name=azureFile"`
    // VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine
    // +optional
    VsphereVolume *VsphereVirtualDiskVolumeSource `json:"vsphereVolume,omitempty" protobuf:"bytes,14,opt,name=vsphereVolume"`
    // Quobyte represents a Quobyte mount on the host that shares a pod's lifetime
    // +optional
    Quobyte *QuobyteVolumeSource `json:"quobyte,omitempty" protobuf:"bytes,15,opt,name=quobyte"`
    // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.
    // +optional
    AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,16,opt,name=azureDisk"`
    // PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine
    PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty" protobuf:"bytes,17,opt,name=photonPersistentDisk"`
    // PortworxVolume represents a portworx volume attached and mounted on kubelets host machine
    // +optional
    PortworxVolume *PortworxVolumeSource `json:"portworxVolume,omitempty" protobuf:"bytes,18,opt,name=portworxVolume"`
    // ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.
    // +optional
    ScaleIO *ScaleIOPersistentVolumeSource `json:"scaleIO,omitempty" protobuf:"bytes,19,opt,name=scaleIO"`
    // Local represents directly-attached storage with node affinity
    // +optional
    Local *LocalVolumeSource `json:"local,omitempty" protobuf:"bytes,20,opt,name=local"`
    // StorageOS represents a StorageOS volume that is attached to the kubelet's host machine and mounted into the pod
    // More info: https://examples.k8s.io/volumes/storageos/README.md
    // +optional
    StorageOS *StorageOSPersistentVolumeSource `json:"storageos,omitempty" protobuf:"bytes,21,opt,name=storageos"`
    // CSI represents storage that is handled by an external CSI driver (Beta feature).
    // +optional
    CSI *CSIPersistentVolumeSource `json:"csi,omitempty" protobuf:"bytes,22,opt,name=csi"`
}