【kubernetes/k8s源码分析】 k8s local storage 源码分析

亦凉 2022-01-26 10:27 362阅读 0赞

>     Local Storage 就是 Kubernetes 集群中每个节点的本地存储，在 Kubernetes 1.7 中 kubelet 可以支持对 kube-reserved 和system-reserved 指定本地存储资源。Local 存储是为了替换 **hostPath** volume
> 
>    kubernetes v1.14.0 正式发布，新特性就是本地持久化管理( Local StorageManagement)特性正式 GA(稳定)

> apiVersion: storage.k8s.io/v1  
> kind: StorageClass  
> metadata:  
>   name: local-storage  
> provisioner: kubernetes.io/no-provisioner  
> volumeBindingMode: WaitForFirstConsumer  
> \# Supported policies: Delete, Retain  
> reclaimPolicy: Delete

> apiVersion: v1  
> kind: PersistentVolume  
> metadata:  
>   name: example-local-pv  
> spec:  
>   capacity:  
>     storage: 5Gi  
>   accessModes:  
>   - ReadWriteOnce  
>   persistentVolumeReclaimPolicy: Delete  
>   storageClassName: local-storage  
>   local:  
>     path: /tmp/aaaa  
>   nodeAffinity:  
>     required:  
>       nodeSelectorTerms:  
>       - matchExpressions:  
>         - key: kubernetes.io/hostname  
>           operator: In  
>           values:  
>           - master-node

# 1. NewMounter 函数 #

实例化 localVolumeMounter，熟悉的套路

func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
    	_, readOnly, err := getVolumeSource(spec)
    	if err != nil {
    		return nil, err
    	}
    
    	globalLocalPath, err := plugin.getGlobalLocalPath(spec)
    	if err != nil {
    		return nil, err
    	}
    
    	return &localVolumeMounter{
    		localVolume: &localVolume{
    			pod:             pod,
    			podUID:          pod.UID,
    			volName:         spec.Name(),
    			mounter:         plugin.host.GetMounter(plugin.GetPluginName()),
    			plugin:          plugin,
    			globalPath:      globalLocalPath,
    			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(pod.UID, stringsutil.EscapeQualifiedNameForDisk(localVolumePluginName), spec.Name())),
    		},
    		mountOptions: util.MountOptionFromSpec(spec),
    		readOnly:     readOnly,
    	}, nil
    
    }

# 2. SetUp #

###     2.1 global path 不能为空 ###

m.plugin.volumeLocks.LockKey(m.globalPath)
    defer m.plugin.volumeLocks.UnlockKey(m.globalPath)
    
    if m.globalPath == "" {
    	return fmt.Errorf("LocalVolume volume %q path is empty", m.volName)
    }

###     2.2 IsNotMountPoint 函数查验是否已经mount了 ###

notMnt, err := m.mounter.IsNotMountPoint(dir)
    klog.V(4).Infof("LocalVolume mount setup: PodDir(%s) VolDir(%s) Mounted(%t) Error(%v), ReadOnly(%t)", dir, m.globalPath, !notMnt, err, m.readOnly)
    if err != nil && !os.IsNotExist(err) {
    	klog.Errorf("cannot validate mount point: %s %v", dir, err)
    	return err
    }
    
    if !notMnt {
    	return nil
    }

###     2.3 mount 操作 ###

将 /var/lib/kubelet/pods/fc695a36-8475-11e9-a5a0-0800271c9f15/volumes/kubernetes.io~local-volume/example-local-pv 挂载在容器的目录

err = m.mounter.Mount(globalPath, dir, "", mountOptions)
    if err != nil {
    	klog.Errorf("Mount of volume %s failed: %v", dir, err)
    	notMnt, mntErr := m.mounter.IsNotMountPoint(dir)
    	if mntErr != nil {
    		klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
    		return err
    	}

** 注意： **

需要运行节点上面去**创建本地目录**，否则POD将会启动失败

> Events:  
>   Type     Reason       Age                  From                  Message  
>   ----     ------       ----                 ----                  -------  
>   Normal   Scheduled    2m30s                default-scheduler     Successfully assigned default/example-pod to master-node  
>   Warning  FailedMount  86s (x8 over 2m30s)  kubelet, master-node  MountVolume.NewMounter initialization failed for volume "example-local-pv" : path "/tmp/aaaa" does not exist