【kubernetes/k8s源码分析】 rook operator flexvolume provision 源码分析

ゝ一纸荒年。 2022-02-04 00:15 199阅读 0赞

初始化 provisioner，分别为 ceph.rook.io 和 rook.io

controller.NewProvisionController中provisioner实现为RookVolumeProvisioner'

定义 rook/pkg/operator/ceph/provisioner/provisioner.go

// Run volume provisioner for each of the supported configurations
    for name, vendor := range provisionerConfigs {
    	volumeProvisioner := provisioner.New(o.context, vendor)
    	pc := controller.NewProvisionController(
    		o.context.Clientset,
    		name,
    		volumeProvisioner,
    		serverVersion.GitVersion,
    	)
    	go pc.Run(stopChan)
    	logger.Infof("rook-provisioner %s started using %s flex vendor dir", name, vendor)
    }

ProvisionController主要是处理 pv pvc

>     controller.claimQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "claims")
>     controller.volumeQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "volumes")

哪里调用Provision 函数呢？ kubernetes中这里

> volume, err = provisioner.Provision(selectedNode, allowedTopologies)

# 1. Provision 函数 #

###     1.1 createVolume 函数 ###

创建一个卷，CreateImage调用命令 ***rbd create replicapool/pvc-c304ca86-70fc-11e9-8a6c-0800271c9f15 --size 30720*** \--cluster=rook-ceph --conf=/var/lib/rook/rook-ceph/rook-ceph.config --keyring=/var/lib/rook/rook-ceph/client.admin.keyring

// createVolume creates a rook block volume.
    func (p *RookVolumeProvisioner) createVolume(image, pool, dataPool string, clusterNamespace string, size int64) (*ceph.CephBlockImage, error) {
    	if image == "" || pool == "" || clusterNamespace == "" || size == 0 {
    		return nil, fmt.Errorf("image missing required fields (image=%s, pool=%s, clusterNamespace=%s, size=%d)", image, pool, clusterNamespace, size)
    	}
    
    	createdImage, err := ceph.CreateImage(p.context, clusterNamespace, image, pool, dataPool, uint64(size))
    	if err != nil {
    		return nil, fmt.Errorf("Failed to create rook block image %s/%s: %v", pool, image, err)
    	}
    	logger.Infof("Rook block image created: %s, size = %d", createdImage.Name, createdImage.Size)
    
    	return createdImage, nil
    }

1.2 返回PV信息

pv := &v1.PersistentVolume{
    	ObjectMeta: metav1.ObjectMeta{
    		Name: imageName,
    	},
    	Spec: v1.PersistentVolumeSpec{
    		PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
    		AccessModes:                   options.PVC.Spec.AccessModes,
    		Capacity: v1.ResourceList{
    			v1.ResourceName(v1.ResourceStorage): quantity,
    		},
    		PersistentVolumeSource: v1.PersistentVolumeSource{
    			FlexVolume: &v1.FlexPersistentVolumeSource{
    				Driver: flexdriver,
    				FSType: cfg.fstype,
    				Options: map[string]string{
    					flexvolume.StorageClassKey:     storageClass,
    					flexvolume.PoolKey:             cfg.blockPool,
    					flexvolume.ImageKey:            imageName,
    					flexvolume.ClusterNamespaceKey: cfg.clusterNamespace,
    					flexvolume.DataBlockPoolKey:    cfg.dataBlockPool,
    				},
    			},
    		},
    	},
    }

# 2. Delete 函数 #

这要是删除 rbd 操作

// Delete removes the storage asset that was created by Provision represented
    // by the given PV.
    func (p *RookVolumeProvisioner) Delete(volume *v1.PersistentVolume) error {
    	logger.Infof("Deleting volume %s", volume.Name)
    	if volume.Spec.PersistentVolumeSource.FlexVolume == nil {
    		return fmt.Errorf("Failed to delete rook block image %s: %v", volume.Name, "PersistentVolume is not a FlexVolume")
    	}
    	if volume.Spec.PersistentVolumeSource.FlexVolume.Options == nil {
    		return fmt.Errorf("Failed to delete rook block image %s: %v", volume.Name, "PersistentVolume has no image defined for the FlexVolume")
    	}
    	name := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ImageKey]
    	pool := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.PoolKey]
    	var clusterns string
    	if _, ok := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNamespaceKey]; ok {
    		clusterns = volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNamespaceKey]
    	} else if _, ok := volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNameKey]; ok {
    		// Fallback to `clusterName` as it was used in Rook version earlier v0.8
    		clusterns = volume.Spec.PersistentVolumeSource.FlexVolume.Options[flexvolume.ClusterNameKey]
    	}
    	if clusterns == "" {
    		return fmt.Errorf("Failed to delete rook block image %s/%s: no clusterNamespace or (deprecated) clusterName option given", pool, volume.Name)
    	}
    	err := ceph.DeleteImage(p.context, clusterns, name, pool)
    	if err != nil {
    		return fmt.Errorf("Failed to delete rook block image %s/%s: %v", pool, volume.Name, err)
    	}
    	logger.Infof("succeeded deleting volume %+v", volume)
    	return nil
    }

###    PV 资源如下： ###

> apiVersion: v1  
> kind: PersistentVolume  
> metadata:  
>   annotations:  
>     pv.kubernetes.io/provisioned-by: ceph.rook.io/block  
>   creationTimestamp: "2019-05-23T06:50:15Z"  
>   finalizers:  
>   - kubernetes.io/pv-protection  
>   name: pvc-03ccc03f-7d27-11e9-82d7-0800271c9f15  
>   resourceVersion: "135093"  
>   selfLink: /api/v1/persistentvolumes/pvc-03ccc03f-7d27-11e9-82d7-0800271c9f15  
>   uid: 05512a01-7d27-11e9-82d7-0800271c9f15  
> spec:  
>   accessModes:  
>   - ReadWriteOnce  
>   capacity:  
>     storage: 20Gi  
>   claimRef:  
>     apiVersion: v1  
>     kind: PersistentVolumeClaim  
>     name: mysql-pv-claim  
>     namespace: default  
>     resourceVersion: "135083"  
>     uid: 03ccc03f-7d27-11e9-82d7-0800271c9f15  
>   flexVolume:  
>     driver: ceph.rook.io/rook-ceph  
>     fsType: xfs  
>     options:  
>       clusterNamespace: rook-ceph  
>       dataBlockPool: ""  
>       image: pvc-03ccc03f-7d27-11e9-82d7-0800271c9f15  
>       pool: replicapool  
>       storageClass: rook-ceph-block  
>   persistentVolumeReclaimPolicy: Delete  
>   storageClassName: rook-ceph-block  
>   volumeMode: Filesystem  
> status:  
>   phase: Bound

attach 操作在agent 模块