【kubernetes/k8s源码分析】CNI macvlan源码分析

今天药忘吃喽~ 2022-04-22 01:36 242阅读 0赞

# macvlan原理 #

在linux命令行执行 `lsmod | grep macvlan` 查看当前内核是否加载了该driver；如果没有查看到，可以通过 `modprobe macvlan` 来载入

macvlan：使用 macvlan 技术，从某个物理网卡虚拟出多个虚拟网卡有独立的 ip 和 mac 地址

### 工作模式（后面中提到的mode） ###

*  Bridge：属于同一个parent接口的macvlan接口之间挂到同一个bridge上，可以二层互通（经过测试，发现这些macvlan接口都无法与parent 接口互通）。
 *  VPEA：所有接口的流量都需要到外部switch才能够到达其他接口。
 *  Private：接口只接受发送给自己MAC地址的报文。

手动添加macvlan  
docker inspect -f '\{ \{.State.Pid\}\}' 5c63ae4340f9  
ip link set macvlan1 netns 4849  
ip link add link eth0 name macv1 type macvlan mode bridge

ip netns exec 2109 ip link set macvlan1 name eth1  
ip net exec 4849 ip addr add 10.12.51.180/24 dev eth1  
ip net exec 4849 route add -net 10.12.51.186 netmask 255.255.255.255 dev eth0

## macvlan cni配置 ##

\{  
    "name": "macvlannet",  
    "cniVersion": "0.1.0",  
    "type": "macvlan",  
    "master": "eth0",  
    "mode": "vepa",  
    "ipam": \{  
        "type": "host-local",  
        "subnet": "10.12.0.0/16",  
        "rangeStart": "10.12.52.100",  
        "rangeEnd": "10.12.52.250",  
        "gateway": "10.12.51.11",  
        "routes": \[  
            \{ "dst": "0.0.0.0/0" \}  
        \]  
    \}  
\}

## Network configuration reference
    
    * `name` (string, required): the name of the network
    * `type` (string, required): "macvlan"
    * `master` (string, required): name of the host interface to enslave
    * `mode` (string, optional): one of "bridge", "private", "vepa", "passthrough". Defaults to "bridge".
    * `mtu` (integer, optional): explicitly set MTU to the specified value. Defaults to the value chosen by the kernel.
    * `ipam` (dictionary, required): IPAM configuration to be used for this network.

# 结构体 #

###   NetConf结构体 ###

type NetConf struct {
    	types.NetConf
    	Master string `json:"master"`
    	Mode   string `json:"mode"`
    	MTU    int    `json:"mtu"`
    }

###   NetConf结构体 ###

// NetConf describes a network.
    type NetConf struct {
    	CNIVersion string `json:"cniVersion,omitempty"`
    	Name         string          `json:"name,omitempty"`
    	Type         string          `json:"type,omitempty"`
    	Capabilities map[string]bool `json:"capabilities,omitempty"`
    	IPAM         IPAM            `json:"ipam,omitempty"`
    	DNS          DNS             `json:"dns"`
    }

# 1. cmdAdd函数 #

###   1.1 LoadIPAMConfig函数 ###

主要是将参数转为NetConf结构体

n, cniVersion, err := loadConf(args.StdinData)
    	if err != nil {
    		return err
    	}

###   1.2 GetNS函数 ###

路径为/proc/$\{容器PID\}/ns/net，打开文件描述符

netns, err := ns.GetNS(args.Netns)
    	if err != nil {
    		return fmt.Errorf("failed to open netns %q: %v", netns, err)
    	}
    	defer netns.Close()

###   1.3 createMacvlan函数 ###

第2章节讲解，相当于执行命令ip link add link eth0 name macv1 type macvlan mode bridge

macvlanInterface, err := createMacvlan(n, args.IfName, netns)
    	if err != nil {
    		return err
    	}

###   1.4 ipam.ExecAdd函数 ###

这个函数最终调用的是ipam插件，本文使用的为host-local，将参数一起传给host-local，主要是获得ip，管理ip，先忽略这个，假设去调用这个函数正确返回IP，将结果存入如下结构体中

type Result struct {
       CNIVersion string         `json:"cniVersion,omitempty"`
       Interfaces []*Interface   `json:"interfaces,omitempty"`
       IPs        []*IPConfig    `json:"ips,omitempty"`
       Routes     []*types.Route `json:"routes,omitempty"`
       DNS        types.DNS      `json:"dns,omitempty"`
    }

// run the IPAM plugin and get back the config to apply
    	r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
    	if err != nil {
    		return err
    	}
    
    	// Convert whatever the IPAM result was into the current Result type
    	result, err := current.NewResultFromResult(r)
    	if err != nil {
    		return err
    	}

###   1.5 ipam.ConfigureIface函数 ###

第3章节讲解，主要是将网卡启动，设置IP MASK ROUTE等

if err := ipam.ConfigureIface(args.IfName, result); err != nil {
    			return err
    		}

# 2. createMacvlan函数 #

###   2.1 默认为bridge mode，生成Macvlan结构体 ###

macvlan := &current.Interface{}
    
    	mode, err := modeFromString(conf.Mode)
    	if err != nil {
    		return nil, err
    	}
    
    	m, err := netlink.LinkByName(conf.Master)
    	if err != nil {
    		return nil, fmt.Errorf("failed to lookup master %q: %v", conf.Master, err)
    	}
    
    	// due to kernel bug we have to create with tmpName or it might
    	// collide with the name on the host and error out
    	tmpName, err := ip.RandomVethName()
    	if err != nil {
    		return nil, err
    	}
    
    	mv := &netlink.Macvlan{
    		LinkAttrs: netlink.LinkAttrs{
    			MTU:         conf.MTU,
    			Name:        tmpName,
    			ParentIndex: m.Attrs().Index,
    			Namespace:   netlink.NsFd(int(netns.Fd())),
    		},
    		Mode: mode,
    	}

###   2.2 LinkAdd函数 ###

相当于命令执行：ip link add link eth0 name macv1 type macvlan mode bridge

if err := netlink.LinkAdd(mv); err != nil {
    		return nil, fmt.Errorf("failed to create macvlan: %v", err)
    	}

# 3. ConfigureIface函数 #

路径plugins/pkg/ipam/ipam\_linux.go

###   3.1 LinkSetUp函数 ###

将网卡启动，相当于ip net exec 4849 ifconfig macv1 up

if err := netlink.LinkSetUp(link); err != nil {
    		return fmt.Errorf("failed to set %q UP: %v", ifName, err)
    	}

###   3.2 netlink.AddrAdd函数 ###

设置IP地址，相当于命令ip net exec 4849 ip addr add 10.12.51.250 netmask 255.255.0.0 dev eth0

addr := &netlink.Addr{IPNet: &ipc.Address, Label: ""}
    		if err = netlink.AddrAdd(link, addr); err != nil {
    			return fmt.Errorf("failed to add IP addr %v to %q: %v", ipc, ifName, err)
    		}

###   3.3 ip.AddRoute函数 ###

添加路由，相当于命令ip net exec 4849 route add -net 10.12.51.186 netmask 255.255.255.255 dev eth0

for _, r := range res.Routes {
    		routeIsV4 := r.Dst.IP.To4() != nil
    		gw := r.GW
    		log.Infof("ipam_linux routeIsV4: %v gw: %v", routeIsV4, gw)
    		if gw == nil {
    			if routeIsV4 && v4gw != nil {
    				gw = v4gw
    			} else if !routeIsV4 && v6gw != nil {
    				gw = v6gw
    			}
    		}
    
    		if err = ip.AddRoute(&r.Dst, gw, link); err != nil {
    			// we skip over duplicate routes as we assume the first one wins
    			if !os.IsExist(err) {
    				return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, ifName, err)
    			}
    		}
    	}