IPVS负载均衡（六）浅析ip_vs_core.c

待我称王封你为后i 2022-06-17 00:00 253阅读 0赞

# IPVS负载均衡（六）浅析ip\_vs\_core.c #

ip\_vs\_core.c

先从linux内核module\_init和module\_exit开始。

## module\_init ##

函数：ip\_vs\_init

函数原型：static int \_\_init ip\_vs\_init(void)

初始化ip\_vs模块所需的各种

/* * Initialize IP Virtual Server */
    static int __init ip_vs_init(void)
    {
        int ret;
    
        ret = ip_vs_control_init();
        if (ret < 0) {
            pr_err("can't setup control.\n");
            goto exit;
        }
    
        ip_vs_protocol_init();
    
        ret = ip_vs_conn_init();
        if (ret < 0) {
            pr_err("can't setup connection table.\n");
            goto cleanup_protocol;
        }
    
        ret = register_pernet_subsys(&ipvs_core_ops); /* Alloc ip_vs struct */
        if (ret < 0)
            goto cleanup_conn;
    
        ret = register_pernet_device(&ipvs_core_dev_ops);
        if (ret < 0)
            goto cleanup_sub;
    
        ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
        if (ret < 0) {
            pr_err("can't register hooks.\n");
            goto cleanup_dev;
        }
    
        ret = ip_vs_register_nl_ioctl();
        if (ret < 0) {
            pr_err("can't register netlink/ioctl.\n");
            goto cleanup_hooks;
        }
    
        pr_info("ipvs loaded.\n");
    
        return ret;
    
    cleanup_hooks:
        nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
    cleanup_dev:
        unregister_pernet_device(&ipvs_core_dev_ops);
    cleanup_sub:
        unregister_pernet_subsys(&ipvs_core_ops);
    cleanup_conn:
        ip_vs_conn_cleanup();
    cleanup_protocol:
        ip_vs_protocol_cleanup();
        ip_vs_control_cleanup();
    exit:
        return ret;
    }

主要调用了：

->ip_vs_control_init ->ip_vs_protocol_init，协议所需初始化 ->ip_vs_conn_init，链接管理初始化 ->register_pernet_subsys ->register_pernet_device ->nf_register_hooks，注册netfilter钩子函数，后续代码流程会从这里开始 ->ip_vs_register_nl_ioctl

其中跟IPVS主要流程最相关的是nf\_register\_hooks、ip\_vs\_protocol\_init、ip\_vs\_conn\_init，后续会主要从nf\_register\_hooks这里–在netfilter上注册的钩子函数上梳理主干流程

## module\_exit ##

函数：ip\_vs\_cleanup

函数原型：static void \_\_exit ip\_vs\_cleanup(void)

卸载ip\_vs模块，释放占用的资源等操作

static void __exit ip_vs_cleanup(void)
    {
        ip_vs_unregister_nl_ioctl();
        nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
        unregister_pernet_device(&ipvs_core_dev_ops);
        unregister_pernet_subsys(&ipvs_core_ops);   /* free ip_vs struct */
        ip_vs_conn_cleanup();
        ip_vs_protocol_cleanup();
        ip_vs_control_cleanup();
        pr_info("ipvs unloaded.\n");
    }

主要调用：

->ip_vs_unregister_nl_ioctl ->nf_unregister_hooks ->unregister_pernet_device ->unregister_pernet_subsys ->ip_vs_conn_cleanup ->ip_vs_protocol_cleanup ->ip_vs_control_cleanup

可以注意到，ip\_vs\_cleanup资源释放顺序，与ip\_vs\_init申请资源顺序刚好相反；且与ip\_vs\_init中的出错处理部分顺序基本相同。

看完模块注册去注册流程，那么后面则开始寻找

## nf\_register\_hooks钩子函数 ##

通过nf\_register\_hooks注册的钩子函数进一步梳理代码流程

ret = nf\_register\_hooks(ip\_vs\_ops, ARRAY\_SIZE(ip\_vs\_ops));

ip\_vs\_ops的定义

static struct nf_hook_ops ip_vs_ops[] __read_mostly = {
        /* After packet filtering, change source only for VS/NAT */
        {
            .hook       = ip_vs_reply4,
            .pf     = NFPROTO_IPV4,
            .hooknum    = NF_INET_LOCAL_IN,
            .priority   = NF_IP_PRI_NAT_SRC - 2,
        },
        /* After packet filtering, forward packet through VS/DR, VS/TUN, * or VS/NAT(change destination), so that filtering rules can be * applied to IPVS. */
        {
            .hook       = ip_vs_remote_request4,
            .pf     = NFPROTO_IPV4,
            .hooknum    = NF_INET_LOCAL_IN,
            .priority   = NF_IP_PRI_NAT_SRC - 1,
        },
        /* Before ip_vs_in, change source only for VS/NAT */
        {
            .hook       = ip_vs_local_reply4,
            .pf     = NFPROTO_IPV4,
            .hooknum    = NF_INET_LOCAL_OUT,
            .priority   = NF_IP_PRI_NAT_DST + 1,
        },
        /* After mangle, schedule and forward local requests */
        {
            .hook       = ip_vs_local_request4,
            .pf     = NFPROTO_IPV4,
            .hooknum    = NF_INET_LOCAL_OUT,
            .priority   = NF_IP_PRI_NAT_DST + 2,
        },
        /* After packet filtering (but before ip_vs_out_icmp), catch icmp * destined for 0.0.0.0/0, which is for incoming IPVS connections */
        {
            .hook       = ip_vs_forward_icmp,
            .pf     = NFPROTO_IPV4,
            .hooknum    = NF_INET_FORWARD,
            .priority   = 99,
        },
        /* After packet filtering, change source only for VS/NAT */
        {
            .hook       = ip_vs_reply4,
            .pf     = NFPROTO_IPV4,
            .hooknum    = NF_INET_FORWARD,
            .priority   = 100,
        },
    #ifdef CONFIG_IP_VS_IPV6
        /* After packet filtering, change source only for VS/NAT */
        {
            .hook       = ip_vs_reply6,
            .pf     = NFPROTO_IPV6,
            .hooknum    = NF_INET_LOCAL_IN,
            .priority   = NF_IP6_PRI_NAT_SRC - 2,
        },
        /* After packet filtering, forward packet through VS/DR, VS/TUN, * or VS/NAT(change destination), so that filtering rules can be * applied to IPVS. */
        {
            .hook       = ip_vs_remote_request6,
            .pf     = NFPROTO_IPV6,
            .hooknum    = NF_INET_LOCAL_IN,
            .priority   = NF_IP6_PRI_NAT_SRC - 1,
        },
        /* Before ip_vs_in, change source only for VS/NAT */
        {
            .hook       = ip_vs_local_reply6,
            .pf     = NFPROTO_IPV6,
            .hooknum    = NF_INET_LOCAL_OUT,
            .priority   = NF_IP6_PRI_NAT_DST + 1,
        },
        /* After mangle, schedule and forward local requests */
        {
            .hook       = ip_vs_local_request6,
            .pf     = NFPROTO_IPV6,
            .hooknum    = NF_INET_LOCAL_OUT,
            .priority   = NF_IP6_PRI_NAT_DST + 2,
        },
        /* After packet filtering (but before ip_vs_out_icmp), catch icmp * destined for 0.0.0.0/0, which is for incoming IPVS connections */
        {
            .hook       = ip_vs_forward_icmp_v6,
            .pf     = NFPROTO_IPV6,
            .hooknum    = NF_INET_FORWARD,
            .priority   = 99,
        },
        /* After packet filtering, change source only for VS/NAT */
        {
            .hook       = ip_vs_reply6,
            .pf     = NFPROTO_IPV6,
            .hooknum    = NF_INET_FORWARD,
            .priority   = 100,
        },
    #endif
    };

由于是浅析，故这里就不会分析IPV6的场景，主要看ipv4的场景。

.hook涉及到的函数有：

*  ip\_vs\_reply4
 *  ip\_vs\_remote\_request4
 *  ip\_vs\_local\_reply4
 *  ip\_vs\_local\_request4
 *  ip\_vs\_forward\_icmp
 *  ip\_vs\_reply4

netfilter钩子函数最后返回值主要有这几种：

<table> 
 <thead> 
  <tr> 
   <th>名称</th> 
   <th>含义</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>NF_DROP</td> 
   <td>丢弃该数据包</td> 
  </tr> 
  <tr> 
   <td>NF_ACCEPT</td> 
   <td>保留该数据包</td> 
  </tr> 
  <tr> 
   <td>NF_STOLEN</td> 
   <td>忘掉该数据包</td> 
  </tr> 
  <tr> 
   <td>NF_QUEUE</td> 
   <td>将该数据包插入到用户空间</td> 
  </tr> 
  <tr> 
   <td>NF_REPEAT</td> 
   <td>再次调用该hook函数</td> 
  </tr> 
 </tbody> 
</table>

.pf是IP类型，这里只讨论IPV4的情况

.hooknum则是涉及到netfilter的五处钩子点，简单可以理解为，netfilter在内核协议栈中五处地方有函数指针，如果注册了钩子函数，则会在对应点调用指定的钩子函数，处理报文。netfilter的五处分别为：NF\_IP\_PRE\_ROUTING，在完整性校验之后，选路确定之前；NF\_IP\_LOCAL\_IN，在选路确定之后，且数据包的目的是本地主机；NF\_IP\_FORWARD，目的地是其它主机地数据包；NF\_IP\_LOCAL\_OUT，来自本机进程的数据包在其离开本地主机的过程中；NF\_IP\_POST\_ROUTING，在数据包离开本地主机“上线”之前。

可以看到在ipvs中，主要用到了NF\_IP\_LOCAL\_IN、NF\_IP\_LOCAL\_OUT、NF\_IP\_FORWARD。

.priority则是表示优先级，因为可能会挂多个钩子函数，所以设置优先级，会先调用优先级高的，再调用优先级低的进行处理。

## 下一步 ##

可以看到ip\_vs\_reply4、ip\_vs\_local\_reply4都是调用了ip\_vs\_out

ip\_vs\_remote\_request4、ip\_vs\_local\_request4都是调用了ip\_vs\_in

下面将会简要分析ip\_vs\_out和ip\_vs\_in两个函数的处理流程