移植facebook flashcache模块支持 linux kernel 4.13以上

Dear 丶 2022-01-17 03:43 212阅读 0赞

[为什么80%的码农都做不了架构师？>>> ][80_] ![hot3.png][]

## 环境 ##

准备在ubuntu 18.04 lts下搭建kvm+libvirt+glusterfs的虚拟化环境。

4台服务器是2块SSD，4块SATA（5400RPM）。

尝试了gluster tier，效果不好。

对于虚拟化环境下大文件的存储，tier的promote和demote的策略很难设置，

在大文件发生大量数据交换时会出现tier层"no space left"的错误，文件无法写入。

在没有更好的解决方案之前，facebook的flashcache（https://github.com/facebookarchive/flashcache )对于提速SATA盘的作用还是非常显著的。

但ubuntu18.04的apt源还没有。

https://launchpad.net/flashcache/+packages

## 错误 ##

于是下载flashcache源码编译dkms模块的时候，报错：

DKMS make.log for flashcache-3.1.3+git20150701 for kernel 4.15.0-36-generic (x86_64)
    Fri Oct 12 15:06:00 UTC 2018
    make -C /lib/modules/4.15.0-36-generic/build M=/var/lib/dkms/flashcache/3.1.3+git20150701/build modules V=0
    make[1]: Entering directory '/usr/src/linux-headers-4.15.0-36-generic'
    Makefile:975: "Cannot use CONFIG_STACK_VALIDATION=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel"
      CC [M]  /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_conf.o
      CC [M]  /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_main.o
      CC [M]  /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o
    /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.c: In function ‘flashcache_bio_endio’:
    /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.c:742:7: error: ‘struct bio’ has no member named ‘bi_error’; did you mean ‘bi_iter’?
      bio->bi_error = error;
           ^~~~~~~~
           bi_iter
    scripts/Makefile.build:332: recipe for target '/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o' failed
    make[2]: *** [/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o] Error 1
    Makefile:1551: recipe for target '_module_/var/lib/dkms/flashcache/3.1.3+git20150701/build' failed
    make[1]: *** [_module_/var/lib/dkms/flashcache/3.1.3+git20150701/build] Error 2
    make[1]: Leaving directory '/usr/src/linux-headers-4.15.0-36-generic'
    Makefile:35: recipe for target 'modules' failed
    make: *** [modules] Error 2

OS内核的版本是4.15.0-36。

## 原因 ##

根据错误提示：bio->bi*error ‘struct bio’ has no member named ‘bi*error’;

找了4.15的内核源码看看bio的struct的定义,

bio是在include/linux/blk*types.h中定义。*

*https://elixir.bootlin.com/linux/v4.15/source/include/linux/blk*types.h

struct bio {
        struct bio      *bi_next;   /* request queue link */
        struct gendisk      *bi_disk;
        unsigned int        bi_opf;     /* bottom bits req flags,
                             * top bits REQ_OP. Use
                             * accessors.
                             */
        unsigned short      bi_flags;   /* status, etc and bvec pool number */
        unsigned short      bi_ioprio;
        unsigned short      bi_write_hint;
        blk_status_t        bi_status;
        u8          bi_partno;
    
        /* Number of segments in this BIO after
         * physical address coalescing is performed.
         */
        unsigned int        bi_phys_segments;
    
        /*
         * To keep track of the max segment size, we account for the
         * sizes of the first and last mergeable segments in this bio.
         */
        unsigned int        bi_seg_front_size;
        unsigned int        bi_seg_back_size;
    
        struct bvec_iter    bi_iter;
    
        atomic_t        __bi_remaining;
        bio_end_io_t        *bi_end_io;
    
        void            *bi_private;
    #ifdef CONFIG_BLK_CGROUP
        /*
         * Optional ioc and css associated with this bio.  Put on bio
         * release.  Read comment on top of bio_associate_current().
         */
        struct io_context   *bi_ioc;
        struct cgroup_subsys_state *bi_css;
    #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
        void            *bi_cg_private;
        struct blk_issue_stat   bi_issue_stat;
    #endif
    #endif
        union {
    #if defined(CONFIG_BLK_DEV_INTEGRITY)
            struct bio_integrity_payload *bi_integrity; /* data integrity */
    #endif
        };
    
        unsigned short      bi_vcnt;    /* how many bio_vec's */
    
        /*
         * Everything starting with bi_max_vecs will be preserved by bio_reset()
         */
    
        unsigned short      bi_max_vecs;    /* max bvl_vecs we can hold */
    
        atomic_t        __bi_cnt;   /* pin count */
    
        struct bio_vec      *bi_io_vec; /* the actual vec list */
    
        struct bio_set      *bi_pool;
    
        /*
         * We can inline a number of vecs at the end of the bio, to avoid
         * double allocations for a small number of bio_vecs. This member
         * MUST obviously be kept at the very end of the bio.
         */
        struct bio_vec      bi_inline_vecs[0];
    };

里面根本没有bi*error。 再翻翻以前版本的代码，在4.12的bio定义里发现了bi*error： https://elixir.bootlin.com/linux/v4.12.14/source/include/linux/blk\_types.h

应该是从4.13版本内核之后，bio的结构定义发生了变化。

struct bio {
        struct bio      *bi_next;   /* request queue link */
        struct block_device *bi_bdev;
        int         bi_error;
        unsigned int        bi_opf;     /* bottom bits req flags,
                             * top bits REQ_OP. Use
                             * accessors.
                             */
        unsigned short      bi_flags;   /* status, etc and bvec pool number */
        unsigned short      bi_ioprio;

## 解决 ##

修改flashcache\_subr.c，将 `bi_error`改成`bi_status`。

重新编译，ok。

更新：

flashcache\_subr.c按如下修改，可以兼容以前版本的内核。

> void  
> flashcache\_bio\_endio(struct bio \*bio, int error,  
> struct cache\_c \*dmc, struct timeval \*start\_time)  
> \{  
> if (unlikely(dmc->sysctl\_io\_latency\_hist &&  
> start\_time != NULL &&  
> start\_time->tv\_sec != 0))  
> flashcache\_record\_latency(dmc, start\_time);  
> \#if LINUX\_VERSION\_CODE < KERNEL\_VERSION(2,6,24)  
> bio\_endio(bio, bio->bi\_size, error);  
> \#elif LINUX\_VERSION\_CODE < KERNEL\_VERSION(4,3,0)  
> bio\_endio(bio, error);  
> \#elif LINUX\_VERSION\_CODE < KERNEL\_VERSION(4,13,0)  
> bio->bi\_error = error;  
> bio\_endio(bio);  
> \#else  
> bio->bi\_status = error;  
> bio\_endio(bio);  
> \#endif  
> \}  
>

转载于:https://my.oschina.net/hand/blog/2244957

[80_]: https://my.oschina.net/u/2663968/blog/3051541
[hot3.png]: /images/20220117/859f21a6e0d74647a02156a81f5a9380.png