Keepalived+drbd+nfs 实现nfs数据同步

落日映苍穹つ 2022-03-11 17:38 449阅读 0赞

# 前言 #

Centos7.5下Keepalive+drbd+nfs练习笔记

# 主机准备 #

drbd服务主机需要可用的分区或者磁盘

<table> 
 <thead> 
  <tr> 
   <th>主机名</th> 
   <th>IP</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>test(部署及nfs客户端)</td> 
   <td>192.168.0.35</td> 
  </tr> 
  <tr> 
   <td>drbd-0001</td> 
   <td>192.168.0.81</td> 
  </tr> 
  <tr> 
   <td>drbd-0002</td> 
   <td>192.168.0.82</td> 
  </tr> 
 </tbody> 
</table>

# 部署test主机ansible #

ansible 环境部署

[root@test ~]# yum -y install ansible
    [root@test ~]# vim /etc/hosts
    192.168.0.81 drbd-0001
    192.168.0.82 drbd-0002
    [root@test ~]# vim /etc/ansible/ansible.cfg 
    [defaults]
    inventory      = /etc/ansible/hosts
    host_key_checking = False
    [root@test ~]# vim /etc/ansible/hosts 
    [drbd]
    drbd-000[1:2]
    [root@test ~]# ssh-copy-id drbd-0001
    [root@test ~]# ssh-copy-id drbd-0002

ansible 环境检测

[root@test ~]# ansible drbd -m ping
    drbd-0001 | SUCCESS => { 
        "changed": false, 
        "ping": "pong"
    }
    drbd-0002 | SUCCESS => { 
        "changed": false, 
        "ping": "pong"
    }

# drbd部署 #

检查可用磁盘

[root@test ~]# ansible drbd -m shell -a 'ls /dev/vdb'
    drbd-0002 | SUCCESS | rc=0 >>
    /dev/vdb
    
    drbd-0001 | SUCCESS | rc=0 >>
    /dev/vdb

磁盘分区 vdb1作为数据盘 vdb2作为元数据存储

[root@test ~]# ansible drbd -m shell -a 'parted /dev/vdb mklabel GPT ;parted /dev/vdb mkpart primary 1M 90%;parted /dev/vdb mkpart primary 90% 100%'
    [root@test ~]# ansible drbd -m shell -a 'lsblk'
    drbd-0001 | SUCCESS | rc=0 >>
    NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
    vda    253:0    0   40G  0 disk 
    └─vda1 253:1    0   40G  0 part /
    vdb    253:16   0   10G  0 disk 
    ├─vdb1 253:17   0    9G  0 part 
    └─vdb2 253:18   0 1023M  0 part 
    
    drbd-0002 | SUCCESS | rc=0 >>
    NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
    vda    253:0    0   40G  0 disk 
    └─vda1 253:1    0   40G  0 part /
    vdb    253:16   0   10G  0 disk 
    ├─vdb1 253:17   0    9G  0 part 
    └─vdb2 253:18   0 1023M  0 part

配置/etc/hosts **此处主机名必须和drbd主机名一致**

[root@test ~]# vim hosts
    ::1	localhost	localhost.localdomain	localhost6	localhost6.localdomain6
    127.0.0.1	localhost	localhost.localdomain	localhost4	localhost4.localdomain4
    192.168.0.81 drbd-0001
    192.168.0.82 drbd-0002
    [root@test ~]# ansible drbd -m copy -a 'src=hosts dest=/etc/hosts'

安装drbd84和kmod-drbd84

[root@test ~]# ansible drbd -m yum -a 'name=drbd84-utils,kmod-drbd84 state=installed'

加载模块

[root@test ~]# ansible drbd -m shell -a 'modprobe drbd'
    drbd-0002 | FAILED | rc=1 >>
    modprobe: FATAL: Module drbd not found.non-zero return code
    
    drbd-0001 | FAILED | rc=1 >>
    modprobe: FATAL: Module drbd not found.non-zero return code

若有以上报错,需要再安装内核升级包,再加载模块

[root@test ~]# ansible drbd -m yum -a 'name=kernel-devel,kernel,kernel-headers state=installed'
    [root@test ~]# ansible drbd -m shell -a 'reboot'
    [root@test ~]# ansible drbd -m shell -a 'modprobe drbd'
    [root@test ~]# ansible drbd -m shell -a 'lsmod | grep drbd'
    drbd-0001 | SUCCESS | rc=0 >>
    drbd                  397041  0 
    libcrc32c              12644  1 drbd
    
    drbd-0002 | SUCCESS | rc=0 >>
    drbd                  397041  0 
    libcrc32c              12644  1 drbd

给/dev/vdb1创建文件系统 **元数据盘不能创建文件系统**

[root@test ~]# ansible drbd -m shell -a 'mkfs.ext4 /dev/vdb1'

编辑drbd配置文件

[root@test ~]# vim global_common.conf
    global { 
        usage-count no;        
    }
    common { 
        protocol C;                     #c模式传输(此模式更可靠)
        disk { 
     	    on-io-error detach;
        }
        syncer { 
    	    rate 100M;                  #设定传输速率
        }
    }
    resource data {                      #为资源起名为data
        on drbd-0001 {                   #节点名必须为主机名
      	    device /dev/drbd1;          #为drbd虚拟磁盘取名
    	    disk /dev/vdb1;             #指定数据盘
    	    address 192.168.0.81:7899;
    	    meta-disk /dev/vdb2[0];     #指定元数据盘
        }
        on drbd-0002 {               
    	    device /dev/drbd1;      
    	    disk /dev/vdb1;
    	    address 192.168.0.82:7899;
    	    meta-disk /dev/vdb2[0];
        }
    }
    [root@test ~]# ansible drbd -m copy -a 'src=global_common.conf dest=/etc/drbd.d/global_common.conf'

创建 资源名为data的元数据

[root@test ~]# ansible drbd -m shell -a 'drbdadm create-md data'

启动 资源名为data的元数据

[root@test ~]# ansible drbd -m shell -a 'drbdadm up data'

在drbd-0001节点上查看信息 此时2者均为从状态

[root@test ~]# ansible drbd-0001 -m shell -a 'cat /proc/drbd'
    drbd-0001 | SUCCESS | rc=0 >>
    version: 8.4.11-1 (api:1/proto:86-101)
    GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by mockbuild@, 2018-11-03 01:26:55
    
     1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r-----
        ns:0 nr:0 dw:0 dr:0 al:8 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:9435904

设定drbd-0001为主并同步数据

[root@test ~]# ansible drbd-0001 -m shell -a 'drbdadm -- --overwrite-data-of-peer primary data'

此时查看,正在同步数据

[root@test ~]# ansible drbd-0001 -m shell -a 'drbdadm -- --overwrite-data-of-peer primary data'
    [root@test ~]# ansible drbd-0001 -m shell -a 'cat /proc/drbd'drbd-0001 | SUCCESS | rc=0 >>
    version: 8.4.11-1 (api:1/proto:86-101)
    GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by mockbuild@, 2018-11-03 01:26:55
    
     1: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r-----
        ns:1028096 nr:0 dw:0 dr:1030168 al:8 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:8408064
    	[=>..................] sync'ed: 11.0% (8208/9212)M
    	finish: 0:03:31 speed: 39,680 (39,540) K/sec

数据同步完成后,drbd-0001为主,且数据同步完成则配置完成

[root@test ~]# ansible drbd-0001 -m shell -a 'cat /proc/drbd'
    drbd-0001 | SUCCESS | rc=0 >>
    version: 8.4.11-1 (api:1/proto:86-101)
    GIT-hash: 66145a308421e9c124ec391a7848ac20203bb03c build by mockbuild@, 2018-11-03 01:26:55
    
     1: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r-----
        ns:9436160 nr:0 dw:0 dr:9438232 al:8 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0

测试drbd配置 将虚拟磁盘/dev/drbd1 挂载/data 创建测试文件

[root@test ~]# ansible drbd-0001 -m shell -a 'mkdir /data;mount /dev/drbd1 /data;touch /data/a.txt'

在drbd-0002节点上关闭drbd服务,挂载/dev/vdb1 查看测试文件是否存在

[root@test ~]# ansible drbd-0002 -m shell -a 'drbdadm down data;mkdir /data ;mount /dev/vdb1 /data;ls /data'
    drbd-0002 | SUCCESS | rc=0 >>
    a.txt
    lost+found

恢复drbd节点的drbd服务

[root@test ~]# ansible drbd-0002 -m shell -a 'umount /data;drbdadm up data'

# keepavle+nfs部署 #

安装keepalived 和nfs-utils,192.168.0.210为vip

[root@test ~]# ansible drbd -m yum -a 'name=keepalived,nfs-utils state=installed'

编辑keepalived配置文件  
drbd-0001配置为

[root@drbd-0001 ~]# vim /etc/keepalived/keepalived.conf
    global_defs { 
       notification_email { 
         acassen@firewall.loc
         failover@firewall.loc
         sysadmin@firewall.loc
       }
       notification_email_from Alexandre.Cassen@firewall.loc
       smtp_server 192.168.200.1
       smtp_connect_timeout 30
       router_id drbd1
       vrrp_skip_check_adv_addr
       vrrp_garp_interval 0
       vrrp_gna_interval 0
    }
    vrrp_script check_service { 
       script "ss -nualpt | grep 2049" 
       interval 2
       weight -50
    }
    
    vrrp_instance VI_1 { 
        state MASTER
        interface eth0
        virtual_router_id 51
        priority 105
        advert_int 1
        authentication { 
            auth_type PASS
            auth_pass 1111
        }
        track_script { 
            check_service
        }
        notify_master /etc/keepalived/notify_master.sh
        notify_backup /etc/keepalived/notify_backup.sh
        notify_stop /etc/keepalived/notify_stop.sh
        virtual_ipaddress { 
            192.168.0.210 dev eth0 label eth0:1
        }
    }

drbd-0002的配置为

[root@drbd-0002 ~]# vim /etc/keepalived/keepalived.conf
    global_defs { 
       notification_email { 
         acassen@firewall.loc
         failover@firewall.loc
         sysadmin@firewall.loc
       }
       notification_email_from Alexandre.Cassen@firewall.loc
       smtp_server 192.168.200.1
       smtp_connect_timeout 30
       router_id drbd2
       vrrp_skip_check_adv_addr
       vrrp_garp_interval 0
       vrrp_gna_interval 0
    }
    vrrp_script check_service { 
       script "ss -nualpt | grep 2049 "
       interval 2
       weight -50
    }
    
    vrrp_instance VI_1 { 
        state BACKUP
        interface eth0
        virtual_router_id 51
        priority 95
        advert_int 1
        authentication { 
            auth_type PASS
            auth_pass 1111
        }
        track_script { 
            check_service
        }
        notify_master /etc/keepalived/notify_master.sh
        notify_backup /etc/keepalived/notify_backup.sh
        notify_stop /etc/keepalived/notify_stop.sh
        virtual_ipaddress { 
            192.168.0.210 dev eth0 label eth0:1
        }
    }

配置nfs文件

[root@test ~]# vim exports
    /data 192.168.0.0/24(ro)
    [root@test ~]# ansible drbd -m copy -a 'src=exports dest=/etc/'

配置脚本文件

[root@test ~]# vim notify_master.sh 
    #!/bin/bash
    drbdadm primary data
    mount /dev/drbd1 /data
    systemctl start nfs
    [root@test ~]# vim notify_backup.sh 
    #!/bin/bash
    systemctl stop nfs
    umount /data
    drbdadm secondary data
    [root@test ~]# vim notify_stop.sh 
    #!/bin/bash
    systemctl stop nfs
    umount /data
    drbdadm secondary data
    systemctl restart nfs
    [root@test ~]# ansible drbd -m copy -a 'src=notify_master.sh dest=/etc/keepalived/'
    [root@test ~]# ansible drbd -m copy -a 'src=notify_backup.sh dest=/etc/keepalived/'
    [root@test ~]# ansible drbd -m copy -a 'src=notify_stop.sh dest=/etc/keepalived/'
    [root@test ~]# ansible drbd -m shell -a 'chmod +x /etc/keepalived/noti*'

# 启动服务调试 #

drbd主从检查

[root@test ~]# ansible drbd -m shell -a 'cat /proc/drbd'

启动nfs服务

ansible drbd -m shell -a 'systemctl start nfs'

启动keepalive

[root@test ~]# ansible drbd -m shell -a 'systemctl start keepalived'

nfs客户端挂载

[root@test ~]# mkdir /data
    [root@test ~]# mount 192.168.0.210:/data /data/

在主节点停止keepalive服务或者nfs服务测试