大数据服务器磁盘空间的分区

叁歲伎倆 2022-12-30 13:45 185阅读 0赞

之前总感觉是在实验室里面做大数据，搞个3台服务器，每台服务器2T磁盘空间，感觉很容易驾驭，这次真的买了8台8\*3750GB的服务器感觉，却有点无从下手了，买回来的是裸服务器，磁盘需要自己挂载。

rpm -qa |grep lvm2
    yum install lvm2
    fdisk -l | grep /dev/sd
    
    pvcreate /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
    pvdisplay
    
    vgcreate vgdata /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
    vgdisplay

当执行`lvcreate`的时候，逻辑卷的大小成了犯难的事情。一台服务器29T，我设置多少个逻辑卷合适呢，最佳实践是什么呢？2T以下的参考[LVM（逻辑卷管理）][LVM]，讲的比较清晰，2T以上就不行了。

[root@bigdata-0001 ~]# vgdisplay
      --- Volume group ---
      VG Name               vgdata
      System ID             
      Format                lvm2
      Metadata Areas        8
      Metadata Sequence No  1
      VG Access             read/write
      VG Status             resizable
      MAX LV                0
      Cur LV                0
      Open LV               0
      Max PV                0
      Cur PV                8
      Act PV                8
      VG Size               <29.11 TiB
      PE Size               4.00 MiB
      Total PE              7630888
      Alloc PE / Size       0 / 0   
      Free  PE / Size       7630888 / <29.11 TiB
      VG UUID               Pviqvq-QYxa-YKjP-2Jej-h3OT-pugw-Ad6kk6

查阅资料[Linux对大于2T的磁盘分区方法及创建逻辑卷LVM][Linux_2T_LVM]，原来linux分区有两种，当磁盘超过2.2T时，就不能采用fdisk进行分区，而应该采用Parted分区。  
先将之前的操作回滚

vgremove vgdata
    pvremove /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
    pvscan
    vgscan

查看初始状态

[root@bigdata-0003 ~]# parted -l
    Error: /dev/sda: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sda: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sdb: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sdb: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sdc: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sdc: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sdd: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sdd: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sde: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sde: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sdf: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sdf: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sdg: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sdg: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Error: /dev/sdh: unrecognised disk label
    Model: ATA ST4000NM0035-1V4 (scsi)                                        
    Disk /dev/sdh: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: unknown
    Disk Flags: 
    
    Model: Virtio Block Device (virtblk)
    Disk /dev/vda: 42.9GB
    Sector size (logical/physical): 512B/512B
    Partition Table: msdos
    Disk Flags: 
    
    Number  Start   End     Size    Type     File system  Flags
    1      1049kB  42.9GB  42.9GB  primary  ext4         boot

参考[分区工具parted的详解及常用分区使用方法][parted]

[root@bigdata-0001 ~]# parted /dev/sda
    GNU Parted 3.1
    Using /dev/sda
    Welcome to GNU Parted! Type 'help' to view a list of commands.
    (parted) mklabel gpt                                                      
    Warning: The existing disk label on /dev/sda will be destroyed and all data on this disk will be lost. Do you want to continue?
    Yes/No? yes
    (parted) mkpart primary ext4 0 -1                                         
    Warning: The resulting partition is not properly aligned for best performance.
    Ignore/Cancel? i                                                          
    (parted) print                                                            
    Model: ATA ST4000NM0035-1V4 (scsi)
    Disk /dev/sda: 4001GB
    Sector size (logical/physical): 512B/512B
    Partition Table: gpt
    Disk Flags: 
    
    Number  Start   End     Size    File system  Name     Flags
    1      17.4kB  4001GB  4001GB               primary
    
    (parted) q                                                                
    Information: You may need to update /etc/fstab.
    
    # 将磁盘格式为ext4
    mkfs.ext4 /dev/sda
    # 磁盘挂载
    mkdir -p /dataa
    mount /dev/sda /dataa
    # 从下面可以看到磁盘空间有了
    [root@bigdata-0001 ~]# df -lh
    Filesystem      Size  Used Avail Use% Mounted on
    devtmpfs         32G     0   32G   0% /dev
    tmpfs            32G     0   32G   0% /dev/shm
    tmpfs            32G  8.8M   32G   1% /run
    tmpfs            32G     0   32G   0% /sys/fs/cgroup
    /dev/vda1        40G  2.3G   35G   7% /
    tmpfs           6.3G     0  6.3G   0% /run/user/0
    /dev/sda        3.6T   89M  3.4T   1% /dataa

其他的磁盘按照这个思路执行

# 这里把-1改成4T,4T作为一整个磁盘
    parted /dev/sdb mklabel gpt mkpart primary ext4 0 4T
    mkfs.ext4 /dev/sdb -y
    mkdir -p /datab
    mount /dev/sdb /datab
    # 这个时候看到了两个大空间
    [root@bigdata-0001 ~]# df -lh
    Filesystem      Size  Used Avail Use% Mounted on
    devtmpfs         32G     0   32G   0% /dev
    tmpfs            32G     0   32G   0% /dev/shm
    tmpfs            32G  8.8M   32G   1% /run
    tmpfs            32G     0   32G   0% /sys/fs/cgroup
    /dev/vda1        40G  2.3G   35G   7% /
    tmpfs           6.3G     0  6.3G   0% /run/user/0
    /dev/sda        3.6T   89M  3.4T   1% /dataa
    /dev/sdb        3.6T   89M  3.4T   1% /datab

参考[parted创建磁盘分区并创建LVM(Linux合并多块大于2T的磁盘并合并到一个分区)][parted_LVM_Linux_2T]这篇文档，可以看到它将多个磁盘合并成为一个逻辑分区，这样做目的是啥呢？大家都这样做吗？如果增加了磁盘，又将怎么办呢？是交给分布式系统，还是按照它的方式去操作呢，还需仔细考虑。  
参考[Linux LVM硬盘管理及LVM扩容][Linux LVM_LVM]，LVM将一个或多个硬盘的分区在逻辑上集合，相当于一个大硬盘来使用，当硬盘的空间不够使用的时候，可以继续将其它的硬盘的分区加入其中，这样可以实现磁盘空间的动态管理，相对于普通的磁盘分区有很大的灵活性。与传统的磁盘与分区相比，LVM为计算机提供了更高层次的磁盘存储。  
这么看来，多个磁盘合成一个分区，是当前主流的最佳实践。所以方案要发生变化了，接下来实践一下，看如何合并  
这里再次回滚

umount /dataa
    umount /datab
    rm -r /dataa
    rm -r /datab
    # 删掉分区,按照同样的方式把/dev/sdb也删掉
    [root@bigdata-0001 ~]# parted /dev/sda
    GNU Parted 3.1
    Using /dev/sda
    Welcome to GNU Parted! Type 'help' to view a list of commands.
    (parted) toggle 1 lvm
    Error: The flag 'lvm' is not available for loop disk labels.
    (parted) rm                                                               
    Partition number? 1                                                       
    (parted) print 
    
    # 创建分区
    [root@bigdata-0001 ~]# parted /dev/sda
    GNU Parted 3.1
    Using /dev/sda
    Welcome to GNU Parted! Type 'help' to view a list of commands.
    (parted) mklabel gpt                                                      
    Warning: The existing disk label on /dev/sda will be destroyed and all data on this disk will be lost. Do you want to continue?
    Yes/No? yes                                                               
    (parted) mkpart primary ext4 0 -1                                         
    Warning: The resulting partition is not properly aligned for best performance.
    Ignore/Cancel? i                                                          
    (parted) toggle 1 lvm                                                     
    (parted) q                                                                
    Information: You may need to update /etc/fstab.
    
    [root@bigdata-0001 ~]# partprobe
    [root@bigdata-0001 ~]# pvcreate /dev/sda1
      Physical volume "/dev/sda1" successfully created.
    [root@bigdata-0001 ~]# pvscan
      PV /dev/sda1                      lvm2 [<3.64 TiB]
      Total: 1 [<3.64 TiB] / in use: 0 [0   ] / in no VG: 1 [<3.64 TiB]
      
    vgcreate vgdata /dev/sda1
    [root@bigdata-0001 ~]# vgdisplay
      --- Volume group ---
      VG Name               vgdata
      System ID             
      Format                lvm2
      Metadata Areas        1
      Metadata Sequence No  1
      VG Access             read/write
      VG Status             resizable
      MAX LV                0
      Cur LV                0
      Open LV               0
      Max PV                0
      Cur PV                1
      Act PV                1
      VG Size               <3.64 TiB
      PE Size               4.00 MiB
      Total PE              953861
      Alloc PE / Size       0 / 0   
      Free  PE / Size       953861 / <3.64 TiB
      VG UUID               BhdgNy-T9Xc-UZMs-tvDN-ImFC-7HgC-1ok5sS

在执行`lvcreate -L 4001G -n lvData vgdata`，提示`Volume group "vgdata" has insufficient free space (953861 extents): 1024256 required.`  
参考[Linux LVM学习总结——Insufficient Free Extents for a Logical Volume][Linux LVM_Insufficient Free Extents for a Logical Volume]，解决方案是

# 每个PE的大小是4M
    953861*4/1024~=3726G
    [root@bigdata-0001 ~]# lvcreate -L 3726G -n lvData vgdata
      Logical volume "lvData" created.
    [root@bigdata-0001 ~]# lvdisplay
      --- Logical volume ---
      LV Path                /dev/vgdata/lvData
      LV Name                lvData
      VG Name                vgdata
      LV UUID                L5ZDpQ-qOF9-ZXUQ-iKZt-JQGl-YnBe-2pPeV6
      LV Write Access        read/write
      LV Creation host, time bigdata-0001, 2020-12-25 23:10:13 +0800
      LV Status              available
      # open                 0
      LV Size                <3.64 TiB
      Current LE             953856
      Segments               1
      Allocation             inherit
      Read ahead sectors     auto
      - currently set to     8192
      Block device           252:0
    
    mkfs.ext4 /dev/vgdata/lvData
    mkdir /appdata
    mount /dev/vgdata/lvData /appdata
    [root@bigdata-0001 ~]# df -lh
    Filesystem                 Size  Used Avail Use% Mounted on
    devtmpfs                    32G     0   32G   0% /dev
    tmpfs                       32G     0   32G   0% /dev/shm
    tmpfs                       32G  8.9M   32G   1% /run
    tmpfs                       32G     0   32G   0% /sys/fs/cgroup
    /dev/vda1                   40G  2.3G   35G   7% /
    tmpfs                      6.3G     0  6.3G   0% /run/user/0
    /dev/mapper/vgdata-lvData  3.6T   89M  3.4T   1% /appdata
    
    # 开机自动挂载
    echo "/dev/vgdata/lvData /appdata ext4 defaults 0 0" >> /etc/fstab

下面挂载第二块磁盘

parted /dev/sdb
    mklabel gpt 
    mkpart primary ext4 0 -1
    toggle 1 lvm
    q
    partprobe
    # 普通磁盘转换成PV
    pvcreate /dev/sdb1
    [root@bigdata-0001 ~]# pvs
      PV         VG     Fmt  Attr PSize  PFree 
      /dev/sda1  vgdata lvm2 a--  <3.64t 20.00m
      /dev/sdb1         lvm2 ---  <3.64t <3.64t
    
    vgextend vgdata /dev/sdb1
    [root@bigdata-0001 ~]# vgdisplay
      --- Volume group ---
      VG Name               vgdata
      System ID             
      Format                lvm2
      Metadata Areas        2
      Metadata Sequence No  3
      VG Access             read/write
      VG Status             resizable
      MAX LV                0
      Cur LV                1
      Open LV               1
      Max PV                0
      Cur PV                2
      Act PV                2
      VG Size               <7.28 TiB
      PE Size               4.00 MiB
      Total PE              1907722
      Alloc PE / Size       953856 / <3.64 TiB
      Free  PE / Size       953866 / <3.64 TiB
      VG UUID               BhdgNy-T9Xc-UZMs-tvDN-ImFC-7HgC-1ok5sS
    
    # 扩容
    lvextend -L +3726G /dev/vgdata/lvData
    # resize2fs更新后才能识别
    resize2fs /dev/vgdata/lvData
    # 执行下面的命令，成功了
    [root@bigdata-0001 ~]# df -lh
    Filesystem                 Size  Used Avail Use% Mounted on
    devtmpfs                    32G     0   32G   0% /dev
    tmpfs                       32G     0   32G   0% /dev/shm
    tmpfs                       32G  8.9M   32G   1% /run
    tmpfs                       32G     0   32G   0% /sys/fs/cgroup
    /dev/vda1                   40G  2.3G   35G   7% /
    tmpfs                      6.3G     0  6.3G   0% /run/user/0
    /dev/mapper/vgdata-lvData  7.2T   51M  6.9T   1% /appdata

接下啦重复操作,这里最后在重新梳理，最终形成：

rpm -qa |grep lvm2
    yum install lvm2 -y
    # 查看有多少块盘
    fdisk -l | grep /dev/sd
    
    # 第一块磁盘
    parted /dev/sda
    mklabel gpt
    mkpart primary ext4 0 -1
    i
    toggle 1 lvm
    q
    partprobe
    pvcreate /dev/sda1
    pvs
    vgcreate vgdata /dev/sda1
    vgdisplay
    lvcreate -L 3726G -n lvData vgdata
    mkfs.ext4 /dev/vgdata/lvData
    mkdir /appdata
    mount /dev/vgdata/lvData /appdata
    echo "/dev/vgdata/lvData /appdata ext4 defaults 0 0" >> /etc/fstab
    
    parted /dev/sdb
    mklabel gpt
    mkpart primary ext4 0 -1
    i
    toggle 1 lvm
    q
    partprobe
    pvcreate /dev/sdb1
    pvs
    vgextend vgdata /dev/sdb1
    vgdisplay
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    # c块磁盘
    parted /dev/sdc
    mklabel gpt
    mkpart primary ext4 0 -1
    i
    toggle 1 lvm
    q
    partprobe
    pvcreate /dev/sdc1
    pvs
    vgextend vgdata /dev/sdc1
    vgdisplay
    # 3726G来自Free  PE / Size       953871 / <3.64 TiB
    # 953871*4/1024~=3726G
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    
    parted /dev/sdd
    mklabel gpt
    mkpart primary ext4 0 -1
    toggle 1 lvm
    i
    q
    partprobe
    pvcreate /dev/sdd1
    pvs
    vgextend vgdata /dev/sdd1
    vgdisplay
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    
    parted /dev/sde
    mklabel gpt
    mkpart primary ext4 0 -1
    toggle 1 lvm
    i
    q
    partprobe
    pvcreate /dev/sde1
    pvs
    vgextend vgdata /dev/sde1
    vgdisplay
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    
    parted /dev/sdf
    mklabel gpt
    mkpart primary ext4 0 -1
    i
    toggle 1 lvm
    q
    partprobe
    pvcreate /dev/sdf1
    pvs
    vgextend vgdata /dev/sdf1
    vgdisplay
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    
    parted /dev/sdg
    mklabel gpt
    mkpart primary ext4 0 -1
    i
    toggle 1 lvm
    q
    partprobe
    pvcreate /dev/sdg1
    pvs
    vgextend vgdata /dev/sdg1
    vgdisplay
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    
    parted /dev/sdh
    mklabel gpt
    mkpart primary ext4 0 -1
    i
    toggle 1 lvm
    q
    partprobe
    pvcreate /dev/sdh1
    pvs
    vgextend vgdata /dev/sdh1
    vgdisplay
    lvextend -L +3726G /dev/vgdata/lvData
    resize2fs /dev/vgdata/lvData
    df -lh

[LVM]: https://blog.csdn.net/qq_40628106/article/details/83476302
[Linux_2T_LVM]: https://blog.csdn.net/qq_38265137/article/details/97489823
[parted]: https://www.cnblogs.com/wangmo/p/9243658.html
[parted_LVM_Linux_2T]: https://www.cnblogs.com/mouseleo/p/13825713.html
[Linux LVM_LVM]: https://www.cnblogs.com/gaojun/archive/2012/08/22/2650229.html
[Linux LVM_Insufficient Free Extents for a Logical Volume]: https://blog.csdn.net/wxb880114/article/details/90023677