恢复HDFS服务: 删除了VERSION文件

矫情吗；* 2023-05-30 08:48 20阅读 0赞

### 文章目录 ###

*  1, 备份hdfs元数据
 *   *  namenode备份，恢复
     *   *  a, 备份
         *  b, 恢复
 *  2，备份hdfs(nn,snn,dn)元数据文件
 *   *  a, VERSION文件格式: namenode, datanode
     *  b, 修复VERSION文件: namenode, datanode

*  需求说明：　由于服务器被攻击，hdfs集群的VERSION文件被删除，看似集群正常运行，但遇到修改配置，需要重启hdfs服务时，就会报错（namenode 未格式化）

# 1, 备份hdfs元数据 #

## namenode备份，恢复 ##

### a, 备份 ###

[hdfs@cdh-node1 ~]$ crontab -l
    #VERSION
    * *   1 * * cp /dfs/nn/current/VERSION ~/VERSION.bak
    #fsimage: 格式_000(20个0)
    * */2 * * * hdfs dfsadmin -fetchImage ~/fsimage_0000000000000000000
    
    [hdfs@cdh-node1 ~]$ ll -h
    total 76K
    -rw-r--r-- 1 hdfs hdfs 71K 3月  19 10:37 fsimage_0000000000000000000
    -rw-r--r-- 1 hdfs hdfs 174 3月  19 10:36 VERSION.bak

### b, 恢复 ###

1, Add a new host to your Hadoop cluster.
    2, Add the NameNode role to the host. Make sure it has the same hostname as the original NameNode.
    3, Create a directory path for the NameNode name.dir (for example, /dfs/nn/current), ensuring that the permissions are set correctly.
    4, Copy the VERSION and latest fsimage file to the /dfs/nn/current directory.
    5, Run the following command to create the md5 file for the fsimage
    
    [hdfs@cdh-node1 ~]$ ll /dfs/nn/current/ |tail -4
    -rw-r--r--  1 hdfs hdfs   72554 3月  19 10:37 fsimage_0000000000000000000
    -rw-r--r--  1 hdfs hdfs      62 3月  19 10:37 fsimage_0000000000000000000.md5
    -rw-r--r--. 1 hdfs hdfs     174 3月  19 10:08 VERSION
    
    #启动后，生成的文件如下：
    [hdfs@cdh-node1 current]$ ll
    total 1112
    drwxrwxr-x  3 hdfs hdfs    4096 3月  19 11:01 default
    -rw-r--r--  1 hdfs hdfs 1048576 3月  19 11:18 edits_inprogress_0000000000000015393
    -rw-r--r--  1 hdfs hdfs   72554 3月  19 11:12 fsimage_0000000000000000000
    -rw-rw-r--  1 hdfs hdfs      62 3月  19 11:12 fsimage_0000000000000000000.md5
    -rw-r--r--  1 hdfs hdfs       6 3月  19 11:13 seen_txid
    -rw-r--r--. 1 hdfs hdfs     174 3月  19 10:08 VERSION
    
    [hdfs@cdh-node1 current]$ cat seen_txid
    15393

# 2，备份hdfs(nn,snn,dn)元数据文件 #

[hdfs@cdh-node1 ~]$ cat a.sh 
    #!/bin/bash
    #desc : 检查&&备份cdh集群，hdfs的version文件是否正常,hdfs数据文件是否有丢失
    
    #hdfs数据目录: (example: /a/b ==>but not /a/b/)
    hdfs_dn_dir=/dfs/dn
    hdfs_nn_dir=/dfs/nn
    hdfs_sn_dir=/dfs/sn
    
    backup_dir=/bak/cdh            #备份文件路径(自动创建)
    suffix=$(date +%Y_%m_%d_%H:%M) #备份文件的后缀名
    backup_log=/var/log/cdh-check-version.log
    
    #######################check hdfs all data files#####################
    is_hdfs_good=true
    DN_STATS=0
    SN_STATS=0
    NN_STATS=0
    
    echo "start check/backup cdh version files ============${suffix} ==========="   >> ${backup_log}
    [ -d $backup_dir ] || mkdir -p $backup_dir
    [ -d $backup_tar ] || mkdir -p $backup_tar
    
    #1,dn
    if [ -d ${hdfs_dn_dir} ] ;then
        (cd ${hdfs_dn_dir}/current/; \cp VERSION $backup_dir/dn.current.VERSION.$suffix ; ls VERSION &> /dev/null )
        dn_res1=$(echo $?)
        
        (cd ${hdfs_dn_dir}/current/BP*/current; \cp VERSION $backup_dir/BP.current.VERSION.$suffix ; ls VERSION &>/dev/null)
        dn_res2=$(echo $?)
        
        DN_STATS=$(( dn_res1 + dn_res2 ))
    
        #list last 10 files on dn data dir
        totsize=$( du -hs ${ hdfs_dn_dir} )
        echo "du -hs ${hdfs_dn_dir}: $totsize " >> ${backup_log}
        (cd  ${hdfs_dn_dir}/current/BP* ; ls current -ltRr  |head -10 >> ${backup_log} )
    fi
    
    
    #2,sn
    if [ -d ${hdfs_sn_dir} ] ;then
        [ ! -e ${hdfs_sn_dir}/current/VERSION ] && SN_STATS=1
        \cp ${hdfs_sn_dir}/current/VERSION $backup_dir/sn.current.VERSION.$suffix 
    
        #list last 10 files on dn data dir
        totsize=$( du -hs ${ hdfs_sn_dir} )
        echo "du -hs ${hdfs_sn_dir}: $totsize " >> ${backup_log}
        (cd  ${hdfs_sn_dir}/current/ ; ls  -ltRr  |head -10 >> ${backup_log} )
    fi
    
    #3,nn
    if [ -d ${hdfs_nn_dir} ] ;then
        [ ! -e ${hdfs_nn_dir}/current/VERSION ] && NN_STATS=1
        \cp ${hdfs_nn_dir}/current/VERSION $backup_dir/nn.current.VERSION.$suffix
        
        #list last 10 files on dn data dir
        totsize=$( du -hs ${ hdfs_nn_dir} )
        echo "du -hs ${hdfs_nn_dir}: $totsize " >> $backup_log
        (cd  ${hdfs_nn_dir}/current/ ; ls -ltRr  |head -10 >> $backup_log)
    fi
    
    #4,total res
    res_total=$(( DN_STATS + SN_STATS + NN_STATS ))
    [ $res_total -ne 0 ] && is_hdfs_good=false
    
    if [ ${is_hdfs_good} = "false" ] ;then
        echo -e "\033[05;31m============$(date "+%Y %m %d %H:%M:%S"), error, hdfs version file mayby missing..........===========\033[0m"
    else
        echo -e "\033[32m============$(date "+%Y %m %d %H:%M:%S"), good, hdfs version file exists ! ===========\033[0m" 
    fi

## a, VERSION文件格式: namenode, datanode ##

<table> 
 <thead> 
  <tr> 
   <th align="left">文件路径</th> 
   <th align="left">字段内容</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td align="left">dn/current/VERSION</td> 
   <td align="left">clusterID,storageType,layoutVersion， storageID, datanodeUuid</td> 
  </tr> 
  <tr> 
   <td align="left">dn/current/BP-xx-NN节点IP-xx/current/VERSION</td> 
   <td align="left">namespaceID, cTime=0, blockpoolID,layoutVersion</td> 
  </tr> 
  <tr> 
   <td align="left">nn/current/VERSION</td> 
   <td align="left">clusterID,storageType,　字段同上</td> 
  </tr> 
 </tbody> 
</table>

[root@centos_server dfs]# cat dn/current/VERSION 
    #Tue Nov 05 18:46:13 CST 2019
    storageID=DS-8f11dac8-1585-40b9-979d-3028ffa241dd
    clusterID=cluster13
    cTime=0
    datanodeUuid=907edbd1-7126-457e-890c-1e8550cb5b1c
    storageType=DATA_NODE
    layoutVersion=-56
    
    # cat dn/current/具体的blockpoolID－对应的目录/current/VERSION
    [root@centos_server dfs]# cat dn/current/BP-782628945-服务器具体IP-1572928546988/current/VERSION 
    #Tue Nov 05 18:46:13 CST 2019
    namespaceID=495849129
    cTime=0
    blockpoolID=BP-782628945-服务器具体IP-1572928546988
    layoutVersion=-56
    
    
    [root@centos_server dfs]# cat nn/current/VERSION 
    #Tue Nov 05 12:35:47 CST 2019
    namespaceID=495849129
    clusterID=cluster13
    cTime=0
    storageType=NAME_NODE
    blockpoolID=BP-782628945-服务器具体IP-1572928546988
    layoutVersion=-60

## b, 修复VERSION文件: namenode, datanode ##

###============================ 日志结构：　滚动(DATANODE.out.数字/NAMENODE.out.数字)
    [hbase@centos_server hadoop-hdfs]$ ls
    hadoop-cmf-hdfs-DATANODE-centos_server.log.out           hdfs-audit.log     hdfs-audit.log.17  hdfs-audit.log.7
    hadoop-cmf-hdfs-DATANODE-centos_server.log.out.1         hdfs-audit.log.1   hdfs-audit.log.18  hdfs-audit.log.8
    hadoop-cmf-hdfs-DATANODE-centos_server.log.out.2         hdfs-audit.log.10  hdfs-audit.log.19  hdfs-audit.log.9
    hadoop-cmf-hdfs-NAMENODE-centos_server.log.out           hdfs-audit.log.11  hdfs-audit.log.2   SecurityAuth-hdfs.audit
    hadoop-cmf-hdfs-NAMENODE-centos_server.log.out.1         hdfs-audit.log.12  hdfs-audit.log.20  SecurityAuth-root.audit
    hadoop-cmf-hdfs-NAMENODE-centos_server.log.out.2         hdfs-audit.log.13  hdfs-audit.log.3   SecurityAuth-yskj.audit
    hadoop-cmf-hdfs-NAMENODE-centos_server.log.out.3         hdfs-audit.log.14  hdfs-audit.log.4   stacks
    hadoop-cmf-hdfs-NAMENODE-centos_server.log.out.4         hdfs-audit.log.15  hdfs-audit.log.5
    hadoop-cmf-hdfs-SECONDARYNAMENODE-centos_server.log.out  hdfs-audit.log.16  hdfs-audit.log.6
    
    ###============================= DataNode节点日志　=============================
    ----------find storageID: storageID
    [root@eadage hadoop-hdfs]# grep storageID　 hadoop-cmf-hdfs-DATANODE-eadage.log.out
    2019-11-05 12:44:08,920 INFO org.apache.hadoop.hdfs.server.common.Storage: Generated new storageID DS-8f11dac8-1585-40b9-979d-3028ffa241dd for directory /home/cdh/dfs/dn
    2019-11-05 12:44:10,490 INFO org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl: Added new volume: DS-8f11dac8-1585-40b9-979d-3028ffa241dd
    
    ----------find blockpoolID: blockid
    [hbase@eadage hadoop-hdfs]$ grep blockid hadoop-cmf-hdfs-DATANODE-eadage.log.out 
    2019-11-08 15:26:02,567 INFO org.apache.hadoop.hdfs.server.datanode.DataNode.clienttrace: src: /NN节点具体IP:62657,
    dest: /NN节点具体IP:50010, bytes: 521, op: HDFS_WRITE, cliID: DFSClient_NONMAPREDUCE_1622832045_1, offset: 0, 
    srvID: 907edbd1-7126-457e-890c-1e8550cb5b1c, blockid: BP-782628945-NN节点具体IP-1572928546988:blk_1073767910_27086, duration: 1520832
    
    ----------find datanodeUuid: UUID
    [root@eadage hadoop-hdfs]# grep UUID hadoop-cmf-hdfs-DATANODE-eadage.log.out |more
    2019-11-05 12:44:10,441 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Generated and persisted new Datanode UUID 907edbd1-7126-457e-890
    c-1e8550cb5b1c
    2019-11-05 12:44:10,539 INFO org.apache.hadoop.hdfs.server.datanode.DataNode: Block pool BP-782628945-NN节点具体IP-1572928546988 (Datanode Uu
    id 907edbd1-7126-457e-890c-1e8550cb5b1c) service to eadage/NN节点具体IP:8022 beginning handshake with NN
    
    ###============================= NameNode节点日志　=============================
    ----------find namespaceID: nsid ,  clusterID:cid ,   datanodeUuid:datanodeUuid
    [hbase@eadage hadoop-hdfs]$ grep nsid  hadoop-cmf-hdfs-NAMENODE-eadage.log.out 
    2019-11-06 22:48:11,345 INFO BlockStateChange: BLOCK* processReport 0x1cdb2e6a2337c49b: from storage DS-8f11dac8-1585-40b9-979d-3028ffa241dd node DatanodeRegistration(NN节点具体IP, 
    datanodeUuid=907edbd1-7126-457e-890c-1e8550cb5b1c, infoPort=50075, infoSecurePort=0, ipcPort=50020, storageInfo=lv=-56;
    cid=cluster13;nsid=495849129;c=0), 
    blocks: 4956, hasStaleStorage: false, processing time: 6 msecs, invalidatedBlocks: 0
    2019-11-07 04:48:12,279 INFO BlockStateChange: BLOCK* processReport 0x1cdb2e6a2337c49c: from storage DS-8f11dac8-1585-40b9-979d-3028ffa241dd node DatanodeRegistration(NN节点具体IP, 
    datanodeUuid=907edbd1-7126-457e-890c-1e8550cb5b1c, infoPort=50075, infoSecurePort=0, ipcPort=50020, storageInfo=lv=-56;
    cid=cluster13;nsid=495849129;c=0), 
    blocks: 4957, hasStaleStorage: false, processing time: 3 msecs, invalidatedBlocks: 0