Linux shell 脚本规范

我会带着你远行 2022-05-25 00:16 269阅读 0赞

Linux 运维以及脚本干货: 欢迎分享，欢迎folk  
[https://unnunique.github.io/AADocs/skill-docs/linux/][https_unnunique.github.io_AADocs_skill-docs_linux]  
[https://github.com/unnunique/unnunique.github.io/blob/master/AADocs/skill-docs/linux/index.md][https_github.com_unnunique_unnunique.github.io_blob_master_AADocs_skill-docs_linux_index.md]

Linux shell 脚本规范

#!/bin/bash
    ################################################################################
    ## Copyright: HZGOSUN Tech. Co, BigData
    ## Filename: schema-merge-parquet-file.sh
    ## Description: 定时启动合并小文件的脚本,包括两个功能
    ## 1,合并临时表mid_tabel 中的parquet 文件
    ## 2，定时合并当天的person_table 中的数据。
    ## 3，合并前一天的person_table 中的数据。
    ## 4, 额外的功能，对person 表某一天的数据进行合并。
    ## Author: lidiliang
    ## Created: 2017-11-06
    ## Modified: 2017-12-01(lidiliang)
    ################################################################################
    #set -x ## 用于调试用，不用的时候可以注释掉
    
    ### TODO公共变量统一
    #---------------------------------------------------------------------#
    # 公共变量统一 #
    #---------------------------------------------------------------------#
    
    
    #---------------------------------------------------------------------#
    # 变量定义 #
    #---------------------------------------------------------------------#
    source /etc/profile
    cd `dirname $0`
    BIN_DIR=`pwd`    ### bin目录
    cd ..
    DEPLOY_DIR=`pwd`
    CONF_DIR=$DEPLOY_DIR/conf    ### 项目根目录 ## log 日记目录
    LOG_DIR=$DEPLOY_DIR/logs
    LOG_FILE=${LOG_DIR}/schema-merge-parquet-file.log        ## log 日记文件
    cd ..
    declare -r BIGDATA_SERVICE_DIR=`pwd`
    declare -r COMMMON_DIR=${BIGDATA_SERVICE_DIR}/common
    declare -r FTP_DIR=${BIGDATA_SERVICE_DIR}/ftp
    declare -r SERVICE=${BIGDATA_SERVICE_DIR}/service
    declare -r CLUSTER_DIR=${BIGDATA_SERVICE_DIR}/cluster
    CLUSTER_VERSION=`ls ${COMMMON_DIR}/lib| grep ^cluster-[0-9].[0-9].[0-9].jar$`
    SPARKJOB_PROPERTIES=${CONF_DIR}/sparkJob.properties
    hdfsClusterName=$(grep job.smallfile.merge.hdfs.name  ${SPARKJOB_PROPERTIES} | awk -F "=" '{print $2}')
    tmpTableHdfsPath=$(grep job.smallfile.merge.mid_table.hdfs_path  ${SPARKJOB_PROPERTIES} | awk -F "=" '{print $2}')
    hisTableHdfsPath=$(grep job.smallfile.merge.person_table.hdfs_path  ${SPARKJOB_PROPERTIES} | awk -F "=" '{print $2}')
    tableName=$(grep job.smallfile.merge.person_table.name   ${SPARKJOB_PROPERTIES}| awk -F "=" '{print $2}')
    dateString=""
    BIGDATA_ENV_FILE=/opt/hzgc/env_bigdata.sh
    source ${BIGDATA_ENV_FILE}
    mkdir -p ${LOG_DIR}
    
    #---------------------------------------------------------------------#
    # 函数定义 #
    #---------------------------------------------------------------------#
    
    #####################################################################
    # 函数名: showUsage
    # 描述: 输入参数不对的时候的提示信息
    # 参数: N/A
    # 返回值: N/A
    # 其他: N/A
    #####################################################################
    function showUsage() {
        echo "usage: "
        echo "1, sh schema-merge-parquet-file.sh mid_table to 和并临时表mid_table 的parquet 文件."
        echo "2, sh schema-merge-parquet-file.sh person_table now 合并person_table 表中当天的parquet文件. "
        echo "3, sh schema-merge-parquet-file.sh person_table before 合并person_table 表格中前一天的parquet文件."
        echo "4, sh schema-merge-parquet-file.sh person_table 2018-01-03 合并person_table 表格中指定某天的文件."
        exit 1;
    }
    
    #####################################################################
    # 函数名: merge_parquet
    # 描述: 小文件合并
    # 参数: N/A
    # 返回值: N/A
    # 其他: N/A
    #####################################################################
    function merge_parquet()
    {
        if [ ! -d $LOG_DIR ]; then
            mkdir $LOG_DIR;
        fi
        if [ "${dateString}" == "" ];then
            spark-submit --class com.hzgc.cluster.smallfile.MergeParquetFileV1 \
            --master local[*] \
            --driver-memory 4g \
            ${COMMMON_DIR}/lib/${CLUSTER_VERSION} ${hdfsClusterName} ${tmpTableHdfsPath} ${hisTableHdfsPath} ${tableName} ${dateString}
            checkReturn $?
        else
            spark-submit --class com.hzgc.cluster.smallfile.MergeParquetFileV2 \
            --master local[*] \
            --driver-memory 4g \
            ${COMMMON_DIR}/lib/${CLUSTER_VERSION} ${hdfsClusterName} ${tmpTableHdfsPath} ${hisTableHdfsPath} ${tableName} ${dateString}
            checkReturn $?
        fi
        if [ $? -eq 0 ];then
            echo "==================================================="  | tee -a ${LOG_FILE}
            echo "$(date "+%Y-%m-%d %H:%M:%S") Schema merge parquet file Success!"  | tee  -a  ${LOG_FILE}
        else
            echo "==================================================="  | tee -a ${LOG_FILE}
            echo "$(date "+%Y-%m-%d %H:%M:%S") Schema merge parquet file Failure!"     | tee  -a  ${LOG_FILE}
            exit 1
        fi
    }
    
    #####################################################################
    # 函数名: showRunTime
    # 描述: 显示脚本执行的时间
    # 参数: N/A
    # 返回值: N/A
    # 其他: N/A
    #####################################################################
    function showRunTime() {
        echo ""  | tee  -a  $LOG_FILE
        echo "当前时间为："  | tee  -a  $LOG_FILE
        echo "==================================================="  | tee -a $LOG_FILE
        echo "$(date "+%Y-%m-%d %H:%M:%S")"                       | tee  -a  $LOG_FILE
    }
    
    #####################################################################
    # 函数名: checkReturn
    # 描述: 检查小文件合并的成功与否
    # 参数: N/A
    # 返回值: N/A
    # 其他: N/A
    #####################################################################
    function checkReturn() {
        if [ $1 == 0 ];then
            echo "Schema merge parquet file success!!!"
            echo "==================================================="  | tee -a ${LOG_FILE}
            echo "$(date "+%Y-%m-%d %H:%M:%S") Schema merge parquet file Success!"  | tee  -a  ${LOG_FILE}
        else
            echo "==================================================="  | tee -a ${LOG_FILE}
            echo "$(date "+%Y-%m-%d %H:%M:%S") Schema merge parquet file Failure!"     | tee  -a  ${LOG_FILE}
            exit 1
        fi
    }
    
    
    #---------------------------------------------------------------------#
    # 功能代码块 #
    #---------------------------------------------------------------------#
    #---------------------------------------------------------------------#
    # 参数验证 #
    #---------------------------------------------------------------------#
    ## 不符合情况的条件下，展示提示信息，并且退出
    showRunTime
    tmp=$(echo “$2” | grep ^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]$) 
    if [[ !(($# == 1 && "$1" == "mid_table") || ($# == 2 && "$1" == "person_table" && "$2" == "now") || \
    ($# == 2 && "$1" == "person_table" && "$2" == "before") || ($# == 2 && "$1" == "person_table" && "$tmp" != "")) ]];then
        showUsage
    fi
    
    
    #---------------------------------------------------------------------#
    # 执行业务 #
    #---------------------------------------------------------------------#
    
    if [[ $# == 1 && "$1" == "mid_table" ]];then
        echo "1, sh schema-merge-parquet-file.sh mid_table 合并临时表mid_table 的parquet 文件."
        merge_parquet
    fi
    
    tmpTableHdfsPath=$(grep job.smallfile.merge.person_table.hdfs_path  ${SPARKJOB_PROPERTIES} | awk -F "=" '{print $2}')
    if [[ $# == 2 && "$1" == "person_table" && "$2" == "now" ]];then
        echo "2, sh schema-merge-parquet-file.sh person_table now 合并person_table 表中当天的parquet文件."
        dateString=$(date "+%Y-%m-%d")
        merge_parquet
        checkReturn $?
    fi
    
    if [[ $# == 2 && "$1" == "person_table" && "$2" == "before" ]];then
        echo "3, sh schema-merge-parquet-file.sh person_table before 合并person_table"
        dateString=$(date -d "1 day ago" +"%Y-%m-%d")
        merge_parquet
        checkReturn $?
    fi
    
    if [[ $# == 2 && "$1" == "person_table" && "$tmp" != "" ]];then
        echo "4, sh schema-merge-parquet-file.sh person_table before 2018-01-03 合并person_table 中指定某天的文件."
        dateString=$2
        merge_parquet
        checkReturn $?
    fi 
    
    set +x

[https_unnunique.github.io_AADocs_skill-docs_linux]: https://unnunique.github.io/AADocs/skill-docs/linux/
[https_github.com_unnunique_unnunique.github.io_blob_master_AADocs_skill-docs_linux_index.md]: https://github.com/unnunique/unnunique.github.io/blob/master/AADocs/skill-docs/linux/index.md