Apache Hive(一) 安装

本是古典何须时尚 2023-03-13 03:37 49阅读 0赞

#### 环境 ####

*  centos7
 *  hadoop 2.7.7
 *  hive 3.1.2
 *  mysql 5.7.22
 *  安装目录 /opt/apache-hive-3.1.2-bin/

#### 下载bin ####

wget https://mirror.bit.edu.cn/apache/hive/hive-3.1.2/apache-hive-3.1.2-bin.tar.gz

#### 解压 ####

tar -zxvf apache-hive-3.1.2-bin.tar.gz
    
    cd apache-hive-3.1.2-bin

#### 配置conf ####

cd conf
    mv hive-env.sh.template hive-env.sh
    vi hive-env.sh

export HADOOP_HOME=/opt/hadoop-2.7.7
    export HIVE_CONF_DIR=/opt/apache-hive-3.1.2-bin/conf

#### 配置元数据库(Mysql) ####

创建数据库

CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
    
    GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'Cdh123456.';
    FLUSH PRIVILEGES;

安装驱动

cd /opt/apache-hive-3.1.2-bin/lib
    
    wget http://maven.aliyun.com/nexus/content/groups/public/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar

配置

cd /opt/apache-hive-3.1.2-bin/lib
    vi hive-site.xml

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
    	  <name>javax.jdo.option.ConnectionURL</name>
    	  <value>jdbc:mysql://192.168.1.101:3306/metastore?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF-8&useSSL=false</value>
    	  <description>JDBC connect string for a JDBC metastore</description>
    	</property>
    
    	<property>
    	  <name>javax.jdo.option.ConnectionDriverName</name>
    	  <value>com.mysql.jdbc.Driver</value>
    	  <description>Driver class name for a JDBC metastore</description>
    	</property>
    
    	<property>
    	  <name>javax.jdo.option.ConnectionUserName</name>
    	  <value>hive</value>
    	  <description>username to use against metastore database</description>
    	</property>
    
    	<property>
    	  <name>javax.jdo.option.ConnectionPassword</name>
    	  <value>Cdh123456.</value>
    	  <description>password to use against metastore database</description>
    	</property>
    	<property>
    		<name>datanucleus.readOnlyDatastore</name>
    		<value>false</value>
    	</property>
    	<property>
    		<name>datanucleus.fixedDatastore</name>
    		<value>false</value>
    	</property>
    	<property>
    		<name>datanucleus.autoCreateSchema</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>datanucleus.schema.autoCreateAll</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>datanucleus.autoCreateTables</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>datanucleus.autoCreateColumns</name>
    		<value>true</value>
    	</property>
    	<property>
    		<name>hive.metastore.local</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>hive.cli.print.header</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>hive.cli.print.current.db</name>
    		<value>true</value>
    	</property>
    </configuration>

初始化数据库

bin/schematool -dbType mysql -initSchema

![image-20200505151822963][]

#### 准备 ####

> 保证 Hive启动成功

hadoop fs -mkdir /tmp
    hadoop fs -mkdir -p /user/hive/warehouse
    
    hadoop fs -chmod g+w /tmp
    hadoop fs -chmod g+w /user/hive/warehouse

#### 启动 ####

nohup bin/hiveserver2 1>/dev/null 2>&1 &
    
    bin/beeline -u jdbc:hive2://localhost:10000 -n root

#### 测试 CRUD ####

hive (default)> show databases;
    OK
    database_name
    default
    Time taken: 0.448 seconds, Fetched: 1 row(s)
    ------------------------------------------------------
    hive (default)> use default;
    OK
    Time taken: 0.03 seconds
    ------------------------------------------------------
    hive (default)> show tables;
    OK
    tab_name
    Time taken: 0.029 seconds
    ------------------------------------------------------
    hive (default)> create table student(id int, name string);
    OK
    Time taken: 0.808 seconds
    ------------------------------------------------------
    hive (default)> show tables;
    OK
    tab_name
    student
    Time taken: 0.035 seconds, Fetched: 1 row(s)
    ------------------------------------------------------
    hive (default)> desc student;
    OK
    col_name	data_type	comment
    id                  	int                 	                    
    name                	string              	                    
    Time taken: 0.082 seconds, Fetched: 2 row(s)
    ------------------------------------------------------
    hive (default)> insert into student values(1000,"ss");
    Query ID = root_20200505152527_6d6beeff-a7f0-4b5e-ab63-5666fcf1787c
    Total jobs = 3
    Launching Job 1 out of 3
    Number of reduce tasks determined at compile time: 1
    In order to change the average load for a reducer (in bytes):
      set hive.exec.reducers.bytes.per.reducer=<number>
    In order to limit the maximum number of reducers:
      set hive.exec.reducers.max=<number>
    In order to set a constant number of reducers:
      set mapreduce.job.reduces=<number>
    Starting Job = job_1588650622830_0001, Tracking URL = http://flinkhadoop:8088/proxy/application_1588650622830_0001/
    Kill Command = /opt/hadoop-2.7.7/bin/mapred job  -kill job_1588650622830_0001
    Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 1
    2020-05-05 15:25:42,696 Stage-1 map = 0%,  reduce = 0%
    2020-05-05 15:25:48,953 Stage-1 map = 100%,  reduce = 0%, Cumulative CPU 3.04 sec
    2020-05-05 15:25:54,135 Stage-1 map = 100%,  reduce = 100%, Cumulative CPU 5.3 sec
    MapReduce Total cumulative CPU time: 5 seconds 300 msec
    Ended Job = job_1588650622830_0001
    Stage-4 is selected by condition resolver.
    Stage-3 is filtered out by condition resolver.
    Stage-5 is filtered out by condition resolver.
    Moving data to directory hdfs://flinkhadoop:9000/user/hive/warehouse/student/.hive-staging_hive_2020-05-05_15-25-27_596_5270541623851469596-1/-ext-10000
    Loading data to table default.student
    MapReduce Jobs Launched: 
    Stage-Stage-1: Map: 1  Reduce: 1   Cumulative CPU: 5.3 sec   HDFS Read: 15286 HDFS Write: 248 SUCCESS
    Total MapReduce CPU Time Spent: 5 seconds 300 msec
    OK
    col1	col2
    Time taken: 28.45 seconds
    ------------------------------------------------------
    hive (default)> select * from student;
    OK
    student.id	student.name
    1000	ss
    Time taken: 0.174 seconds, Fetched: 1 row(s)

> CDH 安装比较简单，请自由搜索

[image-20200505151822963]: /images/20230312/8acfe914a6eb411ba33c30de5455bb8f.png