Spark写入数据到hbase，本地idea运行

r囧r小猫 2022-12-08 04:18 166阅读 0赞

废话不多说，上代码：

package day02.com.toHbase
    
    import org.apache.hadoop.hbase.HBaseConfiguration
    import org.apache.hadoop.hbase.client.Put
    import org.apache.hadoop.hbase.io.ImmutableBytesWritable
    import org.apache.hadoop.hbase.mapred.TableOutputFormat
    import org.apache.hadoop.hbase.util.Bytes
    import org.apache.hadoop.mapred.JobConf
    import org.apache.spark.SparkConf
    import org.apache.spark.SparkContext
    import org.apache.spark.rdd.RDD
    import org.apache.spark.rdd.RDD.rddToPairRDDFunctions
    /**
      * 该类实现初始化数据，运用put方法写入到hbase中，成功！
      */
    object HbaseTestWrite01 {
      def main(args: Array[String]): Unit = {
    
        val sparkConf = new SparkConf().setAppName("HBaseTest").setMaster("local")
        val sc = new SparkContext(sparkConf)
    
        val conf = HBaseConfiguration.create()
        //设置zooKeeper集群地址，也可以通过将hbase-site.xml导入classpath，但是建议在程序里这样设置
        //conf.set("hbase.zookeeper.quorum","node01,node02,node03")
        //设置zookeeper连接端口，默认2181
        //conf.set("hbase.zookeeper.property.clientPort", "2181")
    
        val tablename = "test666"
        //初始化jobconf，TableOutputFormat必须是org.apache.hadoop.hbase.mapred包下的！
        val jobConf = new JobConf(conf)
        jobConf.setOutputFormat(classOf[TableOutputFormat])
        jobConf.set(TableOutputFormat.OUTPUT_TABLE, tablename)
    
        val indataRDD: RDD[String] = sc.makeRDD(Array("1,jack,15","2,Lily,16","3,mike,16"))
    
    
        val rdd = indataRDD
          .map(_.split(','))
          .map{arr=>{
          /*一个Put对象就是一行记录，在构造方法中指定主键
           * 所有插入的数据必须用org.apache.hadoop.hbase.util.Bytes.toBytes方法转换
           * Put.add方法接收三个参数：列族，列名，数据
           */
          val put = new Put(Bytes.toBytes(arr(0).toInt))
          put.add(Bytes.toBytes("cf"),Bytes.toBytes("id"),Bytes.toBytes(arr(0)))
          put.add(Bytes.toBytes("cf"),Bytes.toBytes("name"),Bytes.toBytes(arr(1)))
          put.add(Bytes.toBytes("cf"),Bytes.toBytes("age"),Bytes.toBytes(arr(2)))
          //转化成RDD[(ImmutableBytesWritable,Put)]类型才能调用saveAsHadoopDataset
          (new ImmutableBytesWritable, put)
        }}
    
        rdd.saveAsHadoopDataset(jobConf)
    
        sc.stop()
      }
    }

配置文件：把hbase-site.xml放到工程的resources目录下

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTExMTAzMDE_size_16_color_FFFFFF_t_70][]

hbase-site.xml文件如下：

<?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
            <property>
                    <name>hbase.rootdir</name>
                    <value>hdfs://node01:9000/hbase</value>  
            </property>
    
            <property>
                    <name>hbase.cluster.distributed</name>
                    <value>true</value>
            </property>
    
            <property>
                    <name>hbase.master.port</name>
                    
                    <value>60010</value>
            </property>
    
            <property>
                    <name>hbase.zookeeper.quorum</name>
                    <value>node01:2181,node02:2181,node03:2181</value>
            </property>
    
            <property>
                    <name>hbase.zookeeper.property.dataDir</name>
                    <value>/export/data/zkdata</value>
            </property>
    
    </configuration>

hbase中的表数据如下：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTExMTAzMDE_size_16_color_FFFFFF_t_70 1][]

运行，查看结果：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTExMTAzMDE_size_16_color_FFFFFF_t_70 2][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTExMTAzMDE_size_16_color_FFFFFF_t_70]: /images/20221123/03130832418b42bda3bd41141ba5998a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTExMTAzMDE_size_16_color_FFFFFF_t_70 1]: /images/20221123/b8e8594744d54fb59fee349daf204b55.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3UwMTExMTAzMDE_size_16_color_FFFFFF_t_70 2]: /images/20221123/1c74da1f9f7a4953943dd6cde63ea750.png