Spark通过三种方式创建DataFrame

灰太狼 2024-02-18 10:28 32阅读 0赞

### 通过toDF方法创建DataFrame ###

**通过toDF的方法创建**

*  集合rdd中元素类型是样例类的时候，转成DataFrame之后列名默认是属性名
 *  集合rdd中元素类型是元组的时候，转成DataFrame之后列名默认就是\_N
 *  集合rdd中元素类型是元组/样例类的时候，转成DataFrame(toDF(“ID”,“NAME”,“SEX”,“AGE6”))可以自定义列名

import org.apache.spark.sql.{
        DataFrame, SparkSession}
    import org.junit.Test
    
    
    case class Person(id:Int,name:String,sex:String,age:Int)
    class TestScala {
        
    
      val spark = SparkSession
        .builder()
        .appName("test")
        .master("local[4]")
        .getOrCreate()
        import spark.implicits._
    
      /**
       * 通过toDF的方法创建
       *    集合rdd中元素类型是样例类的时候，转成DataFrame之后列名默认是属性名
       *    集合rdd中元素类型是元组的时候，转成DataFrame之后列名默认就是_N
       */
      @Test
      def createDataFrameByToDF():Unit={
        
        //TODO 样例类是属性名
        val list = List(Person(1,"zhangsan","man",10),Person(2,"zhang2","woman",66),Person(3,"zhang3","man",70),Person(4,"zhang4","man",22))
        //需要隐士转换
        val df:DataFrame = list.toDF()
        df.show()
        //TODO 元祖是_N
        val list2 = List((1,"zhangsan","man",10),(1,"zhang2","woman",66),(1,"zhang3","man",70),(1,"zhang4","man",22))
        //需要隐士转换
        val df1:DataFrame = list2.toDF()
        df1.show()
        //TODO 自定义属性名
        val list3 = List((1,"zhangsan","man",10),(1,"zhang2","woman",66),(1,"zhang3","man",70),(1,"zhang4","man",22))
        //需要隐士转换
        val df2:DataFrame = list3.toDF("ID","NAME","SEX","AGE6")
        df2.show()
      }
      
      
    }

结果  
![在这里插入图片描述][426823039fa8438db46da6d478e1b939.png]

### 通过读取文件创建DataFrame ###

json数据

{
        "age":20,"name":"qiaofeng"}
    {
        "age":19,"name":"xuzhu"}
    {
        "age":18,"name":"duanyu"}

/**
       * 通过读取文件创建
       */
      @Test
      def createDataFrame():Unit={
        
        val df = spark.read.json("src/main/resources/user.json")
        df.show()
      }

### 通过createDataFrame方法创建DF ###

@Test
      def createDataFrameByMethod():Unit={
        
        val fields = Array(StructField("id",IntegerType),StructField("name",StringType),StructField("sex",StringType),StructField("age",IntegerType))
        val schema = StructType(fields)
        val rdd = spark.sparkContext.parallelize(List(Row(1, "zhangsan", "man", 10), Row(2, "zhang2", "woman", 66), Row(3, "zhang3", "man", 70), Row(4, "zhang4", "man", 22)))
        val df = spark.createDataFrame(rdd, schema)
        df.show()
      }

![在这里插入图片描述][a5c38e09e9cf421a840407586025ac52.png]

[426823039fa8438db46da6d478e1b939.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/18/2834416dbf934bdfa523675f9defda24.png
[a5c38e09e9cf421a840407586025ac52.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/18/03962f49eb1d4e6aa3e11bd2d1c1a911.png