《利用Python 进行数据分析》第二章：引言中的分析（含pdf和数据集下载链接）

骑猪看日落 2022-12-06 05:48 159阅读 0赞

对《利用Python 进行数据分析》（Wes Mckinney著）一书中的第二章中的第二部分——1880-2010全美婴儿姓名分析进行代码实验。原书中采用的是Python2.7，而我采用的Python3.7利用Pycharm调试的，因此对源代码进行了一定的修改，并且增加了部分画图代码，每步打印结果和画图与原文校验对照一致（部分打印结果已经关闭，需要查看结果则可以打开），全手工敲写，供参考。  
       （1）《利用Python 进行数据分析》pdf下载链接：  
       链接：[https://pan.baidu.com/s/1NiFQb5ZIR0UpKy-K9cGeTQ][https_pan.baidu.com_s_1NiFQb5ZIR0UpKy-K9cGeTQ]  
       提取码：vn9m  
       （2）《利用Python 进行数据分析》数据集链接如下：  
       链接：[https://pan.baidu.com/s/1jfWBk8WMHDQh6nGoSBAsmw][https_pan.baidu.com_s_1jfWBk8WMHDQh6nGoSBAsmw]  
       提取码：bgcq

第二章中的第二部分的代码还原如下：

#-*- coding:utf-8 -*-
    import pandas as pd
    import matplotlib.pyplot as plt
    import numpy as np
    
    ### 1880-2010全美婴儿姓名 ###
    names880 = pd.read_csv("PythonForDataAnalysis-master/ch02/names/yob1880.txt",
                           names = ["name", "sex", "births"])   # 路径修改为自己的路径
    # print(names880)
    
    
    # 按性别计算每个类别总数
    print(names880.groupby('sex').births.sum())
    years = range(1880,2011)
    pieces = []
    columns = ['name','sex','births']
    for year in years:
        path = "PythonForDataAnalysis-master/ch02/names/yob%d.txt"%(year) # 路径修改为自己的路径
        frame = pd.read_csv(path, names = columns)
        frame['year'] = year
        pieces.append(frame)
    
    # 所有年份合并为一个大表
    names = pd.concat(pieces, ignore_index= True)
    # print(names)
    
    # 数据透视表, 索引为年份，列为性别
    total_births = names.pivot_table('births',index = 'year', columns = 'sex', aggfunc = sum)
    print(total_births.tail())
    
    # total_births.plot(title='total births by sex and year')
    plt.plot(total_births)
    plt.title('total births by sex and year')
    plt.legend(["F","M"])
    plt.show()
    
    
    def add_prop(group):
        births = group.births.astype(float)
        group['prop']=births/ births.sum()
        return group
    
    names = names.groupby(['year','sex']).apply(add_prop)
    # print(names)
    
    # 判断比例总和是否为1
    print(np.allclose(names.groupby(['year','sex']).prop.sum(),1))
    
    # 取子集，sex/year组合的前1000个名字
    def get_top1000(group):
        return group.sort_values(by = 'births', ascending= False)[:1000]
    
    grouped = names.groupby(['year','sex'])
    top1000 = grouped.apply(get_top1000)
    print(top1000)
    
    pieces = []
    for year,group in names.groupby(['year', 'sex']):
        pieces.append(group.sort_values(by = 'births', ascending = False)[:1000])  # Python3中需要用sort_vaules()函数
    top1000 = pd.concat(pieces, ignore_index= True)
    print(top1000)
    
    
    # 分析命名趋势
    boys = top1000[top1000.sex == 'M']
    girls = top1000[top1000.sex == 'F']
    
    total_births = top1000.pivot_table('births', index = 'year', columns = 'name',aggfunc = sum)
    print(total_births)
    
    fig,axes = plt.subplots(4,1, figsize=(10,6))
    names_plot = ["John","Harry","Mary","Marilyn"]
    #设置总标题
    fig.suptitle("Number of births per year")   
    # 绘制多个子图
    for i in range(len(names_plot)):
        axes[i].plot(total_births[names_plot[i]], label = names_plot[i])
        axes[i].legend() # 添加图例
    # plt.plot(total_births['Mary'])
    plt.show()
    
    # 评估命名多样性增长
    table = top1000.pivot_table('prop', index = 'year', columns = 'sex', aggfunc = sum)
    # table.plot(title = 'Sum of table1000.prop by year and sex',yticks = np.linspace(0,1.2,13),xticks = range(1880,2020,10))
    plt.figure('Sum of table1000.prop by year and sex')
    plt.plot(table)
    plt.title('Sum of table1000.prop by year and sex')
    plt.yticks = np.linspace(0,1.2,13)
    plt.xticks = range(1880,2020,10)
    plt.show()
    
    # 只考虑2010年男孩的名字
    df = boys[boys.year == 2010]
    prop_cumsum = df.sort_values(by = 'prop', ascending = False).prop.cumsum()
    print(prop_cumsum[:10])
    print(prop_cumsum.searchsorted(0.5))
    df = boys[boys.year == 1900]
    in1900 = df.sort_values(by = 'prop', ascending = False).prop.cumsum()
    print(in1900.searchsorted(0.5) + 1)
    
    
    # 统计多样性
    def get_quantile_count(group,q=0.5):
        group = group.sort_values(by='prop',ascending=False)
        return group.prop.cumsum().searchsorted(q) + 1
    
    diversity = top1000.groupby(['year','sex']).apply(get_quantile_count)
    diversity = diversity.unstack('sex')
    print(diversity.head())
    
    plt.figure('Number of population names in top 50%')
    plt.plot(diversity)
    plt.title("Number of population names in top 50%")
    plt.legend(diversity.columns)
    plt.show()
    
    
    get_last_letter = lambda x:x[-1]
    last_letters = names.name.map(get_last_letter)
    last_letters.name = 'last_letter'
    table = names.pivot_table('births', last_letters, ['sex', 'year'],aggfunc=sum)
    
    # 只看其中三年的数据
    subtable = table.reindex(columns = [1910, 1960, 2010], level = 'year')
    print(subtable.head())
    print(subtable.sum())
    
    #各性别的条形图
    # plt.figure('sex bar')
    letter_prop = subtable/subtable.sum().astype(float)
    fig,axes = plt.subplots(2,1, figsize=(10,6))
    letter_prop['M'].plot(kind = 'bar', rot = 0, ax = axes[0], title = 'Male')
    letter_prop['F'].plot(kind = 'bar', rot = 0, ax = axes[1], title = 'Female', legend = False)
    plt.show()
    
    # 男孩中某个字母统计
    letter_prop = table/table.sum().astype(float)
    dny_ts = letter_prop.loc[['d','n','y'], 'M'].T # python3 中ix()不可用
    print(dny_ts.head())
    plt.plot(dny_ts)
    print(dny_ts.columns) # 注意不要写成dny_ts.columns()
    plt.legend(dny_ts.columns)
    plt.show()
    
    # 变成女孩的男孩名称
    all_names = top1000.name.unique()
    mask= np.array(['lesl' in x.lower() for x in all_names])
    lesl_like = all_names[mask]
    
    
    filtered = top1000[top1000.name.isin(lesl_like)]
    print(top1000.name)
    # 分组查看名字类似lesl的名字
    print(filtered.groupby('name').births.sum())
    
    #按性别和年度进行聚合
    table = filtered.pivot_table('births', index = 'year', columns = 'sex', aggfunc = 'sum')
    print(table.tail())
    print(table.sum(1))  # 按行求和得到，每行总数， 数字+NaN =数字
    table = table.div(table.sum(1), axis = 0)  # table表中每行数/table.sum(1)
    print(table.tail())
    
    # 绘制分性别的年度曲线图
    plt.figure()
    plt.plot(table['M'], label = "M")
    plt.plot(table['F'], label = "F")
    plt.legend()
    plt.show()

[https_pan.baidu.com_s_1NiFQb5ZIR0UpKy-K9cGeTQ]: https://pan.baidu.com/s/1NiFQb5ZIR0UpKy-K9cGeTQ
[https_pan.baidu.com_s_1jfWBk8WMHDQh6nGoSBAsmw]: https://pan.baidu.com/s/1jfWBk8WMHDQh6nGoSBAsmw