python爬取所有的城市名称

怼烎@ 2023-10-05 13:29 35阅读 0赞

爬取所有的城市名称  
import requests  
from lxml import etree

if **name** == ‘**main**’:  
url=‘https://www.aqistudy.cn/historydata/’  
header = \{  
‘user\_Agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36’  
\}  
page\_text=requests.get(url=url,headers=header).text  
\#数据解析  
tree=etree.HTML(page\_text)

# title_list=tree.xpath("//ul[@class='unstyled']/div[1]")
    #print(title_list)
    
    fp=open('./cityname.txt','w',encoding='utf-8')
    # for i in range(1,23):
    #     li_list=tree.xpath("//ul[@class='unstyled'][i]/div[2]/li")
    #     for title in title_list:
    #         city_title=title.xpath("./b/text()")[0]
    #         city_title=str(city_title)+'\n'
    #         fp.write(city_title)
    #         for i in range(1, 23):
    #             # li_list=tree.xpath("//div[@class='bottom']/ul[i]/div[2]/li")
    #             li_list = tree.xpath("//div[@class='bottom']/ul")[i]
    #             # print(li_list)
    #             for li in li_list:
    #                 city_name=li.xpath("./div[2]/li/a/text()")[0]
    #                 #print(city_name)
    #                 city_name=str(city_name)+'\t'
    #                 fp.write(city_name)
    #                 print(city_name,'爬取成功')
    #     fp.write('\n')
    a_list=tree.xpath("//div[@class='bottom']/ul/li/a | //div[@class='bottom']/ul/div[2]/li/a")
    for a in a_list:
        city=a.xpath('./text()')[0]
        city=str(city)+'\t'
        fp.write(city)
        print(city,"爬取成功")

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjQyOTI5MA_size_16_color_FFFFFF_t_70_pic_center]  
因为在网页中热门城市和所有城市不是在一个class标签中，而且标签的层级也不相同，所以可以用两种方法  
方法一就是一个一个爬取，就需要分别解析两个不同层级的标签，即是代码中被注释的部分  
方法二就是把两个不同层级的标签写在一起，就是代码中未被注释的部分  
tree.xpath("//div\[@class=‘bottom’\]/ul/li/a | //div\[@class=‘bottom’\]/ul/div\[2\]/li/a")

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjQyOTI5MA_size_16_color_FFFFFF_t_70_pic_center]: https://img-blog.csdnimg.cn/20201111200351377.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjQyOTI5MA==,size_16,color_FFFFFF,t_70#pic_center