爬取网易云音乐所有歌手名字和链接

短命女 2022-02-05 13:27 412阅读 0赞

采用面向对象的方式代码如下：

import requests
    from lxml import etree
    from urllib import request
    import time
    
    class WangYiYun:
        def __init__(self,base_url):
            # 初始化tree
            self.html = self.request_url(base_url)
            self.parse_html()
    
        def request_url(self,base_url):
            # 请求访问 生成树结构
            headers = {
                'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
            }
            self.html = requests.get(url = base_url, headers = headers).content.decode('utf-8')
            self.tree = etree.HTML(self.html)
            return self.tree
    
        def parse_html(self):
            # 解析各类歌手url
            group_list = self.html.xpath('//div[@class = "blk"]')
    
            for group in group_list:
                # 连接分类
                href_list = group.xpath('.//a/@href')
                # 歌手分类
                group_name_list = group.xpath('.//a/text()')
                # print(group_name_list)
                for href, group_name in zip(href_list, group_name_list):
                    full_href = request.urljoin('https://music.163.com', href)
                    # print(full_href,group_name)
                    html1 = requests.get(full_href).content.decode('utf-8')
                    tree1 = etree.HTML(html1)
                    # print(tree1)
                    # 歌手分类对应的歌手列表页链接
                    url_list = tree1.xpath('//ul[@class="n-ltlst f-cb"]/li[position()>1]/a/@href')
                    # print(url_list)
                    # 歌手列表页链接遍历
                    for url in url_list:
                        full_url = request.urljoin('https://music.163.com', url)
                        single_letter = requests.get(full_url).content.decode('utf-8')
                        #
                        single_letter_tree = etree.HTML(single_letter)
                        # 歌手列表页获取歌手信息
                        li_list = single_letter_tree.xpath('//ul[@class="m-cvrlst m-cvrlst-5 f-cb"]/li')
                        for li in li_list:
                            # 从歌手列表页中遍历获取名字列表及对应的链接列表
                            name_list = li.xpath('//a[@class="nm nm-icn f-thide s-fc0"]/text()')
                            link_list = li.xpath('//a[@class="nm nm-icn f-thide s-fc0"]/@href')
                            for name,link in zip(name_list,link_list):
                                full_link = 'https://music.163.com'+link
                                print(name,full_link)
                                time.sleep(0.5)
    
    if __name__ == '__main__':
        base_url = 'https://music.163.com/discover/artist'
        WangYiYun(base_url)

运行结果：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDIzOTU0MQ_size_16_color_FFFFFF_t_70][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDIzOTU0MQ_size_16_color_FFFFFF_t_70]: /images/20220205/2fa46d48e48a43a082c1aca2812cd540.png