（8）Python爬虫——爬取豆瓣影评数据

一时失言乱红尘 2022-07-29 08:48 205阅读 0赞

利用python爬取豆瓣最受欢迎的影评50条的相关信息，包括标题,作者,影片名,影片详情链接,推荐级,回应数,影评链接,影评,有用数这9项内容，然后将爬取的信息写入Excel表中。具体代码如下：

#!/usr/bin/python
    # -*- coding: utf-8 -*-
    import sys
    
    reload(sys)
    sys.setdefaultencoding('utf8')
    from bs4 import BeautifulSoup
    import re
    import urllib2
    import xlwt
    
    
    # 得到页面全部内容
    def askURL(url):
        request = urllib2.Request(url)  # 发送请求
        try:
            response = urllib2.urlopen(request)  # 取得响应
            html = response.read()  # 获取网页内容
            # print html
        except urllib2.URLError, e:
            if hasattr(e, "code"):
                print e.code
            if hasattr(e, "reason"):
                print e.reason
        return html
    
    
    # 获取相关内容
    def getData(baseurl):
        # 找到评论标题
        pattern_title = re.compile(r'<a href=".*/review/\d+/">(.+)</a>')
        # 找到评论全文链接
        pattern_link = re.compile(r'<a href="(.*/review/\d+/)">.+</a>')
        # 找到作者
        pattern_author = re.compile(r'<a.+property="v:reviewer">(.+)</a>')
        # 找到评论的影片和影评详情链接
        pattern_subject_link = re.compile(r'<a class="subject-img" href="(.+subject.+)"> <img.+title="(.+)".+>')
        # 找到推荐等级
        pattern_star = re.compile(r'<span.+property="v:rating" title="(.+)"></span>')
        # 找到回应数
        pattern_response = re.compile(r'<a class="reply" href=".*/review/\d+/#comments">(\d+)回应</a>')
        # 找到有用数
        pattern_use = re.compile(r'<span id="r-useful_count-\d+">\s*(\d+)\s*</span>')
        remove = re.compile(r'<.+?>')  # 去除标签
        datalist = []
        for i in range(0, 5):
            url = baseurl + str(i * 10)  # 更新url
            # print url
            html = askURL(url)
            # print html
            soup = BeautifulSoup(html, "html.parser")
            # 找到每一个影评项
            for item in soup.find_all('div', class_='main review-item'):
                data = []
                item = str(item)  # 转换成字符串
                # print item
                title = re.findall(pattern_title, item)[0]
                # print title
                reviewlink = re.findall(pattern_link, item)[0]
                # print reviewlink
                data.append(title)  # 添加标题
                author = re.findall(pattern_author, item)[0]
                # print author
                data.append(author)  # 添加作者
                list_subject_link = re.findall(pattern_subject_link, item)[0]
                moviename = list_subject_link[1]
                # print moviename
                movielink = list_subject_link[0]
                # print movielink
                data.append(moviename)  # 添加片名
                data.append(movielink)  # 添加影片链接
                star = re.findall(pattern_star, item)
                if len(star) != 0:
                    star = star[0]
                else:  # 可能没有star
                    star = ''
                # print star
                data.append(star)  # 添加推荐等级
                response = re.findall(pattern_response, item)[0]
                # print response
                data.append(response)  # 添加回应数
                data.append(reviewlink)  # 添加评论正文链接
                use = re.findall(pattern_use, item)[0]
                # print use
                content = askURL(reviewlink)
                content = BeautifulSoup(content, "html.parser")
                desc = content.find_all('div', id='link-report')[0]
                desc = re.sub(remove, '', str(desc))  # 去掉标签
                # print desc
                data.append(desc)  # 添加评论正文
                data.append(use)  # 添加有用数
                # print data
                datalist.append(data)
        return datalist
    
    
    # 将相关数据写入excel中
    def saveData(datalist, savepath):
        book = xlwt.Workbook(encoding='utf-8', style_compression=0)
        sheet = book.add_sheet('豆瓣最受欢迎影评', cell_overwrite_ok=True)
        col = ('标题', '作者', '影片名', '影片详情链接', '推荐级', '回应数', '影评链接', '影评', '有用数')
        for i in range(0, 9):
            sheet.write(0, i, col[i])  # 列名
        for i in range(0, 50):  # 总共50条影评
            data = datalist[i]
            for j in range(0, 9):
                sheet.write(i + 1, j, data[j])  # 数据
        book.save(savepath)  # 保存
    
    
    def main():
        baseurl = 'http://movie.douban.com/review/best/?start='
        datalist = getData(baseurl)
        savapath = u'豆瓣最受欢迎影评.xlsx'
        saveData(datalist, savapath)
        print 'done'
    
    
    main()

Excel部分内容如下：![这里写图片描述][20160318192941156]

国家代表的影评正文如下：  
![这里写图片描述][20160318193001293]

[20160318192941156]: /images/20220728/0cf41f7802e8491fa8deae534fdb78ce.png
[20160318193001293]: /images/20220728/71f9612e0a584586875941f6562a1199.png