爬虫 lxml 模块

以你之姓@ 2021-12-20 08:13 315阅读 0赞

# Xpath  #

在 XML 文档中查找信息的语言, 同样适用于 HTML

## 辅助工具 ##

### Xpath Helper   ###

Chrome插件 　　快捷键 Ctrl + shift + x

![1450319-20190505192742052-765822507.png][]

### XML Quire ###

xpath 编辑工具

![1450319-20190505193057719-1209174308.png][]

## Xpath 表达式 ##

### //   查找所有节点 ###

//book

查找所有的book节点

### /   查找当前节点 ###

//book/title

查找所有book节点下的title子节

### @   获取属性筛选  ###

//book//title/@lang="en"

查找book节点下所有的title节点中,lang属性值为 "en"

### \[\]  当前节点的限制 ###

//bookstore/book[2]/title

查找bookstore下的第2个book节点下的title子节点

//title[@lang]

查找含有 lang 属性的 title节点

### |   匹配多路径 (或匹配) ###

xpath表达式1 | xpath表达式2 | ...

### contains()  匹配一个属性值中包含某些字符串的节点 ###

//div[contains(@id,'qiushi_tag_')]

### text()  匹配文本值 ###

//book/title/text()="羊驼"

查找 book 下的 title 下文本为"羊驼"节点

# lxml  #

## 安装 ##

pip install lxml

## 使用 ##

### 导入模块 ###

from lxml import etree

### 创建解析对象 ###

parse_html = etree.HTML(html)

### 调用 xpath 匹配 ###

r_list = parse_html.xpath('xpath 表达式')

ps: 返回结果以列表形式

### 示例 ###

**抓取指定贴吧所有图片**

*  获取贴吧主页URL,下一页,找URL规律
 *  获取1页中所有帖子URL地址
 *  对每个帖子链接发请求,获取图片URL
 *   对每个图片链接发请求,以wb方式写入本地

import requests
    from lxml import etree
    
    
    class BaiduSpider(object):
        def __init__(self):
            self.baseurl = 'http://tieba.baidu.com/f?'
            # 使用IE的User-Agent
            self.headers = {
                'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)'}
    
        # 获取帖子链接
        def get_turl(self, params):
            res = requests.get(
                self.baseurl,
                params=params,
                headers=self.headers
            )
            res.encoding = 'utf-8'
            html = res.text
            # 提取帖子链接
            parse_html = etree.HTML(html)
            t_list = parse_html.xpath('//div[@class="t_con cleafix"]/div/div/div/a/@href')
            print(t_list)
            # t_list : ['/p/23232/','/p/923423']
            for t in t_list:
                url = 'http://tieba.baidu.com' + t
                # 提取图片链接,对图片链接发请求保存到本地
                self.get_imgurl(url)
    
        # 获取图片链接
        def get_imgurl(self, url):
            res = requests.get(url, headers=self.headers)
            res.encoding = 'utf-8'
            html = res.text
            # 提取图片链接列表
            parse_html = etree.HTML(html)
            # 图片链接列表
            img_list = parse_html.xpath(
                '//div[@class="d_post_content j_d_post_content  clearfix"]/img[@class="BDE_Image"]/@src | //div[@class="video_src_wrapper"]/embed/@data-video')
            # 视频链接列表
            # img_list = parse_html.xpath('//div[@class="video_src_wrapper"]/embed/@data-video')
            print(img_list)
            for img in img_list:
                self.write_img(img)
    
        # 把图片保存到本地
        def write_img(self, img):
            res = requests.get(img, headers=self.headers)
            res.encoding = 'utf-8'
            html = res.content
            # 保存到本地
            filename = img[-10:]
            with open(filename, 'wb') as f:
                f.write(html)
                print('%s下载成功' % filename)
    
        # 主函数
        def main(self):
            name = input('贴吧名:')
            begin = int(input('起始页:'))
            end = int(input('终止页:'))
            for page in range(begin, end + 1):
                pn = (page - 1) * 50
                # 定义查询参数
                params = {
                    'kw': name,
                    'pn': str(pn)
                }
                self.get_turl(params)
    
    
    if __name__ == '__main__':
        spider = BaiduSpider()
        spider.main()

**猫眼电影信息爬取**

from urllib import request
    import time
    import csv
    from lxml import etree
    
    
    class MaoyanSpider(object):
        def __init__(self):
            self.baseurl = 'https://maoyan.com/board/4?offset='
            self.headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'}
            # 爬取页数计数
            self.page = 1
    
        # 获取页面
        def get_page(self, url):
            req = request.Request(url, headers=self.headers)
            res = request.urlopen(req)
            html = res.read().decode('utf-8')
            # 直接调用解析函数
            self.parse_page(html)
    
        # 解析页面
        def parse_page(self, html):
            parse_html = etree.HTML(html)
            # 基准xpath,匹配每个电影信息节点对象列表
            dd_list = parse_html.xpath('//dl[@class="board-wrapper"]/dd')
            # dd_list : [<element dd at xxx>,<...>]
            for dd in dd_list:
                name = dd.xpath('./a/@title')[0].strip()
                star = dd.xpath('.//p[@class="star"]/text()')[0].strip()
                time = dd.xpath('.//p[@class="releasetime"]/text()')[0].strip()
                print([name, star, time])
    
        # 保存数据(存到csv文件)
        def write_page(self, r_list):
            # r_list : [(),(),()]
            with open('猫眼.csv', 'a') as f:
                writer = csv.writer(f)
                for rt in r_list:
                    film = [
                        rt[0].strip(),
                        rt[1].strip(),
                        rt[2].strip()
                    ]
                    writer.writerow(film)
    
        # 主函数
        def main(self):
            # 用range函数可获取某些查询参数的值
            for offset in range(0, 41, 10):
                url = self.baseurl + str(offset)
                self.get_page(url)
                print('第%d页爬取成功' % self.page)
                self.page += 1
                time.sleep(1)
    
    
    if __name__ == '__main__':
        spider = MaoyanSpider()
        spider.main()

转载于:https://www.cnblogs.com/shijieli/p/10816007.html

[1450319-20190505192742052-765822507.png]: /images/20211220/3b5c4ef6b5464425816a087d9e682ffc.png
[1450319-20190505193057719-1209174308.png]: /images/20211220/b95f5721512041dd95e5a5454fbada16.png