爬虫第一篇

川长思鸟来 2022-03-07 15:57 145阅读 0赞

#### 爬虫 ####

1.环境：安装Anconda，是一个科学计算的集成开发环境（集成了好多库，ipython等）

2.chrome浏览器插件：XPath Helper,Proxy-SwitchyOmega-Chromium-2.5.15

3.爬虫的请求模块：

1、版本  
1、python2 ：urllib、urllib2  
2、python3 ：urllib.request

2.urllib.request 用法

#1.获取请求对象，req
    req = urllib.request.Request(url, data=None, headers={ })
    参数：url  请求路径
    	 data   get请求时data=None,post请求时data数据需要用urllib.parse.urlencode(data).encode(‘utf-8’)模块解析
    	 headers  一般需要设置User-Agent
    
    #2. 发送请求
    res = urllib.request.urlopen(req)
    
    #3.获取相应内容
    html = res.read().decode("utf-8")
    
    #响应对象(res)的方法
    1. res.read() : 读取服务器响应的内容
    2. res.getcode() : 返回HTTP的响应码   
    3. res.geturl(): 返回实际数据的URL地址

3.urllib.parse模块 : URL编码模块

#参数解析
    urllib.parse.urlencode(字典)   
    
    urllib.parse.quote(字符串)

4.urllib.erroe模块

错误类型：URLError HTTPError

URLError一般是因为没有网络连接或者server不存在。这种情况下，会产生一个reason属性，是一个tuple，包含了错误码和错误文本

import urllib.request
    import urllib.error
    
    req=urllib.request.Request('http://www.pretend_server.com')
    try: 
        urllib.request.urlopen(req)
    except urllib.error.URLError as e:
        print(e.reason)

5.请求方式及实例

1.GET

特点：查询参数在URL地址中显示

#案例
    import urllib.request
    import urllib.parse
    
    class BaiDuSpider:
        def __init__(self):
            self.baseurl = "http://tieba.baidu.com/f?"
            self.headers = { "User-Agent":"Mozilla/5.0 "}
        
        # 获取页面
        def get_page(self,url):
            req = urllib.request.Request(url,headers=self.headers)
            res = urllib.request.urlopen(req)
            html = res.read().decode("utf-8")
            return html
            
        # 解析页面
        def parse_page(self):
            pass
        
        # 保存数据
        def write_page(self,filename,html):
            with open(filename,"w",encoding="utf-8") as f:
                f.write(html)
        
        # 主函数
        def workOn(self):
            name = input("请输入贴吧名称:")
            begin = int(input("请输入起始页:"))
            end = int(input("请输入终止页:"))
            # 拼接贴吧主页URL地址
            kw = urllib.parse.urlencode({ "kw":name})
            for page in range(begin,end+1):
                # 拼接第page页完整URL地址
                pn = (page - 1) * 50
                url = self.baseurl + kw + "&pn=" + str(pn)
                html = self.getPage(url)
                filename = "第" + str(page) + "页.html"
                self.writePage(filename,html)       
                print("第 %d 页爬取成功" % page)
        
    if __name__ == "__main__":
        spider = BaiduSpider()
        spider.workOn()

2.POST

1.特点:URL地址无变化,数据是在Form表单中

2.data:表单数据要以bytes类型提交,不能是string，使用urllib.parse.urlencode(data).encode(‘utf-8’)

#案例
    import urllib.request
    import urllib.parse
    import json
    
    # 接收用户输入
    key = input("请输入要翻译的内容:")
    # 把Form Data定义成1个大字典
    data = { 
            "i":key,
            "from":"AUTO",
            "to":"AUTO",
            "smartresult":"dict",
            "client":"fanyideskweb",
            "salt":"15458120942800",
            "sign":"108feafc7c01c7461a41034463a8df9b",
            "ts":"1545812094280",
            "bv":"363eb5a1de8cfbadd0cd78bd6bd43bee",
            "doctype":"json",
            "version":"2.1",
            "keyfrom":"fanyi.web",
            "action":"FY_BY_REALTIME",
            "typoResult":"false"
        }
    # 把data转为bytes数据类型
    data = urllib.parse.urlencode(data).encode("utf-8")
    # 发请求,获响应,获取内容
    # 此处的URL地址为F12抓到的POST的地址,去掉translate_o中的 "_o"
    url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule"
    headers = { "User-Agent":"Mozilla/5.0"}
    req = urllib.request.Request(url,data=data,headers=headers)
    res = urllib.request.urlopen(req)
    html = res.read().decode("utf-8")
    
    # 把json格式的字符串转为python中字典
    rDict = json.loads(html)
    result = rDict["translateResult"][0][0]["tgt"]
    print(result)

6.正则表达式匹配响应回来的内容

import re
    
    # 1. 创建编译对象
    regex = re.compile(r'正则表达式')
    result = regex.findall(html)

#案例
    import urllib.request
    import re
    
    class NeiHanSpider:
        def __init__(self):
            self.baseurl = "https://www.neihan8.com/njjzw/"
            self.headers = { "User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)"}
            self.page = 2
        
        # 获取页面
        def get_page(self,url):
            req = urllib.request.Request(url,
                      headers=self.headers)
            res = urllib.request.urlopen(req)
            html = res.read().decode("utf-8") 
            self.parsePage(html)
        
        # 解析页面
        def parse_page(self,html):
            regex = re.compile('<div class="text-.*?title="(.*?)">.*?class="desc">(.*?)</div>',re.S)
            rList = regex.findall(html)
    # print(rList)
            # [("动物贴墙","海豹"),(),(),()]
            self.writePage(rList)
        
        # 保存数据
        def write_page(self,rList):
            for rTuple in rList:
                with open("内涵.txt","a") as f:
                    f.write(rTuple[0].strip()+"\n")
                    f.write(rTuple[1].strip()+"\n\n") 
        # 主函数
        def workOn(self):
            self.getPage(self.baseurl)
            while True:
                c = input("成功,是否继续(y/n):")
                if c.strip().lower() == "y":
                    url = self.baseurl + "index_" + str(self.page) + ".html"
                    self.getPage(url)
                    self.page += 1
                else:
                    print("爬取结束")
                    break
                
    if __name__ == "__main__":
        spider = NeihanSpider()
        spider.workOn()

7.csv模块使用流程

1. 打开csv文件
         with open("测试.csv","w",newline="",encoding="gb18030") as f:
      2. 初始化写入对象
         writer = csv.writer(f)
      3. 写入数据
         writer.writerow(列表)

import urllib.request
    import re
    import csv
    
    class MaoyanSpider:
        def __init__(self):
            self.baseurl = "https://maoyan.com/board/4?offset="
            self.headers = { "User-Agent":"Mozilla/5.0"}
            self.offset = 0
        
        # 获取页面
        def get_page(self,url):
            req = urllib.request.Request(url,
                        headers=self.headers)
            res = urllib.request.urlopen(req)
            html = res.read().decode("utf-8")
            self.parsePage(html)
        
        # 解析页面
        def parse_page(self,html):
            # 创建编译对象
            p = re.compile('<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?releasetime">(.*?)</p>',re.S)
            rList = p.findall(html)
            # rList:[("霸王别姬","张国荣","1993"),()]
            self.writeToCSV(rList)
            
        # 保存数据
        def write_to_csv(self,rList):
            for r in rList:
                #r = list(r)
                r = [r[0].strip(),r[1].strip(),r[2].strip()]
                with open("猫眼.csv","a",newline="",encoding="gb18030") as f:
                    # 创建写入对象
                    writer = csv.writer(f)
                    # 调用writerow()方法
                    writer.writerow(r)
                    
        # 主函数
        def workOn(self):
            while True:
                c = input("爬取按y,退出按q:")
                if c.strip().lower() == "y":           
                    url = self.baseurl + str(self.offset)
                    self.getPage(url)
                    self.offset += 10
                else:
                    print("爬取结束")
                    break
                    
            #for i in range(0,91,10):
            # url = self.baseurl + str(i)
            # self.getPage(url)
            # time.sleep(0.1)
               
    if __name__ == "__main__":
        spider = MaoyanSpider()
        spider.workOn()