爬虫之Scrapy递归爬取网页信息

梦里梦外; 2022-05-19 10:49 232阅读 0赞

# -*- coding: utf-8 -*-
 import re
 
 import scrapy
 
 from zhipin.items import ZhipinItem
 
 
 class BossZhipinSpider(scrapy.Spider):
 name = 'boss_zhipin'
 allowed_domains = ['https://www.zhipin.com']
 url = 'https://www.zhipin.com/c101020100/h_101020100/?query=python&page=%s'
 offset = 1
 start_urls = [url % offset]
 
 # https: // www.zhipin.com / c101020100 / h_101020100 /?query = python & page = 1
 # https: // www.zhipin.com / c101020100 / h_101020100 /?query = python & page = 10
 
 def parse(self, response):
 item = ZhipinItem()
 for response_part in response.css('#main > div > div.job-list > ul').extract():
 company_size = []
 company_info = re.findall('(.*?)', response_part)
 # 算出列表长度，拿到偶数位的数据
 # for idx in range(1, len(company_info)+1, 2):
 for idx, val in enumerate(company_info):
 if int(idx) % 2 != 0:
 if '' in company_info[idx]:
 new_item = company_info[idx].rsplit('')[-1]
 company_size.append(new_item)
 else:
 # 没带直接加到company_size中
 company_size.append(company_info[idx])
 result = zip(re.findall('title">(.*?)</div>', response_part),
 re.findall('(.*?)', response_part),
 re.findall('ka="search_list_company_\d+_custompage" target="_blank">(.*?)</a>', response_part),
 company_size,
 re.findall('发布于(.*?)', response_part))
 for job_item in result:
 """ 处理元组数据，返回item """
 item['job_title'] = job_item[0]
 item['job_salary'] = job_item[1]
 item['job_company'] = job_item[2]
 item['company_size'] = job_item[3]
 item['publish_date'] = job_item[4]
 yield item
 
 if self.offset < 10:
 self.offset += 1
 yield scrapy.Request(self.url % self.offset, callback=self.parse, dont_filter=True)

#### 在递归爬取过程中遇到一个warning： ####

`2018-07-24 16:38:09 [scrapy.spidermiddlewares.offsite] DEBUG: Filtered offsite request to 'www.zhipin.com':`

#### 问题出现原因： ####

> 因为 Request中请求的 URL 和 allowed\_domains 中定义的域名冲突，所以将Request中请求的URL过滤掉了，无法请求

#### 问题解决方案，有个参数dont\_filter： ####

> yield scrapy.Request(self.url % self.offset, callback=self.parse, dont\_filter=True)

##### 递归无法停止？分析下条件，将代码缩进下就可以了！！！ #####

GitHub地址：  
[https://github.com/QuincyC379/zhipin][https_github.com_QuincyC379_zhipin]

[https_github.com_QuincyC379_zhipin]: https://github.com/QuincyC379/zhipin