爬取腾讯招聘网站(多线程和MongoDB)

今天药忘吃喽~ 2022-12-06 01:11 358阅读 0赞

@TOC

# 分析 #

**我们这里搜索Python的职位**  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ExMzA5NTI1ODAy_size_16_color_FFFFFF_t_70_pic_center]  
**打开控制台看数据**  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ExMzA5NTI1ODAy_size_16_color_FFFFFF_t_70_pic_center 1]

**从图中可以看到这个请求有我们所需要的数据**

# 代码实现 #

这里要用到mongodb数据库,所以需要下载模块 pymongo

pip install pymongo

## 多线程 ##

import threading
    import requests
    import pymongo
    from queue import Queue
    class Tencent(threading.Thread):
        def __init__(self,url,name,q_task):
            super().__init__()
            self.url=url
            self.name=name
            self.q_task=q_task
            # 1.创建连接
            self.client = pymongo.MongoClient(host='localhost', port=27017)
            # 连接数据库
            self.db = self.client['tencent_data']
    	#插入到mongo中
        def write_to_mongo(self,data):
            # 插入数据
            self.db['招聘信息'].update({ 'PostId': data['PostId']}, { '$set': data}, upsert=True)
    	#获取相应数据
        def get_rep(self,page):
        	#请求头
            headers = { 
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.6.1000'
            }
            params = { 
                'timestamp': '1599786948721',
                'countryId': '',
                'cityId': '',
                'bgIds': '',
                'productId': '',
                'categoryId': '',
                'parentCategoryId': '',
                'attrId': '',
                'keyword': 'python',
                'pageIndex': str(page),
                'pageSize': '10',
                'language': 'zh-cn',
                'area': 'cn',
            }
            return requests.get(self.url, headers=headers, params=params)
    	#解析json数据
        def parse_json(self,json_data):
            for data in json_data['Data']['Posts']:
                self.write_to_mongo(data)
        #获取页面
        def parse_page(self,page):
            rep = self.get_rep(page).json()
            self.parse_json(rep)
        def run(self):
            while True:
                if self.q_task.empty():
                    break
                #取出页面
                page=self.q_task.get()
                #请求解析
                print(f"=======================第{page}页==========={self.name}")
                self.parse_page(page)
    
    
    
    if __name__ == '__main__':
        url = 'https://careers.tencent.com/tencentcareer/api/post/Query'
        q_page = Queue()
        for page in range(1, 100):
            q_page.put(page)
        #添加线程名称
        threading_names=["线程1","线程2","线程3","线程4"]
        #启动线程
        for name in threading_names:
            t=Tencent(url,name,q_page)
            t.start()

## 生产者和消费者 ##

import threading
    import requests
    import pymongo
    from queue import Queue
    #生产者
    class Product(threading.Thread):
        def __init__(self,url,q_page,name):
            super().__init__()
            self.url=url
            self.q_page=q_page
            self.name=name
        def get_rep(self,page):
            headers = { 
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.6.1000'
            }
            params = { 
                'timestamp': '1599786948721',
                'countryId': '',
                'cityId': '',
                'bgIds': '',
                'productId': '',
                'categoryId': '',
                'parentCategoryId': '',
                'attrId': '',
                'keyword': 'python',
                'pageIndex': str(page),
                'pageSize': '10',
                'language': 'zh-cn',
                'area': 'cn',
            }
            return requests.get(self.url, headers=headers, params=params)
    
        def parse_page(self,page):
            return self.get_rep(page).json()
        def run(self):
            while True:
                if self.q_page.empty():
                    break
                # 取出页面
                page = self.q_page.get()
                # 请求解析
                print(f"|=======================第{page}页==========={self.name}| ")
                #添加json
                q_json.put(self.parse_page(page))
    
    #消费者
    class Consumer(threading.Thread):
        def __init__(self,name):
            super().__init__()
            self.name=name
            # 1.创建连接
            self.client = pymongo.MongoClient(host='localhost', port=27017)
            # 连接数据库
            self.db = self.client['tencent_data']
        def write_to_mongo(self,data):
            # 插入数据
            self.db['招聘信息'].update({ 'PostId': data['PostId']}, { '$set': data}, upsert=True)
        def parse_json(self,json_data):
            for data in json_data['Data']['Posts']:
                self.write_to_mongo(data)
        def run(self):
            while True:
                if q_json.empty() and flag:
                    break
                try:
                    json_data=q_json.get(block=False)
                    print(f"|============{self.name}正在保存======================|")
                    #解析保存
                    self.parse_json(json_data)
                except Exception:
                    continue
    if __name__ == '__main__':
        #轮询参数
        flag=False #判断生产者是否完成
        url = 'https://careers.tencent.com/tencentcareer/api/post/Query'
        q_page = Queue()
        q_json=Queue()
        for page in range(1, 100):
            q_page.put(page)
        #添加生成者线程
        Product_names=["生产者1","生产者2","生产者3","生产者4"]
        Consumer_names=["消费者1","消费者2","消费者3","消费者4"]
        #保存生产者的每个线程的引用
        crawl_p=[]
        #启动线程
        for name in Product_names:
            p=Product(url,q_page,name)
            p.start()
            crawl_p.append(p)
    
        for name in Consumer_names:
            p=Consumer(name)
            p.start()
        #阻塞生产者都完成
        a=[p.join() for p in crawl_p]
        flag=True

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ExMzA5NTI1ODAy_size_16_color_FFFFFF_t_70_pic_center]: /images/20221123/40b6bb631e5c41acb5fed52f0348cf37.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ExMzA5NTI1ODAy_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221123/294a9ad2876b46548bb74dd8d78a945c.png