Python爬虫学习笔记之爬取新浪微博

拼搏现实的明天。 2022-01-05 02:33 301阅读 0赞

1 import requests
     2 from urllib.parse import urlencode
     3 from pyquery import PyQuery as pq
     4 from pymongo import MongoClient
     5 
     6 base_url = 'https://m.weibo.cn/api/container/getIndex?'
     7 headers = {
     8     'Host': 'm.weibo.cn',
     9     'Referer': 'https://m.weibo.cn/u/2803301701',
    10     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36',
    11     'X-Requested-With': 'XMLHttpRequest',
    12 }
    13 client = MongoClient()
    14 db = client['weibo']
    15 collection = db['weibo']
    16 max_page = 10
    17 
    18 
    19 def get_page(page):
    20     params = {
    21         'type': 'uid',
    22         'value': '2803301701',
    23         'containerid': '1076032803301701',
    24         'page': page
    25     }
    26     url = base_url + urlencode(params)
    27     try:
    28         response = requests.get(url, headers=headers)
    29         if response.status_code == 200:
    30             return response.json(), page
    31     except requests.ConnectionError as e:
    32         print('Error', e.args)
    33 
    34 
    35 def parse_page(json, page: int):
    36     if json:
    37         items = json.get('data').get('cards')
    38         for index, item in enumerate(items):
    39             if page == 1 and index == 1:
    40                 continue
    41             else:
    42                 item = item.get('mblog')
    43                 weibo = {}
    44                 weibo['id'] = item.get('id')
    45                 weibo['text'] = pq(item.get('text')).text()
    46                 weibo['attitudes'] = item.get('attitudes_count')
    47                 weibo['comments'] = item.get('comments_count')
    48                 weibo['reposts'] = item.get('reposts_count')
    49                 yield weibo
    50 
    51 
    52 # def save_to_mongo(result):
    53     # if collection.insert(result):
    54         # print('Saved to Mongo')
    55 
    56 
    57 if __name__ == '__main__':
    58     for page in range(1, max_page + 1):
    59         json = get_page(page)
    60         results = parse_page(*json)
    61         for result in results:
    62             print(result)
    63             # save_to_mongo(result)

**运行结果:**

**![1257995-20180815200315386-1801395006.png][]**

转载于:https://www.cnblogs.com/Trojan00/p/9483676.html

[1257995-20180815200315386-1801395006.png]: /images/20211230/5b10ca0cac9743b4aefac43c2683f828.png