【youtubo】爬虫数据采集案例

朴灿烈づ我的快乐病毒、 2024-04-17 11:27 163阅读 0赞

" class="reference-link">从此烟雨落金城，一人撑伞两人行在这里插入图片描述

案例展示

  1. # -*- coding:utf-8 -*-
  2. # @software: PyCharm
  3. # desc:
  5. import datetime
  6. import json
  7. from loguru import logger as logging
  8. import re
  9. from json import load, dumps
  10. from os import path
  11. from re import findall
  13. import pymongo
  14. import requests
  15. import scrapy
  16. from pymongo.errors import DuplicateKeyError
  18. cwd = path.dirname(path.abspath(__file__))
  21. class VideoError(Exception):
  22.     def __init__(self, vid):
  23.         self.message = f'Invalid video ID. Are you sure "{
  24.       vid}" is a valid URL?'
  25.         super().__init__(self.message)
  28. class PlaylistError(Exception):
  29.     def __init__(self, pid):
  30.         self.message = f'Invalid Playlist ID. Are you sure "{
  31.       pid}" is a valid URL and available?'
  32.         super().__init__(self.message)
  35. def fetch_and_save_video_info():
  36.     gMongoClient = pymongo.MongoClient()
  37.     gMongoDb = gMongoClient['crawlers']
  38.     gMongoCollection = gMongoDb['youtube']
  40.     docs = gMongoCollection.find({
  41.     })
  42.     for doc in docs:
  43.         videoId = doc['videoId']
  44.         url = f"https://www.youtube.com/watch?v={
  45.       videoId}"
  46.         title = doc['title']
  47.         shortViewCountText = doc['shortViewCountText']
  48.         channelTitle = doc['channelTitle']
  49.         channelId = doc['channelId']
  50.         canonicalBaseUrl = doc['canonicalBaseUrl']
  51.         subscriberCount = doc['subscriberCount']
  52.         videosCount = doc['videosCount']
  53.         headers = {
  55.             'user-agent': (
  56.                 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'),
  57.             'referer': 'https://youtube.com'}
  59.         vid = "".join([i for i in findall(r"v=(.*?)&|youtu.be\/(.*?)&", url + "&")[0]])
  60.         logging.info(vid)
  61.         json_file = load(open(cwd + "/tube_dl_config.json", "rb"))
  63.         headers["x-youtube-client-version"] = json_file['cver']
  64.         headers["x-youtube-client-name"] = json_file['cname']
  65.         y_data = requests.get(url=f"https://youtube.com/watch?v={
  66.       vid}&pbj=1", headers=headers,
  67.                               ).json()
  68.         yt_data = [i for i in y_data if "playerResponse" in i.keys()][0]["playerResponse"]
  69.         if yt_data["playabilityStatus"]["status"] == "ERROR":
  70.             raise VideoError(vid)
  72.         shortDescription = yt_data['videoDetails']['shortDescription']
  73.         logging.info(dumps(y_data))
  74.         # logging.info(shortDescription)
  76.         # "label": "102,293 likes"
  77.         likes = re.findall('"defaultText": \{"accessibility": \{"accessibilityData": \{"label": "(.*?) likes"\}\}',
  78.                            dumps(y_data))[0]
  79.         try:
  80.             if likes:
  81.                 # 格式化成数字
  82.                 likes = int(likes.replace(',', ''))
  83.         except Exception as e:
  84.             logging.info(e)
  85.             likes = 0
  86.         publishDate = yt_data['microformat']['playerMicroformatRenderer']['publishDate']  # 2023-11-04T16:00:11-07:00
  87.         publishDateDay = publishDate.split('T')[0]  # 2023-11-04
  89.         viewCount = re.findall('"allowRatings": true, "viewCount": "(.*?)",', dumps(y_data))[0]
  91.         if '万 个视频' in videosCount:
  92.             videosCount = videosCount.replace('万 个视频', '')
  93.             videosCount = float(videosCount) * 10000
  94.         if '万位订阅者' in subscriberCount:
  95.             subscriberCount = subscriberCount.replace('万位订阅者', '')
  96.             subscriberCount = float(subscriberCount) * 10000
  98.         item = {
  100.             'videoId': videoId,
  101.             'title': title,
  102.             'viewCount': viewCount,
  103.             'channelTitle': channelTitle,
  104.             'channelId': channelId,
  105.             'canonicalBaseUrl': canonicalBaseUrl,
  106.             'subscriberCount': subscriberCount,
  107.             'videosCount': videosCount,
  108.             'shortDescription': shortDescription,
  109.             'likes': likes,
  110.             'createTime': datetime.datetime.now(),
  111.             'day': datetime.datetime.now().strftime('%Y-%m-%d'),
  112.             "publishDate": publishDate,
  113.             "publishDateDay": publishDateDay,
  114.         }
  115.         gMongoDb.get_collection('youtube_detail_info').insert_one(item)
  118. def fetch_and_save_comments_info():
  119.     gMongoClient = pymongo.MongoClient()
  120.     gMongoDb = gMongoClient['crawlers']
  121.     docs = gMongoDb.get_collection('youtube_detail_info').find({
  122.     "commentsCount": {
  123.     "$exists": False}})
  124.     for doc in docs:
  125.         url = f'https://www.youtube.com/watch?v={
  126.       doc["videoId"]}'
  128.         headers = {
  130.             'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
  131.             'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  132.             'x-client-data': '自定义',
  133.             'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1',
  134.             'Cookie': f"{
  135.       cookie}",
  136.             'sec-fetch-dest': 'document', 'accept-encoding': 'gzip, deflate, br',
  137.             'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
  138.         }
  139.         resp = requests.get(url=url, headers=headers)
  141.         response = scrapy.Selector(text=resp.text)
  142.         token = re.findall(r'"continuationCommand":{"token":"(.*?)"', resp.text)[0]
  144.         apiKey = re.findall(r'"innertubeApiKey":"(.*?)"', resp.text)[0]
  146.         url = f'https://www.youtube.com/youtubei/v1/next?key={
  147.       apiKey}&prettylogging.info=false'
  149.         payload = {
  151.             "context": {
  152.     "client": {
  153.     "deviceMake": "Apple", "deviceModel": "", "visitorData": "", "clientName": "WEB",
  154.                                    "clientVersion": "2.20240123.01.00", "configInfo": {
  155.     "appInstallData": ""},
  156.                                    "mainAppWebInfo": {
  157.     "graftUrl": ""}}, "user": {
  158.     "lockedSafetyMode": False},
  159.                         "request": {
  160.     "useSsl": True, "internalExperimentFlags": [], "consistencyTokenJars": []},
  161.                         "clickTracking": {
  162.     "clickTrackingParams": ""}, "adSignalsInfo": {
  163.     "params": [], "bid": ""}},
  164.             "continuation": f"{
  165.       token}"}
  166.         videoResp = requests.post(url=url, headers=headers, json=payload)
  167.         videoRespJson = json.loads(videoResp.text)
  168.         try:
  169.             commentsCount = re.findall(r'"commentsCount":\{"runs":\[\{"text":"(.*?)"\}\]\}', videoResp.text)[0]
  170.         except Exception as e:
  171.             logging.info(e)
  172.             commentsCount = 0
  174.         gMongoDb.get_collection('youtube_detail_info').update_one({
  175.     '_id': doc['_id']},
  176.                                                                   {
  177.     '$set': {
  178.     'commentsCount': commentsCount}})
  181. def get_video_list(token, channelTitle, channelId, canonicalBaseUrl, subscriberCount, videosCount, gMongoDb):
  182.     videlListUrl = 'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettylogging.info=false'
  183.     payload = {
  184.     "context": {
  185.     "client": {
  186.     "deviceMake": "Apple", "deviceModel": "", "visitorData": "", "clientName": "WEB",
  187.                                       "clientVersion": "2.20240123.01.00", "configInfo": {
  188.     "appInstallData": ""},
  189.                                       "mainAppWebInfo": {
  190.     "graftUrl": ""}}, "user": {
  191.     "lockedSafetyMode": False},
  192.                            "request": {
  193.     "useSsl": True, "internalExperimentFlags": [], "consistencyTokenJars": []},
  194.                            "clickTracking": {
  195.     "clickTrackingParams": ""}, "adSignalsInfo": {
  196.     "params": [], "bid": ""}},
  197.                "continuation": f"{
  198.       token}"}
  200.     headers = {
  202.         'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
  203.         'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  204.         'x-client-data': '自定义',
  205.         'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1',
  206.         'Cookie': f"{
  207.       cookie}",
  208.         'sec-fetch-dest': 'document', 'accept-encoding': 'gzip, deflate, br',
  209.         'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
  210.     }
  211.     videoResp = requests.post(url=videlListUrl, headers=headers, json=payload, )
  212.     videoRespJson = json.loads(videoResp.text)
  213.     logging.info(videoResp.text)
  214.     # .onResponseReceivedActions[0].appendContinuationItemsAction.continuationItems
  215.     videoList = videoRespJson['onResponseReceivedActions'][0]['appendContinuationItemsAction']['continuationItems']
  216.     for eVideo in videoList[:-1]:
  217.         videoId = eVideo['richItemRenderer']['content']['videoRenderer']['videoId']
  218.         title = eVideo['richItemRenderer']['content']['videoRenderer']['title']['runs'][0]['text']
  219.         # richItemRenderer.content.videoRenderer.shortViewCountText.simpleText
  220.         shortViewCountText = eVideo['richItemRenderer']['content']['videoRenderer']['shortViewCountText']['simpleText']
  221.         item = {
  223.             'videoId': videoId,
  224.             'title': title,
  225.             'shortViewCountText': shortViewCountText,
  226.             'channelTitle': channelTitle,
  227.             'channelId': channelId,
  228.             'canonicalBaseUrl': canonicalBaseUrl,
  229.             'subscriberCount': subscriberCount,
  230.             'videosCount': videosCount,
  231.         }
  232.         logging.info(item)
  233.         try:
  234.             gMongoDb['youtube'].insert_one(item)
  235.         except DuplicateKeyError as e:
  236.             logging.info("重复数据")
  237.     try:
  238.         continuationCommand = videoList[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']
  239.         token = continuationCommand['token']
  240.         return get_video_list(token)
  241.     except Exception as e:
  242.         logging.info(e)
  243.         logging.info('没有下一页了')
  246. def fetch_and_save_list_info():
  247.     gMongoClient = pymongo.MongoClient()
  248.     gMongoDb = gMongoClient['crawlers']
  249.     url = 'https://www.youtube.com/@ganfutong/videos'
  251.     headers = {
  253.         'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
  254.         'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  255.         'x-client-data': '自定义',
  256.         'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1',
  257.         'Cookie': f"{
  258.       cookie}",
  259.         'sec-fetch-dest': 'document', 'accept-encoding': 'gzip, deflate, br',
  260.         'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
  261.     }
  262.     resp = requests.get(url=url, headers=headers,
  263.                         )
  265.     response = scrapy.Selector(text=resp.text)
  266.     token = re.findall(r'"continuationCommand":{"token":"(.*?)"', resp.text)[0]
  267.     context = response.xpath("//script[contains(string(),'var ytInitialData')]/text()").extract_first()
  268.     contextJson = json.loads(context.replace('var ytInitialData = ', '')[:-1])
  270.     subscriberCount = contextJson['header']['c4TabbedHeaderRenderer']['subscriberCountText']['simpleText']
  271.     videosCountTexts = contextJson['header']['c4TabbedHeaderRenderer']['videosCountText']['runs']
  272.     channelTitle = contextJson['header']['c4TabbedHeaderRenderer']['title']
  273.     channelId = contextJson['header']['c4TabbedHeaderRenderer']['channelId']
  274.     canonicalBaseUrl = contextJson['header']['c4TabbedHeaderRenderer']['navigationEndpoint']['browseEndpoint'][
  275.         'canonicalBaseUrl']
  276.     videosCount = ''
  277.     for i in videosCountTexts:
  278.         videosCount += i['text']
  280.     token = token
  281.     logging.info(token)
  282.     get_video_list(token, channelTitle, channelId, canonicalBaseUrl, subscriberCount, videosCount, gMongoDb)
  285. def fetch_and_save_account_info():
  286.     gMongoClient = pymongo.MongoClient()
  287.     gMongoDb = gMongoClient['crawlers']
  288.     url = 'https://www.youtube.com/@ganfutong'
  290.     headers = {
  292.         'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
  293.         'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
  294.         'x-client-data': 'CIq2yQEIpbbJAQipncoBCLXsygEIk6HLAQia/swBCIagzQEIj+HNAQiE4s0BCN/rzQEI5uzNAQjB7s0BCIrvzQEIg/DNAQiG8M0BCL7xzQEIjPLNARj2yc0BGKfqzQEY+fLNAQ==',
  295.         'Cookie': f"{
  296.       cookie}",
  297.         'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1',
  298.         'sec-fetch-dest': 'document', 'accept-encoding': 'gzip, deflate, br',
  299.         'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
  300.     }
  301.     resp = requests.get(url=url, headers=headers)
  303.     response = scrapy.Selector(text=resp.text)
  304.     token = re.findall(r'"continuationCommand":{"token":"(.*?)"', resp.text)[0]
  305.     context = response.xpath("//script[contains(string(),'var ytInitialData')]/text()").extract_first()
  306.     contextJson = json.loads(context.replace('var ytInitialData = ', '')[:-1])
  308.     # 订阅数.header.c4TabbedHeaderRenderer.subscriberCountText
  309.     subscriberCount = contextJson['header']['c4TabbedHeaderRenderer']['subscriberCountText']['simpleText']
  310.     # 视频数 .header.c4TabbedHeaderRenderer.videosCountText
  311.     videosCountTexts = contextJson['header']['c4TabbedHeaderRenderer']['videosCountText']['runs']
  312.     channelTitle = contextJson['header']['c4TabbedHeaderRenderer']['title']
  313.     channelId = contextJson['header']['c4TabbedHeaderRenderer']['channelId']
  314.     apiKey = re.findall(r'"innertubeApiKey":"(.*?)"', resp.text)[0]
  316.     canonicalBaseUrl = contextJson['header']['c4TabbedHeaderRenderer']['navigationEndpoint']['browseEndpoint'][
  317.         'canonicalBaseUrl']
  318.     videosCount = ''
  319.     for i in videosCountTexts:
  320.         videosCount += i['text']
  321.     logging.info(json.dumps(contextJson))
  322.     gMongoDb.get_collection('youtube_account_info').insert_one(item := {
  324.         'channelTitle': channelTitle,
  325.         'channelId': channelId,
  326.         'canonicalBaseUrl': canonicalBaseUrl,
  327.         'subscriberCount': subscriberCount,
  328.         'videosCount': videosCount,
  329.         'apiKey': apiKey,
  330.         "createTime": datetime.datetime.now(),
  331.     })
  334. if __name__ == '__main__':
  335.     cookie = ''
  336.     fetch_and_save_comments_info()

发表评论

表情：
评论列表 （有 0 条评论，163人围观）

还没有评论，来说两句吧...

相关阅读

    相关 Day7.数据采集-爬虫

    数据采集 我们进行数据分析以及挖掘时，前提条件就是需要有数据；如果在公司里作业，我们可以从数据库中导入数据，但同时我们也可以对采集数据来进行分析。采集数据最常用就是我们听

    爱被打了一巴掌爱被打了一巴掌/ 0 赞/ 42 阅读