使用scrapy抓取Youtube播放列表信息

柔情只为你懂 2022-03-02 00:52 315阅读 0赞

可参看[Knowsmore][]

**抓取Youtube列表数据的前提是scrapy部署的机器可以正常访问Youtube网站**

[示例网址][Link 1]

存取到Mongo中的数据如下：

{
        "playlist_id" : "PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9",
        "videos" : [
            {
                "playlist_id" : "PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9",
                "video_id" : "9pTwztLOvj4",
                "thumbnail" : [
                    {
                        "url" : "https://i.ytimg.com/vi/9pTwztLOvj4/hqdefault.jpg?sqp=-oaymwEZCPYBEIoBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLCmUXUPe-HgXiie0SRfL5cYz0JRrg",
                        "width" : 245,
                        "height" : 137
                    }
                ],
                "title" : "Legend of the galactic heroes (1988) episode 1",
                "index" : 1,
                "length_seconds" : 1445,
                "is_playable" : true
            },
            {
                "playlist_id" : "PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9",
                "video_id" : "zzD1xU37Vtc",
                "thumbnail" : [
                    {
                        "url" : "https://i.ytimg.com/vi/zzD1xU37Vtc/hqdefault.jpg?sqp=-oaymwEZCPYBEIoBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLCnLCYaZVBeHnZR0T73rfEd_Dbyew",
                        "width" : 245,
                        "height" : 137
                    }
                ],
                "title" : "Legend of the galactic heroes (1988) episode 2",
                "index" : 2,
                "length_seconds" : 1447,
                "is_playable" : true
            },

代码如下：

# -*- coding: utf-8 -*-
    import scrapy
    import re
    import json
    from scrapy import Selector
    from knowsmore.items import YoutubePlaylistItem, YoutubePlaylistVideoItem
    from ..common import *
    
    class YoutubeListSpider(scrapy.Spider):
        name = 'youtube_list'
        allowed_domains = ['www.youtube.com']
        start_urls = ['https://www.youtube.com/playlist?list=PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9']
    
        def parse(self, response):
            # Extract JSON Data with Regex Expression
            ytInitialData = r1(r'window\["ytInitialData"\] = (.*?)}};', response.body)
            if ytInitialData:
                ytInitialData = '%s}}' % ytInitialData
                ytInitialDataObj = json.loads(ytInitialData)
    
                # Assign VideoList info to variable
                playListInfo = ytInitialDataObj['contents']['twoColumnBrowseResultsRenderer']['tabs'][0]['tabRenderer']['content']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['playlistVideoListRenderer']
    
                # Build Scrapy Item
                playList = YoutubePlaylistItem(
                    playlist_id = playListInfo['playlistId'],
                    videos = []
                )
    
                # Insert the videoItem to YoutubePlaylistItem videos field
                for videoInfo in playListInfo['contents']:
                    videoInfo = videoInfo['playlistVideoRenderer']
                    videoItem = YoutubePlaylistVideoItem(
                        playlist_id = playListInfo['playlistId'],
                        video_id = videoInfo['videoId'],
                        thumbnail = videoInfo['thumbnail']['thumbnails'],
                        title = videoInfo['title']['simpleText'],
                        index = videoInfo['index']['simpleText'],
                        length_seconds = videoInfo['lengthSeconds'],
                        is_playable = videoInfo['isPlayable']
                    )
                    playList['videos'].append(videoItem)
                
                yield playList

[Knowsmore]: https://github.com/libbit702/knowsmore
[Link 1]: https://www.youtube.com/playlist?list=PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9