获取xhr请求工具类

￡神魔★判官ぃ 2022-10-10 06:04 158阅读 0赞

import json
    from selenium import webdriver
    from selenium.webdriver import DesiredCapabilities
    
    
    def get_xhr_logs(chrome):
        log_xhr_array = []
        for typelog in chrome.log_types:
            perfs = chrome.get_log(typelog)
            for row in perfs:
                log_data = row
                message_ = log_data['message']
                try:
                    log_json = json.loads(message_)
                    log = log_json['message']
                    if log['method'] == 'Network.responseReceived':
                        # 去掉静态js、css等，仅保留xhr请求
                        type_ = log['params']['type']
                        if type_ == "XHR":
                            log_xhr_array.append(log)
                except:
                    pass
        return log_xhr_array
    
    
    def get_log_options():
        option = webdriver.ChromeOptions()
        option.add_argument('--no-sandbox')
        option.add_argument('--headless')
        option.add_argument("--disable-extensions")
        option.add_argument("--allow-running-insecure-content")
        option.add_argument("--ignore-certificate-errors")
        option.add_argument("--disable-single-click-autofill")
        option.add_argument("--disable-autofill-keyboard-accessory-view[8]")
        option.add_argument("--disable-full-form-autofill-ios")
        option.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0')
        option.add_experimental_option('w3c', False)
        option.add_experimental_option('perfLoggingPrefs', {
            'enableNetwork': True,
            'enablePage': False,
        })
        return option
    
    
    def get_caps():
        caps = DesiredCapabilities.CHROME
        caps['loggingPrefs'] = {
            'browser': 'ALL',
            'performance': 'ALL',
        }
        caps['perfLoggingPrefs'] = {
            'enableNetwork': True,
            'enablePage': False,
            'enableTimeline': False
        }
        return caps

使用：

# 拦截进入详情页面，使用selenium绕过反爬
    class SeleniumMiddlewareDetail:
        def process_response(self, request, response, spider):
            if request.meta['code'] == 1:
                chrome = spider.chrome
                chrome.get(request.url)
                # network日志
                logs = get_xhr_logs(chrome)
                return HtmlResponse(url=spider.chrome.current_url,
                                    body=bytes(json.dumps(logs), encoding='utf8'),
                                    encoding="utf8",
                                    request=request)