curl采集

淡淡的烟草味﹌ 2021-10-18 18:24 386阅读 0赞

// $url = 'www.baidu.com';
    $url = 'https://h5.eqxiu.com/s/CzvPVCNt';
    
    function http_curl($url, $post = '', $cookie, $cookiejar, $referer = '')
    {
    
        //使用透明代理时伪造ip
        $headers = forgeIp();
        //POST提交大数据（超过1024字节）异常解决方法
        $headers['Expect:'] = '';
        $options = [
    
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_HEADER => true,//返回内容包含响应头
            CURLOPT_TIMEOUT => 30,//执行时间
            //设置 HTTP 头字段的数组
            CURLOPT_HTTPHEADER => $headers,
            //模拟浏览器行为，在HTTP请求中包含一个"User-Agent: "头的字符串。
            CURLOPT_USERAGENT => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
            //返回 HTTP 头中的 "Location: " 重定向页面
            CURLOPT_FOLLOWLOCATION => true,
            //取消https请求的证书认证
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => 0,
            //追踪请求头信息，用于后续curl_getinfo获取输出
            CURLINFO_HEADER_OUT => true
    
        ];
    
        $ch = curl_init();
        curl_setopt_array($ch, $options);
    
        //post请求
        if ($post) {
            curl_setopt($ch, CURLOPT_POST, true);
            curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($post));
        }
    
        //设定 HTTP 请求中Cookie
        if ($cookie) {
            curl_setopt($ch, CURLOPT_COOKIE, $cookie);
        }
    
        if ($cookiejar) {
            $cachePath = __DIR__ . '/cache/' . $cookiejar;
            //连接结束后，将cookie信息保存到文件中
            curl_setopt($ch, CURLOPT_COOKIEJAR, $cachePath);
            //包含 cookie 数据的文件名
            curl_setopt($ch, CURLOPT_COOKIEFILE, $cachePath);
        }
    
        if ($referer) {
            //在HTTP请求头中"Referer: "的内容。
            curl_setopt($ch, CURLOPT_REFERER, $referer);
        } else {
            // TRUE 时将根据 Location: 重定向时，自动设置 header 中的Referer:信息。
            curl_setopt($ch, CURLOPT_AUTOREFERER, true);
        }
    
        $data = curl_exec($ch);
    
        //执行之后获取请求头信息
        $request_headers = curl_getinfo($ch, CURLINFO_HEADER_OUT);
    
        if(curl_exec($ch) === false) {
            return $data;
        } else {
            echo 'CURL_ERRNO:' . curl_errno($ch);
            echo '<br>';
            echo 'CURL_ERROR:' . curl_error($ch);
        }
    
        curl_close($ch);
    
        return [
          'content' => $data,
          'request_headers' => $request_headers
        ];
    }
    
    //此函数提供了国内的IP地址
    function forgeIp()
    {
        $ip_long = array(
            array('607649792', '608174079'), //36.56.0.0-36.63.255.255
            array('1038614528', '1039007743'), //61.232.0.0-61.237.255.255
            array('1783627776', '1784676351'), //106.80.0.0-106.95.255.255
            array('2035023872', '2035154943'), //121.76.0.0-121.77.255.255
            array('2078801920', '2079064063'), //123.232.0.0-123.235.255.255
            array('-1950089216', '-1948778497'), //139.196.0.0-139.215.255.255
            array('-1425539072', '-1425014785'), //171.8.0.0-171.15.255.255
            array('-1236271104', '-1235419137'), //182.80.0.0-182.92.255.255
            array('-770113536', '-768606209'), //210.25.0.0-210.47.255.255
            array('-569376768', '-564133889'), //222.16.0.0-222.95.255.255
        );
        $rand_key = array_rand($ip_long);
        $ip = long2ip(mt_rand($ip_long[$rand_key][0], $ip_long[$rand_key][1]));
        $headers['CLIENT-IP'] = $ip;
        $headers['X-FORWARDED-FOR'] = $ip;
    
        $headerArr = array();
        foreach ($headers as $n => $v) {
            $headerArr[] = $n . ':' . $v;
        }
        return $headerArr;
    }
    
    $info = http_curl($url, '', 'cookie.txt', 'https://www.jstor.org/', '', true);
    var_dump($info);