php脚本爬取头像图片

水深无声 2022-07-15 14:18 132阅读 0赞

##### 因为要插入系统用户，所以用php爬取百度图片上的头像url，再存储到本地，速度一般，1000张图片差不多花费半个多小时，不知道是不是因为 ***file\_get\_content*** 函数的缘故，或者是没有开多线程 php-fpm , 没有仔细研究优化，提高脚本速度可以从多线程和异步网络请求然后回调去解决。一下是代码 #####

<?php
    error_reporting(E_ALL ^ E_NOTICE);
    set_time_limit(0);
    
    $pageNum = 3;
    //获取数据
    for ($page=0; $page < $pageNum; $page++) {
        //翻页数据
        $pnArr = array('0','30','60','90','120','150','180','210','240','270','300','330','360'); 
        $rand = rand(0,5);
    
        //百度头像图片获取api
        $urlArr = array(
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E6%B5%B7%E8%BE%B9&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E6%B5%B7%E8%BE%B9&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=3c&1476431870063=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%BA%BA%E7%89%A9%E5%BD%A2%E8%B1%A1+%E5%8D%8A%E8%BA%AB&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%BA%BA%E7%89%A9%E5%BD%A2%E8%B1%A1+%E5%8D%8A%E8%BA%AB&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=1e&1476431965788=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%BA%BA%E7%89%A9%E5%BD%A2%E8%B1%A1+%E8%90%9D%E8%8E%89&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%BA%BA%E7%89%A9%E5%BD%A2%E8%B1%A1+%E8%90%9D%E8%8E%89&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=3c&1476432025419=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%BA%BA%E7%89%A9%E5%BD%A2%E8%B1%A1+%E7%94%B7%E7%94%9F&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%BA%BA%E7%89%A9%E5%BD%A2%E8%B1%A1+%E7%94%B7%E7%94%9F&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=5a&1476432073843=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E7%BB%8F%E5%85%B8%E5%8A%A8%E4%BD%9C+%E5%98%9F%E5%98%B4&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E7%BB%8F%E5%85%B8%E5%8A%A8%E4%BD%9C+%E5%98%9F%E5%98%B4&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=3c&1476432128601=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E5%AD%A4%E7%8B%AC&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E5%AD%A4%E7%8B%AC&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=1e&1476432211175=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+90%E5%90%8E&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+90%E5%90%8E&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=1e&1476432252392=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E6%A3%AE%E7%B3%BB&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E6%A3%AE%E7%B3%BB&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=1e&1476432318321=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E5%AD%97%E6%AF%8D&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E5%AD%97%E6%AF%8D&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=5a&1476432384197=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E5%8D%A1%E9%80%9A%E5%8A%A8%E6%BC%AB+%E7%BE%8E%E5%B0%91%E5%A5%B3%E6%88%98%E5%A3%AB&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E5%8D%A1%E9%80%9A%E5%8A%A8%E6%BC%AB+%E7%BE%8E%E5%B0%91%E5%A5%B3%E6%88%98%E5%A3%AB&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=78&1476432429562=",
            "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E6%80%A7%E6%84%9F&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&word=%E5%A4%B4%E5%83%8F+%E4%B8%8D%E5%90%8C%E9%A3%8E%E6%A0%BC+%E6%80%A7%E6%84%9F&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=&fr=&pn=". $pnArr[$rand] ."&rn=30&gsm=3c&1476432470204="
            );
    
        $index = array_rand($urlArr,1);
        $imgJsonData = file_get_contents($urlArr[$index]);
    
        //随机采样，让每次获取的图片尽量不是相同的，可以增加获取图片api去减少获取重复的图片的几率
        $imgJsonData = json_decode($imgJsonData,true);
        foreach ($imgJsonData['data'] as $key => $dataArr) {
            if ($dataArr['thumbURL']) {
                putImgToLocal($dataArr['thumbURL']);
            }
        }
    }
    
    function putImgToLocal($url) {
        //下载本地
        if (!is_dir('./userSysAvatorUrl')) {
            mkdir('./userSysAvatorUrl', 0777, true);
        }
    
        $hdrs = array(
          'http' =>array('header' => 
           "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" .
           "Accept-Language: zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3\r\n" .
           "Connection: keep-alive\r\n" .
           "Host: img0.imgtn.bdimg.com\r\n" .
           "Referer: http://image.baidu.com/search/index?tn=baiduimage&ct=201326592&lm=-1&cl=2&ie=gbk&word=%CD%B7%CF%F1&fr=ala&oriquery=%E5%A4%B4%E5%83%8F&ala=1&alatpl=portait&pos=0\r\n" .
           "User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0\r\n" .
           "X-Requested-With: XMLHttpRequest",
          ),
        );
        $context = stream_context_create($hdrs);
    
        list($msec, $sec) = explode(' ', microtime());
        $file_name = (float)sprintf('%.0f', (floatval($msec) + floatval($sec)) * 1000);
        $shullf = array('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z');
        $rand_key = array_rand($shullf,3);
        $file_name = $shullf[$rand_key[0]].$shullf[$rand_key[1]].$shullf[$rand_key[2]].'_'.$file_name. '.jpg';
    
        $imgBin = file_get_contents($url, 0, $context);
        file_put_contents('./userSysAvatorUrl/'.$file_name, $imgBin);
    }