python动态抓取代理IP

本是古典何须时尚 2022-07-12 06:16 253阅读 0赞

python动态抓取代理IP

#!/usr/bin/python
    # -*- coding: UTF-8 -*-
    #print("Hello, World!");
    #python动态抓取代理IP：代理获取的相关代码，目前抓取了快代理、代理66、有代理、西刺代理、guobanjia这个五个网站的免费代理
    #author:zjcjava@163.com 2016-11-02
    import urllib.parse
    import urllib.request
    import time
    from lxml import etree
    
    #可写函数说明
    def printinfo( name, age ):
       "打印任何传入的字符串"
       print ("Name: ", name);
       print ("Age ", age);
       return;
     
    #调用printinfo函数
    printinfo( age=50, name="miki" );
    
    global url_path
    url_path = [3]
    
    def get_url(url):    #国内高匿代理的链接
        url_path [0] = url
        url_list = []
        if "xici" in url:
            for i in range(1,1000):
                url_new = url + str(i)
                url_list.append(url_new)
        if "66" in url:
            for i in range(1,2):
                url_new = url + str(i)+".html"
                url_list.append(url_new)
        print(url_new)
        return url_list
    
    def get_content(url):     # 获取网页内容
        user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0'
        headers = {'User-Agent': user_agent}
        req = urllib.request.Request(url=url, headers=headers)
        res = urllib.request.urlopen(req)
        content = res.read()
        return content #.decode('utf-8')
    
    
    
    def get_info(content,url):      # 提取网页信息 / ip 端口
        print("111111111111111111");
        print("sss:"+url)
        if "xici" in url:
            datas_ip = etree.HTML(content).xpath('//table[contains(@id,"ip_list")]/tr/td[2]/text()')
            datas_port = etree.HTML(content).xpath('//table[contains(@id,"ip_list")]/tr/td[3]/text()')
        if "66" in url:
            print(66);
            datas_ip = etree.HTML(content).xpath('//div[contains(@id,"main")]/div/div[1]/table/tr[position()>1]/td[1]/text()')
            datas_port = etree.HTML(content).xpath('//div[contains(@id,"main")]/div/div[1]/table/tr[position()>1]/td[2]/text()')
            #print('%s : %s'%(datas_ip,datas_port))
        
        with open("temp.txt", "w") as fd:
            for i in range(0,len(datas_ip)):
                out = u""
                out += u"" + datas_ip[i]
                out += u":" + datas_port[i]
                fd.write(out + u"\n")     # 所有ip和端口号写入data文件
    
    def verif_ip(ip,port):    # 验证ip有效性
        user_agent ='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0'
        headers = {'User-Agent':user_agent}
        proxy = {'http':'http://%s:%s'%(ip,port)}
        print(proxy)
    
        proxy_handler = urllib.request.ProxyHandler(proxy)
        opener = urllib.request.build_opener(proxy_handler)
        urllib.request.install_opener(opener)
    
        test_url = "https://www.baidu.com/"
        req = urllib.request.Request(url=test_url,headers=headers)
        time.sleep(6)
        try:
            res = urllib.request.urlopen(req)
            time.sleep(3)
            content = res.read()
            if content:
                print('that is ok')
                with open("data.txt", "a") as fd:       # 有效ip保存到data2文件夹
                    fd.write(ip + u":" + port)
                    #fd.write("\n")
            else:
                print('its not ok')
        except urllib.request.URLError as e:
            print(e.reason)
    
    # url_list =get_url(url="http://www.xicidaili.com/nn/");
    # for i in url_list:
    #         print(i)
    #url = 'http://www.xicidaili.com/nn/'
    if __name__ == "__main__":
        global url_path
        url = 'http://www.66ip.cn/'
        url_list = get_url(url)
        for i in url_list:
            print(i)
            content = get_content(i)
            time.sleep(3)
            get_info(content,url);
    
    
    file = open("temp.txt")
    while 1:
        lines = file.readlines(100000)
        if not lines:
            break
        for data in lines:
            print(data.split(u":")[0])
            #out=data.split(u":");
    		#print('%s : %s'%(data.split(u":")[0],data.split(u":")[0]))
            verif_ip(data.split(u":")[0],data.split(u":")[1])