python爬虫综合应用实例

缺乏、安全感 2023-02-22 08:24 20阅读 0赞

本文是一个爬虫的综合应用实例，使用了Selenium、用户身份登录、接口爬取、url跳转、excel保存数据等技术。

import time
    import json
    from selenium import webdriver
    from selenium.webdriver.support.wait import WebDriverWait
    from selenium.webdriver.common.keys import Keys
    from selenium.common.exceptions import StaleElementReferenceException
    import requests
    import urllib.parse
    import xlwt
    
    class Gjh():
    
        def __init__(self, index, name):
            self.categoryIndex = index
            self.categoryName = name
            options = webdriver.ChromeOptions()
            # 不加载图片,加快访问速度
            options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
            # 此步骤很重要，设置为开发者模式，防止被各大网站识别出来使用了Selenium
            options.add_experimental_option('excludeSwitches', ['enable-automation'])
            # 添加本地代理
            # options.add_argument("--proxy--server=127.0.0.1:8080")
            # 添加UA
            ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'
            options.add_argument('user-agent=' + ua)
            self.driver = webdriver.Chrome(options=options)
    
        def openPage(self):
            # 这里是一个假的url，需访问实际地址
            url = 'https://www.test.com/'
            self.driver.maximize_window()
            wait = WebDriverWait(self.driver, 2)
            self.driver.get(url)
            time.sleep(2)
    
            # time.sleep(2)
            user = self.driver.find_element_by_css_selector("div.userName input")
            user.send_keys(Keys.CONTROL, 'a')
            user.send_keys(Keys.DELETE)
            user.send_keys('13270538237')
            passwd = self.driver.find_element_by_css_selector("div.pass-fa input")
            passwd.send_keys(Keys.CONTROL, 'a')
            passwd.send_keys(Keys.DELETE)
            passwd.send_keys('booab20')
            # time.sleep(20)
            self.driver.find_element_by_css_selector('div.agreement input').click()
            time.sleep(2)
            self.driver.find_element_by_xpath('/html/body/div[1]/div/div[2]/div[2]/div/div[2]/div/div[3]/div[2]/button').click()
    
            time.sleep(2)
            self.driver.get("https://www.test.com/hello")
            time.sleep(2)
            categories = self.driver.find_elements_by_css_selector('div.left-nav a')
            # print(categories)
            csize = len(categories)
            # 输出有多少个行业
            print(csize)
            # 点击第self.categoryIndex个行业
            categories[self.categoryIndex].click()
            time.sleep(2)
    
            self.driver.find_elements_by_css_selector('ul.tabs-bar li')[0].click()
            time.sleep(2)
    
            company_list = self.getCompanyUrls()
    
            result = []
            try:
                self.getAllData(result, company_list)
            finally:
                # self.printData(result)
                self.saveExcel(result)
    
        def getCompanyUrls(self):
            # 从url中解析出categoryId
            current_url = self.driver.current_url
            print(current_url)
            params = urllib.parse.urlparse(current_url)
            qq = urllib.parse.parse_qs(params.query)
            print(qq)
            # categoryId 是个数组，但是只有一个值
            categoryId = qq['categoryId']
            print(categoryId)
    
            size = 15
            page = 0
            # 先取出全部的url放到company_list中，然后依次迭代爬取每个页面的数据
            company_list = []
            totalPage = self.getOnePageCompanyUrls(categoryId, page, size, company_list)
    
            if totalPage > 1:
                r = range(1, totalPage + 1)
                print(r)
                for i in r:
                    time.sleep(1)
                    self.getOnePageCompanyUrls(categoryId, i, size, company_list)
    
            llentgh = len(company_list)
            print("llentgh===" + str(llentgh))
            return company_list
    
        def getOnePageCompanyUrls(self, categoryIds, page, size, company_list):
            print("page num === " + str(page))
            categoryId = categoryIds[0]
            link = 'https://gateway.cantonfair.org.cn/Api/ESAPI/company/classify-v2?page=' + str(page) + '&size=' + str(size) + '&type=undefined'
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36',
                       'Content-Type': 'application/json;charset=UTF-8'
                       ,'Referer': 'https://ex.cantonfair.org.cn/mainsite/zh?categoryId=' + categoryId + '&_ga=2.102673793.1446854882.1593008478-19231745.1593008478'
                       ,'Origin': 'https://ex.cantonfair.org.cn'}
            categoryId = json.dumps(categoryIds)
            ccdata = '{"page":' + str(page) + ',"size":' + str(size) + ',"searchKeys":["companyNameCN","companyNameEN"],"isSearch":false,"selectAndMap":{"categoryId":' + categoryId + ',"boothAreaSearch":[],"boothNumber":[]},"orderModel":{"order":"asc","properties":[]},"selectOrMap":{"isPovertyAlleviation":[],"isFirstJoin":[],"isContinuousJoin":[],"isBrand":[],"exhibitorType":[],"productTrait":[],"isCfWinner":[],"tradeTypes":[],"isGreenAward":[],"isInvitationAward":[]},"searchValue":""}'
            print(ccdata)
            r = requests.post(link, headers=headers, data=ccdata)
            print(r.text)
            resultJson = json.loads(r.text)
            returnObj = resultJson.get('returnObj')
            print(returnObj)
    
            companies = returnObj.get('list')
            for company in companies:
                company_list.append(company['companyHrefCN'])
    
            totalPage = returnObj.get('page').get('totalPage')
            print("totalPage===" + str(totalPage))
            print("company_list size===" + str(len(company_list)))
            return totalPage
    
        def getAllData(self, result, comany_list):
            i = 0
            for exhibitorLink in comany_list:
                print("exhibitorLink==**********==" + str(i))
                # href = exhibitor.get_attribute('href')
                print(exhibitorLink)
                # 直接跳转到展商详情页
                self.driver.get(exhibitorLink + '/company')
                try:
                    self.getAttr(result)
                except :
                    # 如果出现异常则重试
                    print('try again, num is====' + str(i))
                    time.sleep(10)
                    try:
                        self.getAttr(result)
                    except :
                        # 如果出现异常再次重试
                        print('try reagain, num is====' + str(i))
                        time.sleep(50)
                        self.getAttr(result)
    
                i = i+1
    
        def getAttr(self, result):
            time.sleep(1)
    
            # 滚动到底部
            self.driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
            time.sleep(1)
            # 点击查看联系方式
            self.driver.find_element_by_css_selector('div.ex-60__contact-forbidden a.ex-60__contact-view').click()
            time.sleep(1)
    
            items = self.driver.find_elements_by_css_selector('div.ex-60__inner div.cell')
            one = {}
            for item in items:
                fields = item.find_elements_by_css_selector('div.cell-item')
                for field in fields:
                    label = field.find_element_by_css_selector('div.cell-label').text
                    value = field.find_element_by_css_selector('div.cell-value').text
                    # print(label + "====" + value)
                    one[label] = value
    
            result.append(one)
    
        def saveExcel(self, result):
            # 创建一个workbook 设置编码
            workbook = xlwt.Workbook(encoding='utf-8')
            # 创建一个worksheet
            worksheet = workbook.add_sheet(self.categoryName)
    
            # 写入excel, 参数对应 行, 列, 值
            rowIndex = 0
            # 第一行是标题
            worksheet.write(rowIndex, 0, label='企业名称')
            worksheet.write(rowIndex, 1, label='企业类型')
            worksheet.write(rowIndex, 2, label='成立年份')
            worksheet.write(rowIndex, 3, label='注册资本')
            worksheet.write(rowIndex, 4, label='企业规模')
            worksheet.write(rowIndex, 5, label='主要目标客户')
            worksheet.write(rowIndex, 6, label='主营展品')
            worksheet.write(rowIndex, 7, label='地址')
            worksheet.write(rowIndex, 8, label='所在地区')
            worksheet.write(rowIndex, 9, label='网址')
            worksheet.write(rowIndex, 10, label='业务联系人')
            worksheet.write(rowIndex, 11, label='邮箱')
            worksheet.write(rowIndex, 12, label='电话')
            worksheet.write(rowIndex, 13, label='手机')
            worksheet.write(rowIndex, 14, label='传真')
            worksheet.write(rowIndex, 15, label='邮编')
            rowIndex = rowIndex + 1
    
            for row in result:
                worksheet.write(rowIndex, 0, label=row.get('企业名称：'))
                worksheet.write(rowIndex, 1, label=row.get('企业类型：'))
                worksheet.write(rowIndex, 2, label=row.get('成立年份：'))
                worksheet.write(rowIndex, 3, label=row.get('注册资本：'))
                worksheet.write(rowIndex, 4, label=row.get('企业规模：'))
                worksheet.write(rowIndex, 5, label=row.get('主要目标客户：'))
                worksheet.write(rowIndex, 6, label=row.get('主营展品：'))
                worksheet.write(rowIndex, 7, label=row.get('地址：'))
                worksheet.write(rowIndex, 8, label=row.get('所在地区：'))
                worksheet.write(rowIndex, 9, label=row.get('网址：'))
                worksheet.write(rowIndex, 10, label=row.get('业务联系人'))
                worksheet.write(rowIndex, 11, label=row.get('邮箱'))
                worksheet.write(rowIndex, 12, label=row.get('电话'))
                worksheet.write(rowIndex, 13, label=row.get('手机'))
                worksheet.write(rowIndex, 14, label=row.get('传真'))
                worksheet.write(rowIndex, 15, label=row.get('邮编'))
                rowIndex = rowIndex + 1
    
            # 保存
            filePath = 'd:\\data_' + str(self.categoryIndex) + '.xls'
            workbook.save(filePath)
    
    if __name__ == '__main__':
        # 第11个行业分类
        g = Gjh(11, '纺织服装')
        g.openPage()

本文内容到此结束。

![20191103090939423.jpg][]![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70][]

[20191103090939423.jpg]: https://img-blog.csdnimg.cn/20191103090939423.jpg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20190321224208825.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2Jvd2VpMDI2,size_16,color_FFFFFF,t_70