python爬虫爬取腾讯网招聘信息

朱雀 2021-07-26 20:01 757阅读 0赞

## python爬虫爬取腾讯网招聘信息 ##

**话不多说，直接上代码！**

from bs4 import BeautifulSoup
    import urllib2
    import json    # 使用了json格式存储
    
    def tengxun(detail,num):
        url = 'https://hr.tencent.com/'
        # detail = 'position.php?&start=0#a'
        request = urllib2.Request(url + detail)
        response =urllib2.urlopen(request)
        resHtml = response.read()
        soup = BeautifulSoup(resHtml,'html.parser', from_encoding='utf-8')
        result = soup.select(".even")
        result += soup.select(".odd")
        # print len(result)
        # 处理页面
        items = []
        for node in result:
            item = { }
            # 职位名
            zname = node.select('td')[0].get_text()
            # 职位类别
            ztype = node.select('td')[1].get_text()
            # 人数
            znum = node.select('td')[2].get_text()
            # 地点
            zlocal = node.select('td')[3].get_text()
            # 发布时间
            ztime = node.select('td')[4].get_text()
            # 链接
            detailLink = node.select('td a')[0].attrs['href']
            # 获取工作职责、工作要求
            request1 = urllib2.Request(url + detailLink)
            response1 = urllib2.urlopen(request1)
            jobHtml = response1.read()
            soup1 = BeautifulSoup(jobHtml, 'html.parser', from_encoding='utf-8')
            # print len(soup1.select('ul.squareli'))
            # 工作职责
            jobRes = ''
            for li in soup1.select('ul.squareli')[0].select('li') :
                jobRes += li.get_text() + '\n'
            # 工作要求
            jobReq = ''
            for li in soup1.select('ul.squareli')[1].select('li') :
                jobReq += li.get_text() + '\n'
            # print jobReq
            # 将数据存入item中
            item['zname']=zname;
            item['detailLink'] = detailLink;
            item['ztype']=ztype
            item['znum'] = znum
            item['zlocal'] = zlocal
            item['ztime'] = ztime
            item['jobRes'] = jobRes
            item['jobReq'] = jobReq
            # 处理工作职责和工作要求
            items.append(item)
        origin = []
        print(len(items))
        # 以json格式输出到文件中
        # 禁用ascii编码，按utf-8编码
        output = open('tencent.json'+ str(num), 'w')
        for i in origin:
            items.append(i)
        line = json.dumps(items, ensure_ascii=False);
        # print line
        output.write(line.encode('utf-8'))
        output.close()
    
    for i in range(303):
        print("进行到第" + str(i) + "页")
        url = 'position.php?&start='+ str(i * 10) +'#a'
        tengxun(url, i)