【多进程并行版本】爬取链家二手房前100页标签，进行统计

灰太狼 2022-05-26 00:27 131阅读 0赞

## 前言 ##

python在进程间通信会比较麻烦~  
要注意，这里我们想到的时候多个进程一起维护一个字典。  
所以，我加了锁。这里还是存在一个可能的改进方法，就是考察你的锁加在哪个位置来判断。  
[旧的单进程版本][Link 1]

## 代码一： ##

用时: 10.798075914382935秒  
看了三千个网页的样子。还算不错的啦。但是还是想要跟好的优化~

import requests
    from bs4 import BeautifulSoup
    import re
    from multiprocessing import Manager, Lock, Process
    
    
    def getUrl(url='https://gz.lianjia.com/ershoufang/pg{}/'):
        for i in range(100):
            yield url.format(i + 1)
    
    
    def findAInUrl(url, lock, tag):
        res = requests.get(url)
        soup = BeautifulSoup(res.text, 'lxml')
        lis = soup.find_all('li', attrs={
       'class': "clear"})
        for li in lis:
            info = li.find('div', attrs={
       'class': 'info clear'})
            tags = info.find('div', attrs={
       'class': 'tag'})
            for t in tags:
                ans = re.findall('(?<=\>).+(?=\<)', str(t))
                if '距离' in ans[0]:
                    ans[0] = '地铁'
                lock.acquire()
                if ans[0] not in tag:
                    tag[ans[0]] = 1
                else:
                    tag[ans[0]] += 1
                lock.release()
    
    
    if __name__ == '__main__':
        tag = Manager().dict()
        pList = []
        lock = Lock()
        for url in getUrl():
            p = Process(target=findAInUrl, args=(url, lock, tag))
            pList.append(p)
            p.start()
        for p in pList:
            p.join()
        print(tag)

## 代码二 ##

用时：11.187129020690918秒

import requests
    from bs4 import BeautifulSoup
    import re
    from multiprocessing import Manager, Lock, Process
    import time
    
    def getUrl(url='https://gz.lianjia.com/ershoufang/pg{}/'):
        for i in range(100):
            yield url.format(i + 1)
    
    
    def findAInUrl(url, lock, tag):
        res = requests.get(url)
        soup = BeautifulSoup(res.text, 'lxml')
        lis = soup.find_all('li', attrs={
       'class': "clear"})
        for li in lis:
            info = li.find('div', attrs={
       'class': 'info clear'})
            tags = info.find('div', attrs={
       'class': 'tag'})
            lock.acquire()
            for t in tags:
                ans = re.findall('(?<=\>).+(?=\<)', str(t))
                if '距离' in ans[0]:
                    ans[0] = '地铁'
                if ans[0] not in tag:
                    tag[ans[0]] = 1
                else:
                    tag[ans[0]] += 1
            lock.release()
    
    
    if __name__ == '__main__':
        st = time.time()
        tag = Manager().dict()
        pList = []
        lock = Lock()
        for url in getUrl():
            p = Process(target=findAInUrl, args=(url, lock, tag))
            pList.append(p)
            p.start()
        for p in pList:
            p.join()
        print(tag)
        et = time.time()
        print(et- st)

## 代码三 ##

用时：11.184273481369019秒

import requests
    from bs4 import BeautifulSoup
    import re
    from multiprocessing import Manager, Lock, Process
    import time
    
    def getUrl(url='https://gz.lianjia.com/ershoufang/pg{}/'):
        for i in range(100):
            yield url.format(i + 1)
    
    
    def findAInUrl(url, lock, tag):
        res = requests.get(url)
        soup = BeautifulSoup(res.text, 'lxml')
        lis = soup.find_all('li', attrs={
       'class': "clear"})
        lock.acquire()
        for li in lis:
            info = li.find('div', attrs={
       'class': 'info clear'})
            tags = info.find('div', attrs={
       'class': 'tag'})
            for t in tags:
                ans = re.findall('(?<=\>).+(?=\<)', str(t))
                if '距离' in ans[0]:
                    ans[0] = '地铁'
                if ans[0] not in tag:
                    tag[ans[0]] = 1
                else:
                    tag[ans[0]] += 1
        lock.release()
    
    
    if __name__ == '__main__':
        st = time.time()
        tag = Manager().dict()
        pList = []
        lock = Lock()
        for url in getUrl():
            p = Process(target=findAInUrl, args=(url, lock, tag))
            pList.append(p)
            p.start()
        for p in pList:
            p.join()
        print(tag)
        et = time.time()
        print(et- st)

--------------------

## 前三个版本的代码区别在于锁的位置的不同，导致了时间的不一样。 ##

[Link 1]: https://blog.csdn.net/a19990412/article/details/79810847