Python多进程报错AttributeError: Can‘t pickle local object ‘run_pool.＜locals＞.run‘

朱雀 2022-08-30 05:28 391阅读 0赞

# 原代码 #

import multiprocessing
    import time
    from multiprocessing import Pool
    
    def run_pool():
        def run(k):
            print(k, multiprocessing.current_process().name)
            time.sleep(1)
    
        def pool():
            p = Pool(8)
            p.map(run, range(16))
            p.close()
            p.join()
        pool()
    
    if __name__ == '__main__':
        t0 = time.time()
        run_pool()
        t1 = time.time()
        print(t1 - t0)
    """ 报错信息： Traceback (most recent call last): File "C:/Project/auto_label_english/pool.py", line 19, in <module> run_pool() File "C:/Project/auto_label_english/pool.py", line 15, in run_pool pool() File "C:/Project/auto_label_english/pool.py", line 12, in pool p.map(run, range(16)) File "C:\Users\E14\Anaconda3\envs\auto_label_english\lib\multiprocessing\pool.py", line 266, in map return self._map_async(func, iterable, mapstar, chunksize).get() File "C:\Users\E14\Anaconda3\envs\auto_label_english\lib\multiprocessing\pool.py", line 644, in get raise self._value File "C:\Users\E14\Anaconda3\envs\auto_label_english\lib\multiprocessing\pool.py", line 424, in _handle_tasks put(task) File "C:\Users\E14\Anaconda3\envs\auto_label_english\lib\multiprocessing\connection.py", line 206, in send self._send_bytes(_ForkingPickler.dumps(obj)) File "C:\Users\E14\Anaconda3\envs\auto_label_english\lib\multiprocessing\reduction.py", line 51, in dumps cls(buf, protocol).dump(obj) AttributeError: Can't pickle local object 'run_pool.<locals>.run' """

# 报错原因 #

子进程能识别的符号是跟主进程一样的，我的执行函数是局部函数，进程无法识别。应该改成全局函数，或者类里面的静态方法。

# 修改后的代码 #

import multiprocessing
    import time
    from multiprocessing import Pool
    
    def run(k):
        print(k, multiprocessing.current_process().name)
        time.sleep(1)
    
    def run_pool():
        def pool():
            p = Pool(8)
            p.map(run, range(16))
            p.close()
            p.join()
        pool()
    
    if __name__ == '__main__':
        t0 = time.time()
        run_pool()
        t1 = time.time()
        print(t1 - t0)
        
    """ 结果输出： 0 SpawnPoolWorker-1 1 SpawnPoolWorker-3 2 SpawnPoolWorker-2 3 SpawnPoolWorker-5 4 SpawnPoolWorker-4 5 SpawnPoolWorker-6 6 SpawnPoolWorker-7 7 SpawnPoolWorker-8 8 SpawnPoolWorker-1 9 SpawnPoolWorker-2 10 SpawnPoolWorker-3 11 SpawnPoolWorker-6 12 SpawnPoolWorker-7 13 SpawnPoolWorker-4 14 SpawnPoolWorker-5 15 SpawnPoolWorker-8 2.453700542449951 """