python 多处理无法 pickle <type 'function'>

Question

python2.7 在 windows |我将 mysql 连接添加到 class 并使用多处理，引发错误。

self.ispop 和 self.match_var return 字典

sprawn_self_calcu() 和 unwrap_self_f() 是 Map_class

函数的代理

Map_class的功能需要自变量

我的代码是这样的：

 from analysis_conf.pop_config import pop_config
 import datetime
 import multiprocessing
 from functools import  partial
 from sqlalchemy import create_engine
 from multiprocessing import Pool as threadpool

 def sprawn_self_calcu(arg, **kwarg):
      return Map.mapCin(*arg, **kwarg)

 def unwrap_self_f(arg, **kwarg):
      return Map.mappalg(*arg, **kwarg)
 partial_unwrap = partial(unwrap_self_f)
 partial_sprawn = partial(sprawn_self_calcu)

 class Map:
    def __init__(self):
        self.ispop = pop_config()
        self.match_var = self.ispop.pop_match_var()
    def CreateSqlalchemyEngine(self,config):
        sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'],
                                                         config['ipaddr'],config['port'],config['dbname']
                                                         )
        return create_engine(sigma,pool_recycle=10,pool_timeout=10800)

    def Mapping(self,conSet):
        self.baseCon = conSet
        self.mappalg()

        Time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    def IUCMapping(self,i,con):
        print i
        print self.conf
        l = con.execute('show tables;')

    def mappalg(self):
        mt_val = [1,2,3,4,5]
        pool = threadpool(4)
        result = pool.map(partial_sprawn,zip([self]*5,mt_val))
    # result = pool.map(partial_sprawn,zip([self]*mtlen,mt_val))
        pool.close()
        pool.join()
        return True

    def mapCin(self,i):
        pid_val = multiprocessing.current_process().pid%4
        con = self.baseCon[pid_val]
        print i
        self.IUCMapping(i,con)
        return True

 class Create_MultiCon:
    def __init__(self):
        self.adapter = pop_config()
        self.conf  = self.adapter.pop_baseDB()
        self.match_var = self.adapter.pop_match_var()
    def CreateSqlalchemyEngine(self,config):
        sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'],
                                                         config['ipaddr'],config['port'],config['dbname']
                                                         )
        return create_engine(sigma,pool_recycle=10,pool_timeout=10800)
def RdictXcon(self,x):
        t = {}
        engine = self.CreateSqlalchemyEngine(self.conf)
        for i in xrange(x):
            t[i] = engine.connect()
        return t

if __name__ == '__main__':
    l = Create_MultiCon()
    conSet = l.RdictXcon(4)
    ScMap =  Map()
    ScMap.Mapping(conSet)

错误：

    result = pool.map(partial_sprawn,zip([self]*5,mt_val))
  File "C:\Python27\lib\multiprocessing\pool.py", line 251, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 567, in get
    raise self._value
cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed

如何解决错误

Answer 1

Python 的 multiprocessing 模块无法处理无法 pickled 的 functions/methods，这意味着您无法轻松使用 class 或实例方法.我建议使用 multiprocess，它使用 dill 进行序列化而不是 pickle，并且可以处理 class 或实例方法。

据我所知，该界面与 multiprocessing 中使用的界面完全相同，因此您可以将其用作直接替代品。

另见

python 多处理无法 pickle <type 'function'>

python multiprocessing Can't pickle <type 'function'>

python

pickle

multiprocessing