# -*- coding: utf-8 -*-
"""
:epkg:`joblib` uses a module not documented
in the official :epkg:`Python` documentation:
`Python's undocumented ThreadPool <http://lucasb.eyer.be/snips/python-thread-pool.html>`_.
:githublink:`%|py|8`
"""
from multiprocessing.pool import ThreadPool
[docs]def pyparallel_mapper(fct, gen, threads=None):
"""
Applies function *fct* to a generator.
Relies on *ThreadPool*.
:param fct: function
:param gen: generator
:param threads: number of threads
:return: generator
If the number of threads is None,
it is replaced by ``os.cpu_count() or 1``
(see *multiprocessing.pool*).
.. exref::
:title: mapper
:tag: progfonc
.. runpython::
:showcode:
from sparkouille.fctmr.pyparallel_fctmr import pyparallel_mapper
res = pyparallel_mapper(lambda x: x + 1, [4, 5])
print(list(res))
Unfortunately, the parallelization is not following
the map/reduce concept in a sense that the function
generates an intermediate list and creates an iterator
on it.
:githublink:`%|py|40`
"""
pool = ThreadPool(processes=threads)
return iter(pool.map(fct, gen))