Python multiprocessing pool queue
 |
Наши проекты:
Журнал · Discuz!ML · Wiki · DRKB · Помощь проекту |
|
| ПРАВИЛА | FAQ | Помощь | Поиск | Участники | Календарь | Избранное | RSS |
| [3.145.65.134] |
|
|
Модераторы: RaD
Python multiprocessing pool queue
|
Сообщ.
#1
,
|
|
 |
  #!/usr/bin/env python3 import multiprocessing import logging import time #------------------------------------------------------------------------------- def logger(q, fname): while True: data = q.get() try: with open(fname, 'a', encoding='utf-8', errors='replace') as f: print(data, file=f) except Exception as e: print(e) q.task_done() #------------------------------------------------------------------------------- def worker(args): strings, qLog = args for s in strings: try: result = s.lower() # some payload qLog.put(result) except Exception: pass #------------------------------------------------------------------------------- def split_into_chunks(lst, chunk_size): return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)] #------------------------------------------------------------------------------- def main(): with open('input.txt', 'r') as f: input = [line.strip() for line in f] pool = multiprocessing.Pool(processes=20) manager = multiprocessing.Manager() qLog = manager.JoinableQueue() multiprocessing.Process(target=logger, args=(qLog, 'log.txt'), daemon=True).start() iterable = [(chunk, qLog) for chunk in split_into_chunks(input, 20)] start_time = time.time() for _ in pool.imap_unordered(worker, iterable): pass for q in (qLog, ): q.join() finish_time = time.time() print('Time elapsed:', int(finish_time - start_time), 'seconds.') #------------------------------------------------------------------------------- if __name__ == '__main__': main() |
0 пользователей читают эту тему (0 гостей и 0 скрытых пользователей)
0 пользователей:
[ Script execution time: 0,0346 ] [ 15 queries used ] [ Generated: 28.04.24, 11:22 GMT ]