Mercurial > cpython-lru_cache-weakref

annotate Doc/includes/mp_workers.py @ 80661:94de77bd0b4b 2.7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #16194: document imp.load_dynamic problems when read several extension modules from the same binary object file.
author Andrew Svetlov <andrew.svetlov@gmail.com>
date Thu, 29 Nov 2012 19:51:28 +0200
parents 3c7eef187bea
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
1 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
2 # Simple example which uses a pool of workers to carry out some tasks.
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
3 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
4 # Notice that the results will probably not come out of the output
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
5 # queue in the same in the same order as the corresponding tasks were
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
6 # put on the input queue. If it is important to get the results back
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
7 # in the original order then consider using `Pool.map()` or
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
8 # `Pool.imap()` (which will save on the amount of code needed anyway).
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
9 #
50004
3c7eef187bea Merged revisions 67398,67423-67424,67432,67440-67441,67444-67445,67454,67457,67463 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48773
diff changeset
10 # Copyright (c) 2006-2008, R Oudkerk
3c7eef187bea Merged revisions 67398,67423-67424,67432,67440-67441,67444-67445,67454,67457,67463 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48773
diff changeset
11 # All rights reserved.
3c7eef187bea Merged revisions 67398,67423-67424,67432,67440-67441,67444-67445,67454,67457,67463 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48773
diff changeset
12 #
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
13
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
14 import time
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
15 import random
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
16
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
17 from multiprocessing import Process, Queue, current_process, freeze_support
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
18
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
19 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
20 # Function run by worker processes
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
21 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
22
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
23 def worker(input, output):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
24 for func, args in iter(input.get, 'STOP'):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
25 result = calculate(func, args)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
26 output.put(result)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
27
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
28 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
29 # Function used to calculate result
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
30 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
31
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
32 def calculate(func, args):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
33 result = func(*args)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
34 return '%s says that %s%s = %s' % \
48773
25f06a703f89 Merged revisions 65864 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 47595
diff changeset
35 (current_process().name, func.__name__, args, result)
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
36
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
37 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
38 # Functions referenced by tasks
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
39 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
40
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
41 def mul(a, b):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
42 time.sleep(0.5*random.random())
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
43 return a * b
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
44
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
45 def plus(a, b):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
46 time.sleep(0.5*random.random())
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
47 return a + b
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
48
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
49 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
50 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
51 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
52
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
53 def test():
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
54 NUMBER_OF_PROCESSES = 4
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
55 TASKS1 = [(mul, (i, 7)) for i in range(20)]
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
56 TASKS2 = [(plus, (i, 8)) for i in range(10)]
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
57
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
58 # Create queues
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
59 task_queue = Queue()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
60 done_queue = Queue()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
61
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
62 # Submit tasks
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
63 for task in TASKS1:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
64 task_queue.put(task)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
65
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
66 # Start worker processes
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
67 for i in range(NUMBER_OF_PROCESSES):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
68 Process(target=worker, args=(task_queue, done_queue)).start()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
69
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
70 # Get and print results
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
71 print 'Unordered results:'
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
72 for i in range(len(TASKS1)):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
73 print '\t', done_queue.get()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
74
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
75 # Add more tasks using `put()`
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
76 for task in TASKS2:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
77 task_queue.put(task)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
78
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
79 # Get and print some more results
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
80 for i in range(len(TASKS2)):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
81 print '\t', done_queue.get()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
82
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
83 # Tell child processes to stop
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
84 for i in range(NUMBER_OF_PROCESSES):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
85 task_queue.put('STOP')
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
86
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
87
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
88 if __name__ == '__main__':
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
89 freeze_support()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
90 test()