Mercurial > cpython-lru_cache-weakref

annotate Demo/threads/find.py @ 54294:dba767bc3155 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
remove mention of old ctypes version
author Benjamin Peterson <benjamin@python.org>
date Sat, 23 May 2009 20:59:09 +0000
parents 71a294bd7950
children c048a09d06ff
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3874
dc5c11177339 Remove comment "SGI only"
Guido van Rossum <guido@python.org>
parents: 1633
diff changeset
1 # A parallelized "find(1)" using the thread module.
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3 # This demonstrates the use of a work queue and worker threads.
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
4 # It really does do more stats/sec when using multiple threads,
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
5 # although the improvement is only about 20-30 percent.
25863
e56b52e78bc9 Note lack of speedup. Remove Irix reference. Remove silly extra
Guido van Rossum <guido@python.org>
parents: 16075
diff changeset
6 # (That was 8 years ago. In 2002, on Linux, I can't measure
e56b52e78bc9 Note lack of speedup. Remove Irix reference. Remove silly extra
Guido van Rossum <guido@python.org>
parents: 16075
diff changeset
7 # a speedup. :-( )
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
8
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 # I'm too lazy to write a command line parser for the full find(1)
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 # command line syntax, so the predicate it searches for is wired-in,
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 # see function selector() below. (It currently searches for files with
25863
e56b52e78bc9 Note lack of speedup. Remove Irix reference. Remove silly extra
Guido van Rossum <guido@python.org>
parents: 16075
diff changeset
12 # world write permission.)
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
13
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14 # Usage: parfind.py [-w nworkers] [directory] ...
25863
e56b52e78bc9 Note lack of speedup. Remove Irix reference. Remove silly extra
Guido van Rossum <guido@python.org>
parents: 16075
diff changeset
15 # Default nworkers is 4
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
16
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
17
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
18 import sys
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
19 import getopt
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
20 import string
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
21 import time
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
22 import os
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
23 from stat import *
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
24 import thread
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
25
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
26
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
27 # Work queue class. Usage:
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28 # wq = WorkQ()
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29 # wq.addwork(func, (arg1, arg2, ...)) # one or more calls
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
30 # wq.run(nworkers)
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
31 # The work is done when wq.run() completes.
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
32 # The function calls executed by the workers may add more work.
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
33 # Don't use keyboard interrupts!
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
34
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
35 class WorkQ:
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
36
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
37 # Invariants:
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
38
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
39 # - busy and work are only modified when mutex is locked
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
40 # - len(work) is the number of jobs ready to be taken
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
41 # - busy is the number of jobs being done
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
42 # - todo is locked iff there is no work and somebody is busy
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
43
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
44 def __init__(self):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
45 self.mutex = thread.allocate()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
46 self.todo = thread.allocate()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
47 self.todo.acquire()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
48 self.work = []
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
49 self.busy = 0
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
50
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
51 def addwork(self, func, args):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
52 job = (func, args)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
53 self.mutex.acquire()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
54 self.work.append(job)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
55 self.mutex.release()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
56 if len(self.work) == 1:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
57 self.todo.release()
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
59 def _getwork(self):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
60 self.todo.acquire()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
61 self.mutex.acquire()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
62 if self.busy == 0 and len(self.work) == 0:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
63 self.mutex.release()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
64 self.todo.release()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
65 return None
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
66 job = self.work[0]
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
67 del self.work[0]
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
68 self.busy = self.busy + 1
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
69 self.mutex.release()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
70 if len(self.work) > 0:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
71 self.todo.release()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
72 return job
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
73
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
74 def _donework(self):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
75 self.mutex.acquire()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
76 self.busy = self.busy - 1
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
77 if self.busy == 0 and len(self.work) == 0:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
78 self.todo.release()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
79 self.mutex.release()
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
80
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
81 def _worker(self):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
82 time.sleep(0.00001) # Let other threads run
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
83 while 1:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
84 job = self._getwork()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
85 if not job:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
86 break
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
87 func, args = job
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
88 apply(func, args)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
89 self._donework()
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
90
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
91 def run(self, nworkers):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
92 if not self.work:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
93 return # Nothing to do
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
94 for i in range(nworkers-1):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
95 thread.start_new(self._worker, ())
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
96 self._worker()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
97 self.todo.acquire()
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
98
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
99
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
100 # Main program
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
101
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
102 def main():
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
103 nworkers = 4
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
104 opts, args = getopt.getopt(sys.argv[1:], '-w:')
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
105 for opt, arg in opts:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
106 if opt == '-w':
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
107 nworkers = string.atoi(arg)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
108 if not args:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
109 args = [os.curdir]
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
110
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
111 wq = WorkQ()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
112 for dir in args:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
113 wq.addwork(find, (dir, selector, wq))
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
114
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
115 t1 = time.time()
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
116 wq.run(nworkers)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
117 t2 = time.time()
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
118
31515
71a294bd7950 Replace backticks with repr() or "%r"
Walter Dörwald <walter@livinglogic.de>
parents: 25863
diff changeset
119 sys.stderr.write('Total time %r sec.\n' % (t2-t1))
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
120
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
121
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
122 # The predicate -- defines what files we look for.
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
123 # Feel free to change this to suit your purpose
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
124
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
125 def selector(dir, name, fullname, stat):
25863
e56b52e78bc9 Note lack of speedup. Remove Irix reference. Remove silly extra
Guido van Rossum <guido@python.org>
parents: 16075
diff changeset
126 # Look for world writable files that are not symlinks
e56b52e78bc9 Note lack of speedup. Remove Irix reference. Remove silly extra
Guido van Rossum <guido@python.org>
parents: 16075
diff changeset
127 return (stat[ST_MODE] & 0002) != 0 and not S_ISLNK(stat[ST_MODE])
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
128
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
129
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
130 # The find procedure -- calls wq.addwork() for subdirectories
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
131
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
132 def find(dir, pred, wq):
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
133 try:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
134 names = os.listdir(dir)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
135 except os.error, msg:
31515
71a294bd7950 Replace backticks with repr() or "%r"
Walter Dörwald <walter@livinglogic.de>
parents: 25863
diff changeset
136 print repr(dir), ':', msg
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
137 return
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
138 for name in names:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
139 if name not in (os.curdir, os.pardir):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
140 fullname = os.path.join(dir, name)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
141 try:
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
142 stat = os.lstat(fullname)
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
143 except os.error, msg:
31515
71a294bd7950 Replace backticks with repr() or "%r"
Walter Dörwald <walter@livinglogic.de>
parents: 25863
diff changeset
144 print repr(fullname), ':', msg
16075
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
145 continue
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
146 if pred(dir, name, fullname, stat):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
147 print fullname
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
148 if S_ISDIR(stat[ST_MODE]):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
149 if not os.path.ismount(fullname):
50bbe811b344 Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 7749
diff changeset
150 wq.addwork(find, (fullname, pred, wq))
1431
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
151
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
152
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
153 # Call the main program
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
154
d7168cc2be07 Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
155 main()