Mercurial > cpython-lru_cache-weakref

annotate Tools/scripts/pdeps.py @ 87690:66c1c9f32567 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add use_weakref option to functools.lru_cache The use_weakref option makes lru_cache create weak references to the function arguments when creating the cache key, which prevents them from staying in the memory forever even after all other references to them are gone. This is a proof of concept for http://bugs.python.org/issue19859
author Radomir Dopieralski <devel@sheep.art.pl>
date Tue, 03 Dec 2013 16:02:48 +0100
parents e9b8115c5b25
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
59522
fcfe324663a6 convert shebang lines: python -> python3
Benjamin Peterson <benjamin@python.org>
parents: 47100
diff changeset
1 #! /usr/bin/env python3
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
2
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
3 # pdeps
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
4 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
5 # Find dependencies between a bunch of Python modules.
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
6 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 # Usage:
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
8 # pdeps file1.py file2.py ...
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
10 # Output:
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 # Four tables separated by lines like '--- Closure ---':
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
12 # 1) Direct dependencies, listing which module imports which other modules
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 # 2) The inverse of (1)
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
14 # 3) Indirect dependencies, or the closure of the above
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 # 4) The inverse of (3)
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
16 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
17 # To do:
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
18 # - command line options to select output type
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
19 # - option to automatically scan the Python library for referenced modules
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
20 # - option to limit output to particular modules
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
21
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
22
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
23 import sys
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 33074
diff changeset
24 import re
1105
6f2a6efd019e Adapt to modern times...
Guido van Rossum <guido@python.org>
parents: 559
diff changeset
25 import os
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
26
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
27
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
28 # Main program
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
29 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
30 def main():
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
31 args = sys.argv[1:]
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
32 if not args:
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
33 print('usage: pdeps file.py file.py ...')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
34 return 2
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
35 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
36 table = {}
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
37 for arg in args:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
38 process(arg, table)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
39 #
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
40 print('--- Uses ---')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
41 printresults(table)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
42 #
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
43 print('--- Used By ---')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
44 inv = inverse(table)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
45 printresults(inv)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
46 #
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
47 print('--- Closure of Uses ---')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
48 reach = closure(table)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
49 printresults(reach)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
50 #
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
51 print('--- Closure of Used By ---')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
52 invreach = inverse(reach)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
53 printresults(invreach)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
54 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
55 return 0
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
56
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
57
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 # Compiled regular expressions to search for import statements
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 #
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 33074
diff changeset
60 m_import = re.compile('^[ \t]*from[ \t]+([^ \t]+)[ \t]+')
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 33074
diff changeset
61 m_from = re.compile('^[ \t]*import[ \t]+([^#]+)')
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
62
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
63
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 # Collect data from one file
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
66 def process(filename, table):
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
67 fp = open(filename, 'r')
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
68 mod = os.path.basename(filename)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
69 if mod[-3:] == '.py':
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
70 mod = mod[:-3]
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
71 table[mod] = list = []
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
72 while 1:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
73 line = fp.readline()
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
74 if not line: break
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
75 while line[-1:] == '\\':
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
76 nextline = fp.readline()
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
77 if not nextline: break
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
78 line = line[:-1] + nextline
76134
e9b8115c5b25 #14492: fix some bugs in Tools/scripts/pdeps.py.
R David Murray <rdmurray@bitdance.com>
parents: 59522
diff changeset
79 m_found = m_import.match(line) or m_from.match(line)
e9b8115c5b25 #14492: fix some bugs in Tools/scripts/pdeps.py.
R David Murray <rdmurray@bitdance.com>
parents: 59522
diff changeset
80 if m_found:
e9b8115c5b25 #14492: fix some bugs in Tools/scripts/pdeps.py.
R David Murray <rdmurray@bitdance.com>
parents: 59522
diff changeset
81 (a, b), (a1, b1) = m_found.regs[:2]
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
82 else: continue
25219
8a7a2f1f88f0 Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents: 15930
diff changeset
83 words = line[a1:b1].split(',')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
84 # print '#', line, words
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
85 for word in words:
25219
8a7a2f1f88f0 Apply diff2.txt from SF patch http://www.python.org/sf/572113
Walter Dörwald <walter@livinglogic.de>
parents: 15930
diff changeset
86 word = word.strip()
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
87 if word not in list:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
88 list.append(word)
76134
e9b8115c5b25 #14492: fix some bugs in Tools/scripts/pdeps.py.
R David Murray <rdmurray@bitdance.com>
parents: 59522
diff changeset
89 fp.close()
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
90
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
91
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
92 # Compute closure (this is in fact totally general)
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
94 def closure(table):
47100
893a4dc17a91 Ran 2to3 over scripts directory.
Georg Brandl <georg@python.org>
parents: 42427
diff changeset
95 modules = list(table.keys())
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
96 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
97 # Initialize reach with a copy of table
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
98 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
99 reach = {}
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
100 for mod in modules:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
101 reach[mod] = table[mod][:]
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
102 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
103 # Iterate until no more change
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
104 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
105 change = 1
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
106 while change:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
107 change = 0
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
108 for mod in modules:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
109 for mo in reach[mod]:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
110 if mo in modules:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
111 for m in reach[mo]:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
112 if m not in reach[mod]:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
113 reach[mod].append(m)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
114 change = 1
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
115 #
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
116 return reach
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
117
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
118
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 # Invert a table (this is again totally general).
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
120 # All keys of the original table are made keys of the inverse,
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
121 # so there may be empty lists in the inverse.
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
122 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
123 def inverse(table):
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
124 inv = {}
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
125 for key in table.keys():
76134
e9b8115c5b25 #14492: fix some bugs in Tools/scripts/pdeps.py.
R David Murray <rdmurray@bitdance.com>
parents: 59522
diff changeset
126 if key not in inv:
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
127 inv[key] = []
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
128 for item in table[key]:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
129 store(inv, item, key)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
130 return inv
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
131
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
132
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
133 # Store "item" in "dict" under "key".
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
134 # The dictionary maps keys to lists of items.
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
135 # If there is no list for the key yet, it is created.
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
136 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
137 def store(dict, key, item):
47100
893a4dc17a91 Ran 2to3 over scripts directory.
Georg Brandl <georg@python.org>
parents: 42427
diff changeset
138 if key in dict:
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
139 dict[key].append(item)
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
140 else:
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
141 dict[key] = [item]
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
142
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
143
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
144 # Tabulate results neatly
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
145 #
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
146 def printresults(table):
47100
893a4dc17a91 Ran 2to3 over scripts directory.
Georg Brandl <georg@python.org>
parents: 42427
diff changeset
147 modules = sorted(table.keys())
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
148 maxlen = 0
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
149 for mod in modules: maxlen = max(maxlen, len(mod))
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
150 for mod in modules:
47100
893a4dc17a91 Ran 2to3 over scripts directory.
Georg Brandl <georg@python.org>
parents: 42427
diff changeset
151 list = sorted(table[mod])
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
152 print(mod.ljust(maxlen), ':', end=' ')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
153 if mod in list:
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
154 print('(*)', end=' ')
15930
14a27acc755a Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 4573
diff changeset
155 for ref in list:
42427
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
156 print(ref, end=' ')
46625e6122e9 Convert print statements to function calls in Tools/.
Collin Winter <collinw@gmail.com>
parents: 37879
diff changeset
157 print()
274
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
158
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
159
12a64115a4ee Initial revision
Guido van Rossum <guido@python.org>
parents:
diff changeset
160 # Call main and honor exit status
33074
981fff7363b9 [Patch #1005491 ] use __name__ == '__main__' in scripts
Andrew M. Kuchling <amk@amk.ca>
parents: 25219
diff changeset
161 if __name__ == '__main__':
981fff7363b9 [Patch #1005491 ] use __name__ == '__main__' in scripts
Andrew M. Kuchling <amk@amk.ca>
parents: 25219
diff changeset
162 try:
981fff7363b9 [Patch #1005491 ] use __name__ == '__main__' in scripts
Andrew M. Kuchling <amk@amk.ca>
parents: 25219
diff changeset
163 sys.exit(main())
981fff7363b9 [Patch #1005491 ] use __name__ == '__main__' in scripts
Andrew M. Kuchling <amk@amk.ca>
parents: 25219
diff changeset
164 except KeyboardInterrupt:
981fff7363b9 [Patch #1005491 ] use __name__ == '__main__' in scripts
Andrew M. Kuchling <amk@amk.ca>
parents: 25219
diff changeset
165 sys.exit(1)