Mercurial > cpython-lru_cache-weakref
annotate Demo/parser/example.py @ 80661:94de77bd0b4b 2.7
Issue #16194: document imp.load_dynamic problems
when read several extension modules from the same binary object file.
| author | Andrew Svetlov <andrew.svetlov@gmail.com> |
|---|---|
| date | Thu, 29 Nov 2012 19:51:28 +0200 |
| parents | bdae61aeba55 |
| children | 7d16b42fcdbd |
| rev | line source |
|---|---|
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
1 """Simple code to extract class & function docstrings from a module. |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
2 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
3 This code is used as an example in the library reference manual in the |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
4 section on using the parser module. Refer to the manual for a thorough |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
5 discussion of the operation of this code. |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
6 """ |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
7 |
|
11488
bdae61aeba55
Moved all the imports to the top.
Fred Drake <fdrake@acm.org>
parents:
9108
diff
changeset
|
8 import os |
|
bdae61aeba55
Moved all the imports to the top.
Fred Drake <fdrake@acm.org>
parents:
9108
diff
changeset
|
9 import parser |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
10 import symbol |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
11 import token |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
12 import types |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
13 |
|
11488
bdae61aeba55
Moved all the imports to the top.
Fred Drake <fdrake@acm.org>
parents:
9108
diff
changeset
|
14 from types import ListType, TupleType |
|
bdae61aeba55
Moved all the imports to the top.
Fred Drake <fdrake@acm.org>
parents:
9108
diff
changeset
|
15 |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
16 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
17 def get_docs(fileName): |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
18 """Retrieve information from the parse tree of a source file. |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
19 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
20 fileName |
| 9108 | 21 Name of the file to read Python source code from. |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
22 """ |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
23 source = open(fileName).read() |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
24 basename = os.path.basename(os.path.splitext(fileName)[0]) |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
25 ast = parser.suite(source) |
|
11488
bdae61aeba55
Moved all the imports to the top.
Fred Drake <fdrake@acm.org>
parents:
9108
diff
changeset
|
26 return ModuleInfo(ast.totuple(), basename) |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
27 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
28 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
29 class SuiteInfoBase: |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
30 _docstring = '' |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
31 _name = '' |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
32 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
33 def __init__(self, tree = None): |
| 9108 | 34 self._class_info = {} |
| 35 self._function_info = {} | |
| 36 if tree: | |
| 37 self._extract_info(tree) | |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
38 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
39 def _extract_info(self, tree): |
| 9108 | 40 # extract docstring |
| 41 if len(tree) == 2: | |
| 42 found, vars = match(DOCSTRING_STMT_PATTERN[1], tree[1]) | |
| 43 else: | |
| 44 found, vars = match(DOCSTRING_STMT_PATTERN, tree[3]) | |
| 45 if found: | |
| 46 self._docstring = eval(vars['docstring']) | |
| 47 # discover inner definitions | |
| 48 for node in tree[1:]: | |
| 49 found, vars = match(COMPOUND_STMT_PATTERN, node) | |
| 50 if found: | |
| 51 cstmt = vars['compound'] | |
| 52 if cstmt[0] == symbol.funcdef: | |
| 53 name = cstmt[2][1] | |
| 54 self._function_info[name] = FunctionInfo(cstmt) | |
| 55 elif cstmt[0] == symbol.classdef: | |
| 56 name = cstmt[2][1] | |
| 57 self._class_info[name] = ClassInfo(cstmt) | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
58 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
59 def get_docstring(self): |
| 9108 | 60 return self._docstring |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
61 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
62 def get_name(self): |
| 9108 | 63 return self._name |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
64 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
65 def get_class_names(self): |
| 9108 | 66 return self._class_info.keys() |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
67 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
68 def get_class_info(self, name): |
| 9108 | 69 return self._class_info[name] |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
70 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
71 def __getitem__(self, name): |
| 9108 | 72 try: |
| 73 return self._class_info[name] | |
| 74 except KeyError: | |
| 75 return self._function_info[name] | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
76 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
77 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
78 class SuiteFuncInfo: |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
79 # Mixin class providing access to function names and info. |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
80 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
81 def get_function_names(self): |
| 9108 | 82 return self._function_info.keys() |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
83 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
84 def get_function_info(self, name): |
| 9108 | 85 return self._function_info[name] |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
86 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
87 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
88 class FunctionInfo(SuiteInfoBase, SuiteFuncInfo): |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
89 def __init__(self, tree = None): |
| 9108 | 90 self._name = tree[2][1] |
| 91 SuiteInfoBase.__init__(self, tree and tree[-1] or None) | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
92 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
93 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
94 class ClassInfo(SuiteInfoBase): |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
95 def __init__(self, tree = None): |
| 9108 | 96 self._name = tree[2][1] |
| 97 SuiteInfoBase.__init__(self, tree and tree[-1] or None) | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
98 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
99 def get_method_names(self): |
| 9108 | 100 return self._function_info.keys() |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
101 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
102 def get_method_info(self, name): |
| 9108 | 103 return self._function_info[name] |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
104 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
105 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
106 class ModuleInfo(SuiteInfoBase, SuiteFuncInfo): |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
107 def __init__(self, tree = None, name = "<string>"): |
| 9108 | 108 self._name = name |
| 109 SuiteInfoBase.__init__(self, tree) | |
| 110 if tree: | |
| 111 found, vars = match(DOCSTRING_STMT_PATTERN, tree[1]) | |
| 112 if found: | |
| 113 self._docstring = vars["docstring"] | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
114 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
115 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
116 def match(pattern, data, vars=None): |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
117 """Match `data' to `pattern', with variable extraction. |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
118 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
119 pattern |
| 9108 | 120 Pattern to match against, possibly containing variables. |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
121 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
122 data |
| 9108 | 123 Data to be checked and against which variables are extracted. |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
124 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
125 vars |
| 9108 | 126 Dictionary of variables which have already been found. If not |
| 127 provided, an empty dictionary is created. | |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
128 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
129 The `pattern' value may contain variables of the form ['varname'] which |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
130 are allowed to match anything. The value that is matched is returned as |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
131 part of a dictionary which maps 'varname' to the matched value. 'varname' |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
132 is not required to be a string object, but using strings makes patterns |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
133 and the code which uses them more readable. |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
134 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
135 This function returns two values: a boolean indicating whether a match |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
136 was found and a dictionary mapping variable names to their associated |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
137 values. |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
138 """ |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
139 if vars is None: |
| 9108 | 140 vars = {} |
| 141 if type(pattern) is ListType: # 'variables' are ['varname'] | |
| 142 vars[pattern[0]] = data | |
| 143 return 1, vars | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
144 if type(pattern) is not TupleType: |
| 9108 | 145 return (pattern == data), vars |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
146 if len(data) != len(pattern): |
| 9108 | 147 return 0, vars |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
148 for pattern, data in map(None, pattern, data): |
| 9108 | 149 same, vars = match(pattern, data, vars) |
| 150 if not same: | |
| 151 break | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
152 return same, vars |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
153 |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
154 |
|
4132
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
155 # This pattern identifies compound statements, allowing them to be readily |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
156 # differentiated from simple statements. |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
157 # |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
158 COMPOUND_STMT_PATTERN = ( |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
159 symbol.stmt, |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
160 (symbol.compound_stmt, ['compound']) |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
161 ) |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
162 |
|
c18d52e06238
Final set of changes by Fred before 1.4beta3
Guido van Rossum <guido@python.org>
parents:
4088
diff
changeset
|
163 |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
164 # This pattern will match a 'stmt' node which *might* represent a docstring; |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
165 # docstrings require that the statement which provides the docstring be the |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
166 # first statement in the class or function, which this pattern does not check. |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
167 # |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
168 DOCSTRING_STMT_PATTERN = ( |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
169 symbol.stmt, |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
170 (symbol.simple_stmt, |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
171 (symbol.small_stmt, |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
172 (symbol.expr_stmt, |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
173 (symbol.testlist, |
| 9108 | 174 (symbol.test, |
| 175 (symbol.and_test, | |
| 176 (symbol.not_test, | |
| 177 (symbol.comparison, | |
| 178 (symbol.expr, | |
| 179 (symbol.xor_expr, | |
| 180 (symbol.and_expr, | |
| 181 (symbol.shift_expr, | |
| 182 (symbol.arith_expr, | |
| 183 (symbol.term, | |
| 184 (symbol.factor, | |
| 185 (symbol.power, | |
| 186 (symbol.atom, | |
| 187 (token.STRING, ['docstring']) | |
| 188 )))))))))))))))), | |
|
4088
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
189 (token.NEWLINE, '') |
|
f81d78547f28
Demos for Fred's parser module
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
190 )) |