Mercurial > cpython-lru_cache-weakref

annotate Lib/csv.py @ 70817:dd775b1561ba

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix markup.
author Georg Brandl <georg@python.org>
date Tue, 14 Jun 2011 21:09:55 +0200
parents dc245d681901
children 68ff68f9a0d5
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
2 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
3 csv.py - read/write/investigate CSV files
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
4 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
5
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
6 import re
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
7 from _csv import Error, __version__, writer, reader, register_dialect, \
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
8 unregister_dialect, get_dialect, list_dialects, \
34406
b93845ce4454 Rename csv.set_field_limit to csv.field_size_limit (since it both sets and
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34396
diff changeset
9 field_size_limit, \
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
10 QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
11 __doc__
34393
b6ec1b3b9fce Replace python-coded validation of csv dialect with a call to the C
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34340
diff changeset
12 from _csv import Dialect as _Dialect
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
13
41760
1a9cd76364e3 Change some uses of cStringIO.StringIO to io.StringIO.
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
14 from io import StringIO
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
15
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
16 __all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE",
48733
edb5a8593749 Merged revisions 65795 via svnmerge from
Brett Cannon <bcannon@gmail.com>
parents: 48670
diff changeset
17 "Error", "Dialect", "__doc__", "excel", "excel_tab",
edb5a8593749 Merged revisions 65795 via svnmerge from
Brett Cannon <bcannon@gmail.com>
parents: 48670
diff changeset
18 "field_size_limit", "reader", "writer",
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
19 "register_dialect", "get_dialect", "list_dialects", "Sniffer",
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
20 "unregister_dialect", "__version__", "DictReader", "DictWriter" ]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
21
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
22 class Dialect:
65655
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
23 """Describe a CSV dialect.
34325
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
24
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
25 This must be subclassed (see csv.excel). Valid attributes are:
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
26 delimiter, quotechar, escapechar, doublequote, skipinitialspace,
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
27 lineterminator, quoting.
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
28
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
29 """
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
30 _name = ""
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
31 _valid = False
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
32 # placeholders
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
33 delimiter = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
34 quotechar = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
35 escapechar = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
36 doublequote = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
37 skipinitialspace = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
38 lineterminator = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
39 quoting = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
40
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
41 def __init__(self):
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
42 if self.__class__ != Dialect:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
43 self._valid = True
34393
b6ec1b3b9fce Replace python-coded validation of csv dialect with a call to the C
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34340
diff changeset
44 self._validate()
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
45
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
46 def _validate(self):
34393
b6ec1b3b9fce Replace python-coded validation of csv dialect with a call to the C
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34340
diff changeset
47 try:
b6ec1b3b9fce Replace python-coded validation of csv dialect with a call to the C
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34340
diff changeset
48 _Dialect(self)
40587
6b18a095c2e7 SF patch 1631942 by Collin Winter:
Guido van Rossum <guido@python.org>
parents: 39627
diff changeset
49 except TypeError as e:
34393
b6ec1b3b9fce Replace python-coded validation of csv dialect with a call to the C
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34340
diff changeset
50 # We do this for compatibility with py2.3
b6ec1b3b9fce Replace python-coded validation of csv dialect with a call to the C
Andrew McNamara <andrewm@object-craft.com.au>
parents: 34340
diff changeset
51 raise Error(str(e))
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
52
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
53 class excel(Dialect):
34325
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
54 """Describe the usual properties of Excel-generated CSV files."""
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
55 delimiter = ','
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
56 quotechar = '"'
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
57 doublequote = True
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
58 skipinitialspace = False
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
59 lineterminator = '\r\n'
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
60 quoting = QUOTE_MINIMAL
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
61 register_dialect("excel", excel)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
62
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
63 class excel_tab(excel):
34325
ee808e08dd6f add a couple missing docstrings
Skip Montanaro <skip@pobox.com>
parents: 30632
diff changeset
64 """Describe the usual properties of Excel-generated TAB-delimited files."""
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
65 delimiter = '\t'
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
66 register_dialect("excel-tab", excel_tab)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
67
65655
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
68 class unix_dialect(Dialect):
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
69 """Describe the usual properties of Unix-generated CSV files."""
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
70 delimiter = ','
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
71 quotechar = '"'
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
72 doublequote = True
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
73 skipinitialspace = False
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
74 lineterminator = '\n'
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
75 quoting = QUOTE_ALL
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
76 register_dialect("unix", unix_dialect)
9e4a2809e55c #5975: add unix_dialect to csv module.
Georg Brandl <georg@python.org>
parents: 59081
diff changeset
77
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
78
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
79 class DictReader:
30632
f9dfdcb897bb Make the fieldnames argument optional in the DictReader. If self.fieldnames
Skip Montanaro <skip@pobox.com>
parents: 30356
diff changeset
80 def __init__(self, f, fieldnames=None, restkey=None, restval=None,
30356
7504812fdb7e **kwds arg was missing from __init__ for Dict{Reader,Writer} classes.
Skip Montanaro <skip@pobox.com>
parents: 30292
diff changeset
81 dialect="excel", *args, **kwds):
48670
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
82 self._fieldnames = fieldnames # list of keys for the dict
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
83 self.restkey = restkey # key to catch long rows
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
84 self.restval = restval # default value for short rows
30356
7504812fdb7e **kwds arg was missing from __init__ for Dict{Reader,Writer} classes.
Skip Montanaro <skip@pobox.com>
parents: 30292
diff changeset
85 self.reader = reader(f, dialect, *args, **kwds)
45996
376b5cb8a014 Merged revisions 61687-61688,61696,61700,61704-61705,61707-61709,61711-61712,61714-61716,61718-61722 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents: 43020
diff changeset
86 self.dialect = dialect
376b5cb8a014 Merged revisions 61687-61688,61696,61700,61704-61705,61707-61709,61711-61712,61714-61716,61718-61722 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents: 43020
diff changeset
87 self.line_num = 0
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
88
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
89 def __iter__(self):
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
90 return self
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
91
48670
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
92 @property
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
93 def fieldnames(self):
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
94 if self._fieldnames is None:
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
95 try:
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
96 self._fieldnames = next(self.reader)
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
97 except StopIteration:
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
98 pass
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
99 self.line_num = self.reader.line_num
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
100 return self._fieldnames
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
101
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
102 @fieldnames.setter
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
103 def fieldnames(self, value):
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
104 self._fieldnames = value
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
105
41501
0c5019afc55b PEP 3114: rename .next() to .__next__() and add next() builtin.
Georg Brandl <georg@python.org>
parents: 40809
diff changeset
106 def __next__(self):
48670
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
107 if self.line_num == 0:
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
108 # Used only for its side effect.
1a11717a154b Merged revisions 65605 via svnmerge from
Skip Montanaro <skip@pobox.com>
parents: 46548
diff changeset
109 self.fieldnames
41501
0c5019afc55b PEP 3114: rename .next() to .__next__() and add next() builtin.
Georg Brandl <georg@python.org>
parents: 40809
diff changeset
110 row = next(self.reader)
45996
376b5cb8a014 Merged revisions 61687-61688,61696,61700,61704-61705,61707-61709,61711-61712,61714-61716,61718-61722 via svnmerge from
Christian Heimes <christian@cheimes.de>
parents: 43020
diff changeset
111 self.line_num = self.reader.line_num
30632
f9dfdcb897bb Make the fieldnames argument optional in the DictReader. If self.fieldnames
Skip Montanaro <skip@pobox.com>
parents: 30356
diff changeset
112
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
113 # unlike the basic reader, we prefer not to return blanks,
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
114 # because we will typically wind up with a dict full of None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
115 # values
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
116 while row == []:
41501
0c5019afc55b PEP 3114: rename .next() to .__next__() and add next() builtin.
Georg Brandl <georg@python.org>
parents: 40809
diff changeset
117 row = next(self.reader)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
118 d = dict(zip(self.fieldnames, row))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
119 lf = len(self.fieldnames)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
120 lr = len(row)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
121 if lf < lr:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
122 d[self.restkey] = row[lf:]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
123 elif lf > lr:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
124 for key in self.fieldnames[lr:]:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
125 d[key] = self.restval
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
126 return d
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
127
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
128
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
129 class DictWriter:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
130 def __init__(self, f, fieldnames, restval="", extrasaction="raise",
30356
7504812fdb7e **kwds arg was missing from __init__ for Dict{Reader,Writer} classes.
Skip Montanaro <skip@pobox.com>
parents: 30292
diff changeset
131 dialect="excel", *args, **kwds):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
132 self.fieldnames = fieldnames # list of keys for the dict
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
133 self.restval = restval # for writing short dicts
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
134 if extrasaction.lower() not in ("raise", "ignore"):
43020
907a45df1295 Raise statement normalization in Lib/.
Collin Winter <collinw@gmail.com>
parents: 42142
diff changeset
135 raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'"
907a45df1295 Raise statement normalization in Lib/.
Collin Winter <collinw@gmail.com>
parents: 42142
diff changeset
136 % extrasaction)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
137 self.extrasaction = extrasaction
30356
7504812fdb7e **kwds arg was missing from __init__ for Dict{Reader,Writer} classes.
Skip Montanaro <skip@pobox.com>
parents: 30292
diff changeset
138 self.writer = writer(f, dialect, *args, **kwds)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
139
59081
77de7b1bcd59 Merged revisions 78384 via svnmerge from
R. David Murray <rdmurray@bitdance.com>
parents: 56368
diff changeset
140 def writeheader(self):
77de7b1bcd59 Merged revisions 78384 via svnmerge from
R. David Murray <rdmurray@bitdance.com>
parents: 56368
diff changeset
141 header = dict(zip(self.fieldnames, self.fieldnames))
77de7b1bcd59 Merged revisions 78384 via svnmerge from
R. David Murray <rdmurray@bitdance.com>
parents: 56368
diff changeset
142 self.writerow(header)
77de7b1bcd59 Merged revisions 78384 via svnmerge from
R. David Murray <rdmurray@bitdance.com>
parents: 56368
diff changeset
143
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
144 def _dict_to_list(self, rowdict):
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
145 if self.extrasaction == "raise":
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41501
diff changeset
146 wrong_fields = [k for k in rowdict if k not in self.fieldnames]
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41501
diff changeset
147 if wrong_fields:
43020
907a45df1295 Raise statement normalization in Lib/.
Collin Winter <collinw@gmail.com>
parents: 42142
diff changeset
148 raise ValueError("dict contains fields not in fieldnames: "
907a45df1295 Raise statement normalization in Lib/.
Collin Winter <collinw@gmail.com>
parents: 42142
diff changeset
149 + ", ".join(wrong_fields))
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
150 return [rowdict.get(key, self.restval) for key in self.fieldnames]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
151
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
152 def writerow(self, rowdict):
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
153 return self.writer.writerow(self._dict_to_list(rowdict))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
154
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
155 def writerows(self, rowdicts):
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
156 rows = []
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
157 for rowdict in rowdicts:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
158 rows.append(self._dict_to_list(rowdict))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
159 return self.writer.writerows(rows)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
160
29411
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
161 # Guard Sniffer's type checking against builds that exclude complex()
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
162 try:
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
163 complex
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
164 except NameError:
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
165 complex = float
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
166
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
167 class Sniffer:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
168 '''
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
169 "Sniffs" the format of a CSV file (i.e. delimiter, quotechar)
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
170 Returns a Dialect object.
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
171 '''
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
172 def __init__(self):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
173 # in case there is more than one possible delimiter
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
174 self.preferred = [',', '\t', ';', ' ', ':']
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
175
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
176
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
177 def sniff(self, sample, delimiters=None):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
178 """
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
179 Returns a dialect (or None) corresponding to the sample
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
180 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
181
56368
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
182 quotechar, doublequote, delimiter, skipinitialspace = \
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
183 self._guess_quote_and_delimiter(sample, delimiters)
36157
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
184 if not delimiter:
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
185 delimiter, skipinitialspace = self._guess_delimiter(sample,
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
186 delimiters)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
187
36157
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
188 if not delimiter:
43020
907a45df1295 Raise statement normalization in Lib/.
Collin Winter <collinw@gmail.com>
parents: 42142
diff changeset
189 raise Error("Could not determine delimiter")
36157
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
190
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
191 class dialect(Dialect):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
192 _name = "sniffed"
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
193 lineterminator = '\r\n'
28825
2fad548391de Attempt to deal with some obvious errors in the code. These were all
Fred Drake <fdrake@acm.org>
parents: 28804
diff changeset
194 quoting = QUOTE_MINIMAL
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
195 # escapechar = ''
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
196
56368
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
197 dialect.doublequote = doublequote
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
198 dialect.delimiter = delimiter
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
199 # _csv.reader won't accept a quotechar of ''
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
200 dialect.quotechar = quotechar or '"'
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
201 dialect.skipinitialspace = skipinitialspace
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
202
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
203 return dialect
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
204
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
205
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
206 def _guess_quote_and_delimiter(self, data, delimiters):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
207 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
208 Looks for text enclosed between two identical quotes
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
209 (the probable quotechar) which are preceded and followed
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
210 by the same character (the probable delimiter).
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
211 For example:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
212 ,'some text',
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
213 The quote with the most wins, same with the delimiter.
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
214 If there is no quotechar the delimiter can't be determined
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
215 this way.
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
216 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
217
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
218 matches = []
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
219 for restr in ('(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
220 '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?",
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
221 '(?P<delim>>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\n)', # ,".*?"
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
222 '(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space)
30292
79b1a9ec5223 use the long names for re compilation options; this makes it easier to
Fred Drake <fdrake@acm.org>
parents: 29422
diff changeset
223 regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
224 matches = regexp.findall(data)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
225 if matches:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
226 break
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
227
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
228 if not matches:
56368
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
229 # (quotechar, doublequote, delimiter, skipinitialspace)
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
230 return ('', False, None, 0)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
231 quotes = {}
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
232 delims = {}
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
233 spaces = 0
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
234 for m in matches:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
235 n = regexp.groupindex['quote'] - 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
236 key = m[n]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
237 if key:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
238 quotes[key] = quotes.get(key, 0) + 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
239 try:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
240 n = regexp.groupindex['delim'] - 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
241 key = m[n]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
242 except KeyError:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
243 continue
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
244 if key and (delimiters is None or key in delimiters):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
245 delims[key] = delims.get(key, 0) + 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
246 try:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
247 n = regexp.groupindex['space'] - 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
248 except KeyError:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
249 continue
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
250 if m[n]:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
251 spaces += 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
252
39627
e86c0b29b4a2 Kill reduce(). A coproduction of John Reese, Jacques Frechet, and Alex M.
Guido van Rossum <guido@python.org>
parents: 36157
diff changeset
253 quotechar = max(quotes, key=quotes.get)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
254
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
255 if delims:
39627
e86c0b29b4a2 Kill reduce(). A coproduction of John Reese, Jacques Frechet, and Alex M.
Guido van Rossum <guido@python.org>
parents: 36157
diff changeset
256 delim = max(delims, key=delims.get)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
257 skipinitialspace = delims[delim] == spaces
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
258 if delim == '\n': # most likely a file with a single column
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
259 delim = ''
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
260 else:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
261 # there is *no* delimiter, it's a single column of quoted data
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
262 delim = ''
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
263 skipinitialspace = 0
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
264
56368
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
265 # if we see an extra quote between delimiters, we've got a
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
266 # double quoted format
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
267 dq_regexp = re.compile(r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
268 {'delim':delim, 'quote':quotechar}, re.MULTILINE)
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
269
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
270
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
271
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
272 if dq_regexp.search(data):
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
273 doublequote = True
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
274 else:
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
275 doublequote = False
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
276
c9bd62efe6ce Merged revisions 75231 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48733
diff changeset
277 return (quotechar, doublequote, delim, skipinitialspace)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
278
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
279
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
280 def _guess_delimiter(self, data, delimiters):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
281 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
282 The delimiter /should/ occur the same number of times on
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
283 each row. However, due to malformed data, it may not. We don't want
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
284 an all or nothing approach, so we allow for small variations in this
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
285 number.
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
286 1) build a table of the frequency of each character on every line.
68585
bf94b6a73fc8 #11565: Fix several typos. Patch by Piotr Kasprzyk.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 56368
diff changeset
287 2) build a table of frequencies of this frequency (meta-frequency?),
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
288 e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows,
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
289 7 times in 2 rows'
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
290 3) use the mode of the meta-frequency to determine the /expected/
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
291 frequency for that character
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
292 4) find out how often the character actually meets that goal
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
293 5) the character that best meets its goal is the delimiter
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
294 For performance reasons, the data is evaluated in chunks, so it can
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
295 try and evaluate the smallest portion of the data possible, evaluating
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
296 additional chunks as necessary.
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
297 """
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
298
42142
a53e48be6df9 Merged revisions 56125-56153 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 41760
diff changeset
299 data = list(filter(None, data.split('\n')))
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
300
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
301 ascii = [chr(c) for c in range(127)] # 7-bit ASCII
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
302
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
303 # build frequency tables
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
304 chunkLength = min(10, len(data))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
305 iteration = 0
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
306 charFrequency = {}
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
307 modes = {}
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
308 delims = {}
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
309 start, end = 0, min(chunkLength, len(data))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
310 while start < len(data):
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
311 iteration += 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
312 for line in data[start:end]:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
313 for char in ascii:
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
314 metaFrequency = charFrequency.get(char, {})
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
315 # must count even if frequency is 0
36146
22dbf5818b0b Fix for problem with Sniffer class. If your delimiter is whitespace and the
Skip Montanaro <skip@pobox.com>
parents: 34406
diff changeset
316 freq = line.count(char)
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
317 # value is the mode
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
318 metaFrequency[freq] = metaFrequency.get(freq, 0) + 1
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
319 charFrequency[char] = metaFrequency
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
320
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
321 for char in charFrequency.keys():
40809
f340cb045bf9 - PEP 3106: dict.iterkeys(), .iteritems(), .itervalues() are now gone;
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
322 items = list(charFrequency[char].items())
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
323 if len(items) == 1 and items[0][0] == 0:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
324 continue
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
325 # get the mode of the frequencies
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
326 if len(items) > 1:
39627
e86c0b29b4a2 Kill reduce(). A coproduction of John Reese, Jacques Frechet, and Alex M.
Guido van Rossum <guido@python.org>
parents: 36157
diff changeset
327 modes[char] = max(items, key=lambda x: x[1])
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
328 # adjust the mode - subtract the sum of all
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
329 # other frequencies
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
330 items.remove(modes[char])
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
331 modes[char] = (modes[char][0], modes[char][1]
39627
e86c0b29b4a2 Kill reduce(). A coproduction of John Reese, Jacques Frechet, and Alex M.
Guido van Rossum <guido@python.org>
parents: 36157
diff changeset
332 - sum(item[1] for item in items))
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
333 else:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
334 modes[char] = items[0]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
335
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
336 # build a list of possible delimiters
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
337 modeList = modes.items()
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
338 total = float(chunkLength * iteration)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
339 # (rows of consistent data) / (number of rows) = 100%
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
340 consistency = 1.0
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
341 # minimum consistency threshold
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
342 threshold = 0.9
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
343 while len(delims) == 0 and consistency >= threshold:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
344 for k, v in modeList:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
345 if v[0] > 0 and v[1] > 0:
29169
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
346 if ((v[1]/total) >= consistency and
434ef174f478 * Correct Sniffer doc to correspond to the implementation.
Skip Montanaro <skip@pobox.com>
parents: 28828
diff changeset
347 (delimiters is None or k in delimiters)):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
348 delims[k] = v
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
349 consistency -= 0.01
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
350
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
351 if len(delims) == 1:
40809
f340cb045bf9 - PEP 3106: dict.iterkeys(), .iteritems(), .itervalues() are now gone;
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
352 delim = list(delims.keys())[0]
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
353 skipinitialspace = (data[0].count(delim) ==
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
354 data[0].count("%c " % delim))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
355 return (delim, skipinitialspace)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
356
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
357 # analyze another chunkLength lines
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
358 start = end
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
359 end += chunkLength
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
360
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
361 if not delims:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
362 return ('', 0)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
363
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
364 # if there's more than one, fall back to a 'preferred' list
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
365 if len(delims) > 1:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
366 for d in self.preferred:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
367 if d in delims.keys():
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
368 skipinitialspace = (data[0].count(d) ==
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
369 data[0].count("%c " % d))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
370 return (d, skipinitialspace)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
371
36157
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
372 # nothing else indicates a preference, pick the character that
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
373 # dominates(?)
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
374 items = [(v,k) for (k,v) in delims.items()]
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
375 items.sort()
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
376 delim = items[-1][1]
96eb2a5cfbef Fix a delimiter detection problem in sniffer. Sniffing "a|b|c\r\n" was
Skip Montanaro <skip@pobox.com>
parents: 36146
diff changeset
377
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
378 skipinitialspace = (data[0].count(delim) ==
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
379 data[0].count("%c " % delim))
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
380 return (delim, skipinitialspace)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
381
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
382
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
383 def has_header(self, sample):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
384 # Creates a dictionary of types of data in each column. If any
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
385 # column is of a single type (say, integers), *except* for the first
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
386 # row, then the first row is presumed to be labels. If the type
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
387 # can't be determined, it is assumed to be a string in which case
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
388 # the length of the string is the determining factor: if all of the
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
389 # rows except for the first are the same length, it's a header.
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
390 # Finally, a 'vote' is taken at the end for each column, adding or
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
391 # subtracting from the likelihood of the first row being a header.
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
392
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
393 rdr = reader(StringIO(sample), self.sniff(sample))
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
394
41501
0c5019afc55b PEP 3114: rename .next() to .__next__() and add next() builtin.
Georg Brandl <georg@python.org>
parents: 40809
diff changeset
395 header = next(rdr) # assume first row is header
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
396
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
397 columns = len(header)
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
398 columnTypes = {}
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
399 for i in range(columns): columnTypes[i] = None
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
400
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
401 checked = 0
28828
b37af21b77be rework Sniffer api significantly
Skip Montanaro <skip@pobox.com>
parents: 28825
diff changeset
402 for row in rdr:
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
403 # arbitrary number of rows to check, to keep it sane
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
404 if checked > 20:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
405 break
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
406 checked += 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
407
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
408 if len(row) != columns:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
409 continue # skip rows that have irregular number of columns
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
410
40809
f340cb045bf9 - PEP 3106: dict.iterkeys(), .iteritems(), .itervalues() are now gone;
Guido van Rossum <guido@python.org>
parents: 40644
diff changeset
411 for col in list(columnTypes.keys()):
29411
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
412
46548
85d4671565c8 Obvious simplification, now that "long" has disappeared.
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 45996
diff changeset
413 for thisType in [int, float, complex]:
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
414 try:
29411
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
415 thisType(row[col])
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
416 break
29420
f87fcd0870bf The multiple exception catch should be in a tuple.
Raymond Hettinger <python@rcn.com>
parents: 29411
diff changeset
417 except (ValueError, OverflowError):
29411
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
418 pass
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
419 else:
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
420 # fallback to length of string
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
421 thisType = len(row[col])
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
422
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
423 if thisType != columnTypes[col]:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
424 if columnTypes[col] is None: # add new column type
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
425 columnTypes[col] = thisType
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
426 else:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
427 # type is inconsistent, remove column from
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
428 # consideration
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
429 del columnTypes[col]
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
430
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
431 # finally, compare results against first row and "vote"
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
432 # on whether it's a header
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
433 hasHeader = 0
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
434 for col, colType in columnTypes.items():
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
435 if type(colType) == type(0): # it's a length
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
436 if len(header[col]) != colType:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
437 hasHeader += 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
438 else:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
439 hasHeader -= 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
440 else: # attempt typecast
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
441 try:
29411
bac2c0e7c20a SF Patch #744104: Remove eval() from csv
Raymond Hettinger <python@rcn.com>
parents: 29169
diff changeset
442 colType(header[col])
29422
cf8ae3b0dcdc One more multiple exception catch should be in a tuple.
Raymond Hettinger <python@rcn.com>
parents: 29420
diff changeset
443 except (ValueError, TypeError):
28804
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
444 hasHeader += 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
445 else:
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
446 hasHeader -= 1
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
447
0c950e3323ee cvs is going to be a module again
Skip Montanaro <skip@pobox.com>
parents:
diff changeset
448 return hasHeader > 0