Mercurial > cpython-lru_cache-weakref

annotate Objects/stringlib/codecs.h @ 74588:dfd05348331c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Sort functions in the doc of the time module Move wallclock() at the end.
author Victor Stinner <victor.stinner@haypocalc.com>
date Tue, 24 Jan 2012 01:12:54 +0100
parents fbd797fc3809
children e08c3791f035
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
73668
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
1 /* stringlib: codec implementations */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
2
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
3 #if STRINGLIB_IS_UNICODE
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
4
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
5 /* Mask to check or force alignment of a pointer to C 'long' boundaries */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
6 #define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
7
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
8 /* Mask to quickly check whether a C 'long' contains a
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
9 non-ASCII, UTF8-encoded char. */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
10 #if (SIZEOF_LONG == 8)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
11 # define ASCII_CHAR_MASK 0x8080808080808080L
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
12 #elif (SIZEOF_LONG == 4)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
13 # define ASCII_CHAR_MASK 0x80808080L
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
14 #else
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
15 # error C 'long' size should be either 4 or 8!
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
16 #endif
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
17
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
18 Py_LOCAL_INLINE(int)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
19 STRINGLIB(utf8_try_decode)(const char *start, const char *end,
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
20 STRINGLIB_CHAR *dest,
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
21 const char **src_pos, Py_ssize_t *dest_index)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
22 {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
23 int ret;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
24 Py_ssize_t n;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
25 const char *s = start;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
26 const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
27 STRINGLIB_CHAR *p = dest;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
28
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
29 while (s < end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
30 Py_UCS4 ch = (unsigned char)*s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
31
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
32 if (ch < 0x80) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
33 /* Fast path for runs of ASCII characters. Given that common UTF-8
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
34 input will consist of an overwhelming majority of ASCII
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
35 characters, we try to optimize for this case by checking
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
36 as many characters as a C 'long' can contain.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
37 First, check if we can do an aligned read, as most CPUs have
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
38 a penalty for unaligned reads.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
39 */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
40 if (!((size_t) s & LONG_PTR_MASK)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
41 /* Help register allocation */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
42 register const char *_s = s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
43 register STRINGLIB_CHAR *_p = p;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
44 while (_s < aligned_end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
45 /* Read a whole long at a time (either 4 or 8 bytes),
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
46 and do a fast unrolled copy if it only contains ASCII
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
47 characters. */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
48 unsigned long value = *(unsigned long *) _s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
49 if (value & ASCII_CHAR_MASK)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
50 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
51 _p[0] = _s[0];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
52 _p[1] = _s[1];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
53 _p[2] = _s[2];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
54 _p[3] = _s[3];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
55 #if (SIZEOF_LONG == 8)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
56 _p[4] = _s[4];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
57 _p[5] = _s[5];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
58 _p[6] = _s[6];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
59 _p[7] = _s[7];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
60 #endif
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
61 _s += SIZEOF_LONG;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
62 _p += SIZEOF_LONG;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
63 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
64 s = _s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
65 p = _p;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
66 if (s == end)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
67 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
68 ch = (unsigned char)*s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
69 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
70 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
71
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
72 if (ch < 0x80) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
73 s++;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
74 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
75 continue;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
76 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
77
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
78 n = utf8_code_length[ch];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
79
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
80 if (s + n > end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
81 /* unexpected end of data: the caller will decide whether
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
82 it's an error or not */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
83 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
84 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
85
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
86 switch (n) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
87 case 0:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
88 /* invalid start byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
89 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
90 case 1:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
91 /* internal error */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
92 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
93 case 2:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
94 if ((s[1] & 0xc0) != 0x80)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
95 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
96 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
97 ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
98 assert ((ch > 0x007F) && (ch <= 0x07FF));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
99 s += 2;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
100 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
101 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
102
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
103 case 3:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
104 /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
105 will result in surrogates in range d800-dfff. Surrogates are
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
106 not valid UTF-8 so they are rejected.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
107 See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
108 (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
109 if ((s[1] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
110 (s[2] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
111 ((unsigned char)s[0] == 0xE0 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
112 (unsigned char)s[1] < 0xA0) ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
113 ((unsigned char)s[0] == 0xED &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
114 (unsigned char)s[1] > 0x9F)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
115 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
116 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
117 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
118 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
119 assert ((ch > 0x07FF) && (ch <= 0xFFFF));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
120 s += 3;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
121 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
122 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
123
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
124 case 4:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
125 if ((s[1] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
126 (s[2] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
127 (s[3] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
128 ((unsigned char)s[0] == 0xF0 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
129 (unsigned char)s[1] < 0x90) ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
130 ((unsigned char)s[0] == 0xF4 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
131 (unsigned char)s[1] > 0x8F)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
132 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
133 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
134 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
135 ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
136 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
137 assert ((ch > 0xFFFF) && (ch <= 0x10ffff));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
138 s += 4;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
139 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
140 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
141 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
142 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
143 ret = 0;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
144 goto _ok;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
145 _error:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
146 ret = -1;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
147 _ok:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
148 *src_pos = s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
149 *dest_index = p - dest;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
150 return ret;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
151 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
152
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
153 #undef LONG_PTR_MASK
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
154 #undef ASCII_CHAR_MASK
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
155
74032
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
156
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
157 /* UTF-8 encoder specialized for a Unicode kind to avoid the slow
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
158 PyUnicode_READ() macro. Delete some parts of the code depending on the kind:
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
159 UCS-1 strings don't need to handle surrogates for example. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
160 Py_LOCAL_INLINE(PyObject *)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
161 STRINGLIB(utf8_encoder)(PyObject *unicode,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
162 STRINGLIB_CHAR *data,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
163 Py_ssize_t size,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
164 const char *errors)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
165 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
166 #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
167
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
168 Py_ssize_t i; /* index into s of next input byte */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
169 PyObject *result; /* result string object */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
170 char *p; /* next free byte in output buffer */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
171 Py_ssize_t nallocated; /* number of result bytes allocated */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
172 Py_ssize_t nneeded; /* number of result bytes needed */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
173 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
174 PyObject *errorHandler = NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
175 PyObject *exc = NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
176 PyObject *rep = NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
177 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
178 #if STRINGLIB_SIZEOF_CHAR == 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
179 const Py_ssize_t max_char_size = 2;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
180 char stackbuf[MAX_SHORT_UNICHARS * 2];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
181 #elif STRINGLIB_SIZEOF_CHAR == 2
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
182 const Py_ssize_t max_char_size = 3;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
183 char stackbuf[MAX_SHORT_UNICHARS * 3];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
184 #else /* STRINGLIB_SIZEOF_CHAR == 4 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
185 const Py_ssize_t max_char_size = 4;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
186 char stackbuf[MAX_SHORT_UNICHARS * 4];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
187 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
188
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
189 assert(size >= 0);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
190
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
191 if (size <= MAX_SHORT_UNICHARS) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
192 /* Write into the stack buffer; nallocated can't overflow.
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
193 * At the end, we'll allocate exactly as much heap space as it
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
194 * turns out we need.
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
195 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
196 nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
197 result = NULL; /* will allocate after we're done */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
198 p = stackbuf;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
199 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
200 else {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
201 if (size > PY_SSIZE_T_MAX / max_char_size) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
202 /* integer overflow */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
203 return PyErr_NoMemory();
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
204 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
205 /* Overallocate on the heap, and give the excess back at the end. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
206 nallocated = size * max_char_size;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
207 result = PyBytes_FromStringAndSize(NULL, nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
208 if (result == NULL)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
209 return NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
210 p = PyBytes_AS_STRING(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
211 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
212
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
213 for (i = 0; i < size;) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
214 Py_UCS4 ch = data[i++];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
215
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
216 if (ch < 0x80) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
217 /* Encode ASCII */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
218 *p++ = (char) ch;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
219
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
220 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
221 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
222 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
223 if (ch < 0x0800)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
224 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
225 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
226 /* Encode Latin-1 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
227 *p++ = (char)(0xc0 | (ch >> 6));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
228 *p++ = (char)(0x80 | (ch & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
229 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
230 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
231 else if (Py_UNICODE_IS_SURROGATE(ch)) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
232 Py_ssize_t newpos;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
233 Py_ssize_t repsize, k, startpos;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
234 startpos = i-1;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
235 rep = unicode_encode_call_errorhandler(
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
236 errors, &errorHandler, "utf-8", "surrogates not allowed",
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
237 unicode, &exc, startpos, startpos+1, &newpos);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
238 if (!rep)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
239 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
240
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
241 if (PyBytes_Check(rep))
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
242 repsize = PyBytes_GET_SIZE(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
243 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
244 repsize = PyUnicode_GET_LENGTH(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
245
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
246 if (repsize > max_char_size) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
247 Py_ssize_t offset;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
248
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
249 if (result == NULL)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
250 offset = p - stackbuf;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
251 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
252 offset = p - PyBytes_AS_STRING(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
253
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
254 if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
255 /* integer overflow */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
256 PyErr_NoMemory();
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
257 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
258 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
259 nallocated += repsize - max_char_size;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
260 if (result != NULL) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
261 if (_PyBytes_Resize(&result, nallocated) < 0)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
262 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
263 } else {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
264 result = PyBytes_FromStringAndSize(NULL, nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
265 if (result == NULL)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
266 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
267 Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
268 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
269 p = PyBytes_AS_STRING(result) + offset;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
270 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
271
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
272 if (PyBytes_Check(rep)) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
273 char *prep = PyBytes_AS_STRING(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
274 for(k = repsize; k > 0; k--)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
275 *p++ = *prep++;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
276 } else /* rep is unicode */ {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
277 enum PyUnicode_Kind repkind;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
278 void *repdata;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
279
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
280 if (PyUnicode_READY(rep) < 0)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
281 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
282 repkind = PyUnicode_KIND(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
283 repdata = PyUnicode_DATA(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
284
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
285 for(k=0; k<repsize; k++) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
286 Py_UCS4 c = PyUnicode_READ(repkind, repdata, k);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
287 if (0x80 <= c) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
288 raise_encode_exception(&exc, "utf-8",
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
289 unicode,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
290 i-1, i,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
291 "surrogates not allowed");
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
292 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
293 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
294 *p++ = (char)c;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
295 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
296 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
297 Py_CLEAR(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
298 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
299 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
300 #if STRINGLIB_SIZEOF_CHAR > 2
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
301 if (ch < 0x10000)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
302 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
303 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
304 *p++ = (char)(0xe0 | (ch >> 12));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
305 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
306 *p++ = (char)(0x80 | (ch & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
307 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
308 #if STRINGLIB_SIZEOF_CHAR > 2
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
309 else /* ch >= 0x10000 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
310 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
311 assert(ch <= MAX_UNICODE);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
312 /* Encode UCS4 Unicode ordinals */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
313 *p++ = (char)(0xf0 | (ch >> 18));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
314 *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
315 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
316 *p++ = (char)(0x80 | (ch & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
317 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
318 #endif /* STRINGLIB_SIZEOF_CHAR > 2 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
319 #endif /* STRINGLIB_SIZEOF_CHAR > 1 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
320 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
321
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
322 if (result == NULL) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
323 /* This was stack allocated. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
324 nneeded = p - stackbuf;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
325 assert(nneeded <= nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
326 result = PyBytes_FromStringAndSize(stackbuf, nneeded);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
327 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
328 else {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
329 /* Cut back to size actually needed. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
330 nneeded = p - PyBytes_AS_STRING(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
331 assert(nneeded <= nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
332 _PyBytes_Resize(&result, nneeded);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
333 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
334
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
335 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
336 Py_XDECREF(errorHandler);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
337 Py_XDECREF(exc);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
338 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
339 return result;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
340
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
341 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
342 error:
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
343 Py_XDECREF(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
344 Py_XDECREF(errorHandler);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
345 Py_XDECREF(exc);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
346 Py_XDECREF(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
347 return NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
348 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
349
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
350 #undef MAX_SHORT_UNICHARS
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
351 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
352
73668
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
353 #endif /* STRINGLIB_IS_UNICODE */