Mercurial > cpython-lru_cache-weakref

annotate Objects/stringlib/codecs.h @ 74061:03701ac95d25

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
iobench.py: add more info in the header Write the Python version, Unicode implementation and the platform.
author Victor Stinner <victor.stinner@haypocalc.com>
date Sun, 18 Dec 2011 21:38:51 +0100
parents fbd797fc3809
children e08c3791f035
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
73668
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
1 /* stringlib: codec implementations */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
2
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
3 #if STRINGLIB_IS_UNICODE
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
4
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
5 /* Mask to check or force alignment of a pointer to C 'long' boundaries */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
6 #define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
7
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
8 /* Mask to quickly check whether a C 'long' contains a
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
9 non-ASCII, UTF8-encoded char. */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
10 #if (SIZEOF_LONG == 8)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
11 # define ASCII_CHAR_MASK 0x8080808080808080L
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
12 #elif (SIZEOF_LONG == 4)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
13 # define ASCII_CHAR_MASK 0x80808080L
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
14 #else
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
15 # error C 'long' size should be either 4 or 8!
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
16 #endif
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
17
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
18 Py_LOCAL_INLINE(int)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
19 STRINGLIB(utf8_try_decode)(const char *start, const char *end,
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
20 STRINGLIB_CHAR *dest,
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
21 const char **src_pos, Py_ssize_t *dest_index)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
22 {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
23 int ret;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
24 Py_ssize_t n;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
25 const char *s = start;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
26 const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
27 STRINGLIB_CHAR *p = dest;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
28
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
29 while (s < end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
30 Py_UCS4 ch = (unsigned char)*s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
31
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
32 if (ch < 0x80) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
33 /* Fast path for runs of ASCII characters. Given that common UTF-8
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
34 input will consist of an overwhelming majority of ASCII
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
35 characters, we try to optimize for this case by checking
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
36 as many characters as a C 'long' can contain.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
37 First, check if we can do an aligned read, as most CPUs have
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
38 a penalty for unaligned reads.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
39 */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
40 if (!((size_t) s & LONG_PTR_MASK)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
41 /* Help register allocation */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
42 register const char *_s = s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
43 register STRINGLIB_CHAR *_p = p;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
44 while (_s < aligned_end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
45 /* Read a whole long at a time (either 4 or 8 bytes),
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
46 and do a fast unrolled copy if it only contains ASCII
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
47 characters. */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
48 unsigned long value = *(unsigned long *) _s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
49 if (value & ASCII_CHAR_MASK)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
50 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
51 _p[0] = _s[0];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
52 _p[1] = _s[1];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
53 _p[2] = _s[2];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
54 _p[3] = _s[3];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
55 #if (SIZEOF_LONG == 8)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
56 _p[4] = _s[4];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
57 _p[5] = _s[5];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
58 _p[6] = _s[6];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
59 _p[7] = _s[7];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
60 #endif
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
61 _s += SIZEOF_LONG;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
62 _p += SIZEOF_LONG;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
63 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
64 s = _s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
65 p = _p;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
66 if (s == end)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
67 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
68 ch = (unsigned char)*s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
69 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
70 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
71
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
72 if (ch < 0x80) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
73 s++;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
74 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
75 continue;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
76 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
77
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
78 n = utf8_code_length[ch];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
79
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
80 if (s + n > end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
81 /* unexpected end of data: the caller will decide whether
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
82 it's an error or not */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
83 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
84 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
85
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
86 switch (n) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
87 case 0:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
88 /* invalid start byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
89 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
90 case 1:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
91 /* internal error */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
92 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
93 case 2:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
94 if ((s[1] & 0xc0) != 0x80)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
95 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
96 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
97 ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
98 assert ((ch > 0x007F) && (ch <= 0x07FF));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
99 s += 2;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
100 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
101 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
102
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
103 case 3:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
104 /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
105 will result in surrogates in range d800-dfff. Surrogates are
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
106 not valid UTF-8 so they are rejected.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
107 See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
108 (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
109 if ((s[1] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
110 (s[2] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
111 ((unsigned char)s[0] == 0xE0 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
112 (unsigned char)s[1] < 0xA0) ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
113 ((unsigned char)s[0] == 0xED &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
114 (unsigned char)s[1] > 0x9F)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
115 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
116 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
117 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
118 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
119 assert ((ch > 0x07FF) && (ch <= 0xFFFF));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
120 s += 3;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
121 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
122 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
123
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
124 case 4:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
125 if ((s[1] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
126 (s[2] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
127 (s[3] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
128 ((unsigned char)s[0] == 0xF0 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
129 (unsigned char)s[1] < 0x90) ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
130 ((unsigned char)s[0] == 0xF4 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
131 (unsigned char)s[1] > 0x8F)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
132 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
133 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
134 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
135 ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
136 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
137 assert ((ch > 0xFFFF) && (ch <= 0x10ffff));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
138 s += 4;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
139 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
140 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
141 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
142 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
143 ret = 0;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
144 goto _ok;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
145 _error:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
146 ret = -1;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
147 _ok:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
148 *src_pos = s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
149 *dest_index = p - dest;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
150 return ret;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
151 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
152
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
153 #undef LONG_PTR_MASK
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
154 #undef ASCII_CHAR_MASK
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
155
74032
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
156
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
157 /* UTF-8 encoder specialized for a Unicode kind to avoid the slow
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
158 PyUnicode_READ() macro. Delete some parts of the code depending on the kind:
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
159 UCS-1 strings don't need to handle surrogates for example. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
160 Py_LOCAL_INLINE(PyObject *)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
161 STRINGLIB(utf8_encoder)(PyObject *unicode,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
162 STRINGLIB_CHAR *data,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
163 Py_ssize_t size,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
164 const char *errors)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
165 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
166 #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
167
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
168 Py_ssize_t i; /* index into s of next input byte */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
169 PyObject *result; /* result string object */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
170 char *p; /* next free byte in output buffer */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
171 Py_ssize_t nallocated; /* number of result bytes allocated */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
172 Py_ssize_t nneeded; /* number of result bytes needed */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
173 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
174 PyObject *errorHandler = NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
175 PyObject *exc = NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
176 PyObject *rep = NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
177 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
178 #if STRINGLIB_SIZEOF_CHAR == 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
179 const Py_ssize_t max_char_size = 2;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
180 char stackbuf[MAX_SHORT_UNICHARS * 2];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
181 #elif STRINGLIB_SIZEOF_CHAR == 2
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
182 const Py_ssize_t max_char_size = 3;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
183 char stackbuf[MAX_SHORT_UNICHARS * 3];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
184 #else /* STRINGLIB_SIZEOF_CHAR == 4 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
185 const Py_ssize_t max_char_size = 4;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
186 char stackbuf[MAX_SHORT_UNICHARS * 4];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
187 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
188
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
189 assert(size >= 0);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
190
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
191 if (size <= MAX_SHORT_UNICHARS) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
192 /* Write into the stack buffer; nallocated can't overflow.
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
193 * At the end, we'll allocate exactly as much heap space as it
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
194 * turns out we need.
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
195 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
196 nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
197 result = NULL; /* will allocate after we're done */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
198 p = stackbuf;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
199 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
200 else {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
201 if (size > PY_SSIZE_T_MAX / max_char_size) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
202 /* integer overflow */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
203 return PyErr_NoMemory();
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
204 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
205 /* Overallocate on the heap, and give the excess back at the end. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
206 nallocated = size * max_char_size;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
207 result = PyBytes_FromStringAndSize(NULL, nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
208 if (result == NULL)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
209 return NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
210 p = PyBytes_AS_STRING(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
211 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
212
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
213 for (i = 0; i < size;) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
214 Py_UCS4 ch = data[i++];
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
215
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
216 if (ch < 0x80) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
217 /* Encode ASCII */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
218 *p++ = (char) ch;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
219
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
220 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
221 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
222 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
223 if (ch < 0x0800)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
224 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
225 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
226 /* Encode Latin-1 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
227 *p++ = (char)(0xc0 | (ch >> 6));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
228 *p++ = (char)(0x80 | (ch & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
229 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
230 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
231 else if (Py_UNICODE_IS_SURROGATE(ch)) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
232 Py_ssize_t newpos;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
233 Py_ssize_t repsize, k, startpos;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
234 startpos = i-1;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
235 rep = unicode_encode_call_errorhandler(
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
236 errors, &errorHandler, "utf-8", "surrogates not allowed",
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
237 unicode, &exc, startpos, startpos+1, &newpos);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
238 if (!rep)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
239 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
240
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
241 if (PyBytes_Check(rep))
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
242 repsize = PyBytes_GET_SIZE(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
243 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
244 repsize = PyUnicode_GET_LENGTH(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
245
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
246 if (repsize > max_char_size) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
247 Py_ssize_t offset;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
248
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
249 if (result == NULL)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
250 offset = p - stackbuf;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
251 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
252 offset = p - PyBytes_AS_STRING(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
253
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
254 if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
255 /* integer overflow */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
256 PyErr_NoMemory();
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
257 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
258 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
259 nallocated += repsize - max_char_size;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
260 if (result != NULL) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
261 if (_PyBytes_Resize(&result, nallocated) < 0)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
262 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
263 } else {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
264 result = PyBytes_FromStringAndSize(NULL, nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
265 if (result == NULL)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
266 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
267 Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
268 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
269 p = PyBytes_AS_STRING(result) + offset;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
270 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
271
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
272 if (PyBytes_Check(rep)) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
273 char *prep = PyBytes_AS_STRING(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
274 for(k = repsize; k > 0; k--)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
275 *p++ = *prep++;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
276 } else /* rep is unicode */ {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
277 enum PyUnicode_Kind repkind;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
278 void *repdata;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
279
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
280 if (PyUnicode_READY(rep) < 0)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
281 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
282 repkind = PyUnicode_KIND(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
283 repdata = PyUnicode_DATA(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
284
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
285 for(k=0; k<repsize; k++) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
286 Py_UCS4 c = PyUnicode_READ(repkind, repdata, k);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
287 if (0x80 <= c) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
288 raise_encode_exception(&exc, "utf-8",
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
289 unicode,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
290 i-1, i,
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
291 "surrogates not allowed");
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
292 goto error;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
293 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
294 *p++ = (char)c;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
295 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
296 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
297 Py_CLEAR(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
298 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
299 else
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
300 #if STRINGLIB_SIZEOF_CHAR > 2
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
301 if (ch < 0x10000)
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
302 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
303 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
304 *p++ = (char)(0xe0 | (ch >> 12));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
305 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
306 *p++ = (char)(0x80 | (ch & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
307 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
308 #if STRINGLIB_SIZEOF_CHAR > 2
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
309 else /* ch >= 0x10000 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
310 {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
311 assert(ch <= MAX_UNICODE);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
312 /* Encode UCS4 Unicode ordinals */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
313 *p++ = (char)(0xf0 | (ch >> 18));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
314 *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
315 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
316 *p++ = (char)(0x80 | (ch & 0x3f));
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
317 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
318 #endif /* STRINGLIB_SIZEOF_CHAR > 2 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
319 #endif /* STRINGLIB_SIZEOF_CHAR > 1 */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
320 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
321
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
322 if (result == NULL) {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
323 /* This was stack allocated. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
324 nneeded = p - stackbuf;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
325 assert(nneeded <= nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
326 result = PyBytes_FromStringAndSize(stackbuf, nneeded);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
327 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
328 else {
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
329 /* Cut back to size actually needed. */
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
330 nneeded = p - PyBytes_AS_STRING(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
331 assert(nneeded <= nallocated);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
332 _PyBytes_Resize(&result, nneeded);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
333 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
334
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
335 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
336 Py_XDECREF(errorHandler);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
337 Py_XDECREF(exc);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
338 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
339 return result;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
340
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
341 #if STRINGLIB_SIZEOF_CHAR > 1
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
342 error:
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
343 Py_XDECREF(rep);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
344 Py_XDECREF(errorHandler);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
345 Py_XDECREF(exc);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
346 Py_XDECREF(result);
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
347 return NULL;
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
348 #endif
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
349
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
350 #undef MAX_SHORT_UNICHARS
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
351 }
fbd797fc3809 Issue #13624: Write a specialized UTF-8 encoder to allow more optimization
Victor Stinner <victor.stinner@haypocalc.com>
parents: 73668
diff changeset
352
73668
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
353 #endif /* STRINGLIB_IS_UNICODE */