Mercurial > cpython-lru_cache-weakref

annotate Objects/stringlib/codecs.h @ 73798:6cde416ef03b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Credit Per Øyvind Karlsen for the initial implementation of the lzma module (issue #6715).
author Nadeem Vawda <nadeem.vawda@gmail.com>
date Thu, 01 Dec 2011 01:18:27 +0200
parents 8e6c4acaf530
children fbd797fc3809
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
73668
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
1 /* stringlib: codec implementations */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
2
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
3 #if STRINGLIB_IS_UNICODE
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
4
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
5 /* Mask to check or force alignment of a pointer to C 'long' boundaries */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
6 #define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
7
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
8 /* Mask to quickly check whether a C 'long' contains a
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
9 non-ASCII, UTF8-encoded char. */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
10 #if (SIZEOF_LONG == 8)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
11 # define ASCII_CHAR_MASK 0x8080808080808080L
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
12 #elif (SIZEOF_LONG == 4)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
13 # define ASCII_CHAR_MASK 0x80808080L
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
14 #else
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
15 # error C 'long' size should be either 4 or 8!
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
16 #endif
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
17
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
18 Py_LOCAL_INLINE(int)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
19 STRINGLIB(utf8_try_decode)(const char *start, const char *end,
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
20 STRINGLIB_CHAR *dest,
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
21 const char **src_pos, Py_ssize_t *dest_index)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
22 {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
23 int ret;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
24 Py_ssize_t n;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
25 const char *s = start;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
26 const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
27 STRINGLIB_CHAR *p = dest;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
28
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
29 while (s < end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
30 Py_UCS4 ch = (unsigned char)*s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
31
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
32 if (ch < 0x80) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
33 /* Fast path for runs of ASCII characters. Given that common UTF-8
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
34 input will consist of an overwhelming majority of ASCII
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
35 characters, we try to optimize for this case by checking
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
36 as many characters as a C 'long' can contain.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
37 First, check if we can do an aligned read, as most CPUs have
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
38 a penalty for unaligned reads.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
39 */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
40 if (!((size_t) s & LONG_PTR_MASK)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
41 /* Help register allocation */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
42 register const char *_s = s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
43 register STRINGLIB_CHAR *_p = p;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
44 while (_s < aligned_end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
45 /* Read a whole long at a time (either 4 or 8 bytes),
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
46 and do a fast unrolled copy if it only contains ASCII
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
47 characters. */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
48 unsigned long value = *(unsigned long *) _s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
49 if (value & ASCII_CHAR_MASK)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
50 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
51 _p[0] = _s[0];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
52 _p[1] = _s[1];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
53 _p[2] = _s[2];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
54 _p[3] = _s[3];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
55 #if (SIZEOF_LONG == 8)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
56 _p[4] = _s[4];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
57 _p[5] = _s[5];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
58 _p[6] = _s[6];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
59 _p[7] = _s[7];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
60 #endif
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
61 _s += SIZEOF_LONG;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
62 _p += SIZEOF_LONG;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
63 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
64 s = _s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
65 p = _p;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
66 if (s == end)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
67 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
68 ch = (unsigned char)*s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
69 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
70 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
71
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
72 if (ch < 0x80) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
73 s++;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
74 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
75 continue;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
76 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
77
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
78 n = utf8_code_length[ch];
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
79
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
80 if (s + n > end) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
81 /* unexpected end of data: the caller will decide whether
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
82 it's an error or not */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
83 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
84 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
85
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
86 switch (n) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
87 case 0:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
88 /* invalid start byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
89 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
90 case 1:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
91 /* internal error */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
92 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
93 case 2:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
94 if ((s[1] & 0xc0) != 0x80)
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
95 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
96 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
97 ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
98 assert ((ch > 0x007F) && (ch <= 0x07FF));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
99 s += 2;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
100 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
101 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
102
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
103 case 3:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
104 /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
105 will result in surrogates in range d800-dfff. Surrogates are
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
106 not valid UTF-8 so they are rejected.
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
107 See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
108 (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
109 if ((s[1] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
110 (s[2] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
111 ((unsigned char)s[0] == 0xE0 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
112 (unsigned char)s[1] < 0xA0) ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
113 ((unsigned char)s[0] == 0xED &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
114 (unsigned char)s[1] > 0x9F)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
115 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
116 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
117 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
118 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
119 assert ((ch > 0x07FF) && (ch <= 0xFFFF));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
120 s += 3;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
121 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
122 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
123
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
124 case 4:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
125 if ((s[1] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
126 (s[2] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
127 (s[3] & 0xc0) != 0x80 ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
128 ((unsigned char)s[0] == 0xF0 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
129 (unsigned char)s[1] < 0x90) ||
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
130 ((unsigned char)s[0] == 0xF4 &&
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
131 (unsigned char)s[1] > 0x8F)) {
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
132 /* invalid continuation byte */
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
133 goto _error;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
134 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
135 ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
136 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
137 assert ((ch > 0xFFFF) && (ch <= 0x10ffff));
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
138 s += 4;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
139 *p++ = ch;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
140 break;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
141 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
142 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
143 ret = 0;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
144 goto _ok;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
145 _error:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
146 ret = -1;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
147 _ok:
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
148 *src_pos = s;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
149 *dest_index = p - dest;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
150 return ret;
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
151 }
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
152
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
153 #undef LONG_PTR_MASK
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
154 #undef ASCII_CHAR_MASK
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
155
8e6c4acaf530 Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.
Antoine Pitrou <solipsis@pitrou.net>
parents:
diff changeset
156 #endif /* STRINGLIB_IS_UNICODE */