Mercurial > cpython-lru_cache-weakref

annotate Modules/unicodedata.c @ 58925:eecf1bef052d 2.6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged revisions 78207 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r78207 | ezio.melotti | 2010-02-17 01:26:09 +0200 (Wed, 17 Feb 2010) | 1 line #7930: fix stripid ........
author Ezio Melotti <ezio.melotti@gmail.com>
date Tue, 16 Feb 2010 23:29:44 +0000
parents c37c7dbd7880
children 8f1202ca888e 019244f4b0d0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1 /* ------------------------------------------------------------------------
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
2
49124
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
3 unicodedata -- Provides access to the Unicode 5.1 data base.
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
4
49124
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
5 Data was extracted from the Unicode 5.1 UnicodeData.txt file.
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
6
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14818
diff changeset
7 Written by Marc-Andre Lemburg (mal@lemburg.com).
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14818
diff changeset
8 Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
9 Modified by Martin v. Löwis (martin@v.loewis.de)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14818
diff changeset
11 Copyright (c) Corporation for National Research Initiatives.
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 ------------------------------------------------------------------------ */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
14
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 #include "Python.h"
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
16 #include "ucnhash.h"
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
17 #include "structmember.h"
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
18
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
19 /* character properties */
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
20
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
21 typedef struct {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
22 const unsigned char category; /* index into
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
23 _PyUnicode_CategoryNames */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
24 const unsigned char combining; /* combining class value 0 - 255 */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
25 const unsigned char bidirectional; /* index into
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
26 _PyUnicode_BidirectionalNames */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
27 const unsigned char mirrored; /* true if mirrored in bidir mode */
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
28 const unsigned char east_asian_width; /* index into
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
29 _PyUnicode_EastAsianWidth */
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
30 } _PyUnicode_DatabaseRecord;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
31
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
32 typedef struct change_record {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
33 /* sequence of fields should be the same as in merge_old_version */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
34 const unsigned char bidir_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
35 const unsigned char category_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
36 const unsigned char decimal_changed;
49124
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
37 const unsigned char mirrored_changed;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
38 const int numeric_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
39 } change_record;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
40
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
41 /* data file generated by Tools/unicode/makeunicodedata.py */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
42 #include "unicodedata_db.h"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
43
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
44 static const _PyUnicode_DatabaseRecord*
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
45 _getrecord_ex(Py_UCS4 code)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
46 {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
47 int index;
28100
54b35da9879e Fix SF bug #694816, remove comparison of unsigned value < 0
Neal Norwitz <nnorwitz@gmail.com>
parents: 26571
diff changeset
48 if (code >= 0x110000)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
49 index = 0;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
50 else {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
51 index = index1[(code>>SHIFT)];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
52 index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
53 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
54
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
55 return &_PyUnicode_Database_Records[index];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
56 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
57
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
58 /* ------------- Previous-version API ------------------------------------- */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
59 typedef struct previous_version {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
60 PyObject_HEAD
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
61 const char *name;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
62 const change_record* (*getrecord)(Py_UCS4);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
63 Py_UCS4 (*normalization)(Py_UCS4);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
64 } PreviousDBVersion;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
65
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
66 #define get_old_record(self, v) ((((PreviousDBVersion*)self)->getrecord)(v))
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
67
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
68 static PyMemberDef DB_members[] = {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
69 {"unidata_version", T_STRING, offsetof(PreviousDBVersion, name), READONLY},
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
70 {NULL}
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
71 };
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
72
40319
7059df7c15df Replace C++ comment with C comment (fixes SF bug #1593525).
Walter Dörwald <walter@livinglogic.de>
parents: 39453
diff changeset
73 /* forward declaration */
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
74 static PyTypeObject UCD_Type;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
75
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
76 static PyObject*
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
77 new_previous_version(const char*name, const change_record* (*getrecord)(Py_UCS4),
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
78 Py_UCS4 (*normalization)(Py_UCS4))
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
79 {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
80 PreviousDBVersion *self;
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
81 self = PyObject_New(PreviousDBVersion, &UCD_Type);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
82 if (self == NULL)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
83 return NULL;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
84 self->name = name;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
85 self->getrecord = getrecord;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
86 self->normalization = normalization;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
87 return (PyObject*)self;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
88 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
89
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
90
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
91 static Py_UCS4 getuchar(PyUnicodeObject *obj)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
92 {
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
93 Py_UNICODE *v = PyUnicode_AS_UNICODE(obj);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
94
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
95 if (PyUnicode_GET_SIZE(obj) == 1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
96 return *v;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
97 #ifndef Py_UNICODE_WIDE
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
98 else if ((PyUnicode_GET_SIZE(obj) == 2) &&
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
99 (0xD800 <= v[0] && v[0] <= 0xDBFF) &&
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
100 (0xDC00 <= v[1] && v[1] <= 0xDFFF))
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
101 return (((v[0] & 0x3FF)<<10) | (v[1] & 0x3FF)) + 0x10000;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
102 #endif
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
103 PyErr_SetString(PyExc_TypeError,
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
104 "need a single Unicode character as parameter");
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
105 return (Py_UCS4)-1;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
106 }
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
107
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
108 /* --- Module API --------------------------------------------------------- */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
109
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
110 PyDoc_STRVAR(unicodedata_decimal__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
111 "decimal(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
112 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
113 Returns the decimal value assigned to the Unicode character unichr\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
114 as integer. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
115 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
116
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
117 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
118 unicodedata_decimal(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
120 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
121 PyObject *defobj = NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
122 int have_old = 0;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
123 long rc;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
124 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
125
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
126 if (!PyArg_ParseTuple(args, "O!|O:decimal", &PyUnicode_Type, &v, &defobj))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
127 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
128 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
129 if (c == (Py_UCS4)-1)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
130 return NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
131
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
132 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
133 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
134 if (old->category_changed == 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
135 /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
136 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
137 rc = -1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
138 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
139 else if (old->decimal_changed != 0xFF) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
140 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
141 rc = old->decimal_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
142 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
143 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
144
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
145 if (!have_old)
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
146 rc = Py_UNICODE_TODECIMAL(c);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
147 if (rc < 0) {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
148 if (defobj == NULL) {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
149 PyErr_SetString(PyExc_ValueError,
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
150 "not a decimal");
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
151 return NULL;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
152 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
153 else {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
154 Py_INCREF(defobj);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
155 return defobj;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
156 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
157 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
158 return PyInt_FromLong(rc);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
159 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
160
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
161 PyDoc_STRVAR(unicodedata_digit__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
162 "digit(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
163 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
164 Returns the digit value assigned to the Unicode character unichr as\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
165 integer. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
166 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
167
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
168 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
169 unicodedata_digit(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
170 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
171 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
172 PyObject *defobj = NULL;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
173 long rc;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
174 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
175
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
176 if (!PyArg_ParseTuple(args, "O!|O:digit", &PyUnicode_Type, &v, &defobj))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
177 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
178 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
179 if (c == (Py_UCS4)-1)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
180 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
181 rc = Py_UNICODE_TODIGIT(c);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
182 if (rc < 0) {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
183 if (defobj == NULL) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
184 PyErr_SetString(PyExc_ValueError, "not a digit");
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
185 return NULL;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
186 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
187 else {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
188 Py_INCREF(defobj);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
189 return defobj;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
190 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
191 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
192 return PyInt_FromLong(rc);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
193 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
194
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
195 PyDoc_STRVAR(unicodedata_numeric__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
196 "numeric(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
197 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
198 Returns the numeric value assigned to the Unicode character unichr\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
199 as float. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
200 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
201
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
202 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
203 unicodedata_numeric(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
204 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
205 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
206 PyObject *defobj = NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
207 int have_old = 0;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
208 double rc;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
209 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
210
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
211 if (!PyArg_ParseTuple(args, "O!|O:numeric", &PyUnicode_Type, &v, &defobj))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
212 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
213 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
214 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
215 return NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
216
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
217 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
218 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
219 if (old->category_changed == 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
220 /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
221 have_old = 1;
38422
dfbd457ca67a Patch 1494554: Update numeric properties to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 37780
diff changeset
222 rc = -1.0;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
223 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
224 else if (old->decimal_changed != 0xFF) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
225 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
226 rc = old->decimal_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
227 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
228 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
229
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
230 if (!have_old)
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
231 rc = Py_UNICODE_TONUMERIC(c);
38422
dfbd457ca67a Patch 1494554: Update numeric properties to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 37780
diff changeset
232 if (rc == -1.0) {
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
233 if (defobj == NULL) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
234 PyErr_SetString(PyExc_ValueError, "not a numeric character");
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
235 return NULL;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
236 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
237 else {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
238 Py_INCREF(defobj);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
239 return defobj;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
240 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
241 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
242 return PyFloat_FromDouble(rc);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
243 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
244
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
245 PyDoc_STRVAR(unicodedata_category__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
246 "category(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
247 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
248 Returns the general category assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
249 unichr as string.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
250
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
251 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
252 unicodedata_category(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
253 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
254 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
255 int index;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
256 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
257
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
258 if (!PyArg_ParseTuple(args, "O!:category",
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
259 &PyUnicode_Type, &v))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
260 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
261 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
262 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
263 return NULL;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
264 index = (int) _getrecord_ex(c)->category;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
265 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
266 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
267 if (old->category_changed != 0xFF)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
268 index = old->category_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
269 }
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47451
diff changeset
270 return PyString_FromString(_PyUnicode_CategoryNames[index]);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
271 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
272
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
273 PyDoc_STRVAR(unicodedata_bidirectional__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
274 "bidirectional(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
275 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
276 Returns the bidirectional category assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
277 unichr as string. If no such value is defined, an empty string is\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
278 returned.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
279
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
280 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
281 unicodedata_bidirectional(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
282 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
283 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
284 int index;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
285 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
286
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
287 if (!PyArg_ParseTuple(args, "O!:bidirectional",
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
288 &PyUnicode_Type, &v))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
289 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
290 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
291 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
292 return NULL;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
293 index = (int) _getrecord_ex(c)->bidirectional;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
294 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
295 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
296 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
297 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
298 else if (old->bidir_changed != 0xFF)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
299 index = old->bidir_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
300 }
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47451
diff changeset
301 return PyString_FromString(_PyUnicode_BidirectionalNames[index]);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
302 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
303
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
304 PyDoc_STRVAR(unicodedata_combining__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
305 "combining(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
306 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
307 Returns the canonical combining class assigned to the Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
308 character unichr as integer. Returns 0 if no combining class is\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
309 defined.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
310
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
311 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
312 unicodedata_combining(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
313 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
314 PyUnicodeObject *v;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
315 int index;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
316 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
317
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
318 if (!PyArg_ParseTuple(args, "O!:combining",
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
319 &PyUnicode_Type, &v))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
320 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
321 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
322 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
323 return NULL;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
324 index = (int) _getrecord_ex(c)->combining;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
325 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
326 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
327 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
328 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
329 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
330 return PyInt_FromLong(index);
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
331 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
332
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
333 PyDoc_STRVAR(unicodedata_mirrored__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
334 "mirrored(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
335 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
336 Returns the mirrored property assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
337 unichr as integer. Returns 1 if the character has been identified as\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
338 a \"mirrored\" character in bidirectional text, 0 otherwise.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
339
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
340 static PyObject *
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
341 unicodedata_mirrored(PyObject *self, PyObject *args)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
342 {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
343 PyUnicodeObject *v;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
344 int index;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
345 Py_UCS4 c;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
346
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
347 if (!PyArg_ParseTuple(args, "O!:mirrored",
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
348 &PyUnicode_Type, &v))
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
349 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
350 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
351 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
352 return NULL;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
353 index = (int) _getrecord_ex(c)->mirrored;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
354 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
355 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
356 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
357 index = 0; /* unassigned */
49124
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
358 else if (old->mirrored_changed != 0xFF)
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
359 index = old->mirrored_changed;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
360 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
361 return PyInt_FromLong(index);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
362 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
363
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
364 PyDoc_STRVAR(unicodedata_east_asian_width__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
365 "east_asian_width(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
366 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
367 Returns the east asian width assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
368 unichr as string.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
369
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
370 static PyObject *
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
371 unicodedata_east_asian_width(PyObject *self, PyObject *args)
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
372 {
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
373 PyUnicodeObject *v;
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
374 int index;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
375 Py_UCS4 c;
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
376
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
377 if (!PyArg_ParseTuple(args, "O!:east_asian_width",
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
378 &PyUnicode_Type, &v))
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
379 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
380 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
381 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
382 return NULL;
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
383 index = (int) _getrecord_ex(c)->east_asian_width;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
384 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
385 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
386 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
387 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
388 }
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47451
diff changeset
389 return PyString_FromString(_PyUnicode_EastAsianWidthNames[index]);
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
390 }
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
391
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
392 PyDoc_STRVAR(unicodedata_decomposition__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
393 "decomposition(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
394 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
395 Returns the character decomposition mapping assigned to the Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
396 character unichr as string. An empty string is returned in case no\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
397 such mapping is defined.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
398
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
399 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
400 unicodedata_decomposition(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
401 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
402 PyUnicodeObject *v;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
403 char decomp[256];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
404 int code, index, count, i;
39203
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
405 unsigned int prefix_index;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
406 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
407
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
408 if (!PyArg_ParseTuple(args, "O!:decomposition",
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
409 &PyUnicode_Type, &v))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
410 return NULL;
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
411 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
412 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
413 return NULL;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
414
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
415 code = (int)c;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
416
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
417 if (self) {
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
418 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
419 if (old->category_changed == 0)
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47451
diff changeset
420 return PyString_FromString(""); /* unassigned */
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
421 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
422
25852
1f5f6481886d Update to Unicode 3.2 database.
Martin v. Löwis <martin@v.loewis.de>
parents: 24363
diff changeset
423 if (code < 0 || code >= 0x110000)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
424 index = 0;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
425 else {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
426 index = decomp_index1[(code>>DECOMP_SHIFT)];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
427 index = decomp_index2[(index<<DECOMP_SHIFT)+
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
428 (code&((1<<DECOMP_SHIFT)-1))];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
429 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
430
21112
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
431 /* high byte is number of hex bytes (usually one or two), low byte
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
432 is prefix code (from*/
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
433 count = decomp_data[index] >> 8;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
434
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
435 /* XXX: could allocate the PyString up front instead
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
436 (strlen(prefix) + 5 * count + 1 bytes) */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
437
39203
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
438 /* Based on how index is calculated above and decomp_data is generated
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
439 from Tools/unicode/makeunicodedata.py, it should not be possible
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
440 to overflow decomp_prefix. */
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
441 prefix_index = decomp_data[index] & 255;
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
442 assert(prefix_index < (sizeof(decomp_prefix)/sizeof(*decomp_prefix)));
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
443
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
444 /* copy prefix */
39203
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
445 i = strlen(decomp_prefix[prefix_index]);
0b0cc7a49682 No functional change. Add comment and assert to describe why there cannot be overflow which was reported by Klocwork. Discussed on python-dev
Neal Norwitz <nnorwitz@gmail.com>
parents: 38422
diff changeset
446 memcpy(decomp, decomp_prefix[prefix_index], i);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
447
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
448 while (count-- > 0) {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
449 if (i)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
450 decomp[i++] = ' ';
21112
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
451 assert((size_t)i < sizeof(decomp));
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
452 PyOS_snprintf(decomp + i, sizeof(decomp) - i, "%04X",
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
453 decomp_data[++index]);
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
454 i += strlen(decomp + i);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
455 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
456
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
457 decomp[i] = '\0';
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
458
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47451
diff changeset
459 return PyString_FromString(decomp);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
460 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
461
37780
b95b5c33cb3a No reason to export get_decomp_record, make static
Neal Norwitz <nnorwitz@gmail.com>
parents: 37664
diff changeset
462 static void
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
463 get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *count)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
464 {
28100
54b35da9879e Fix SF bug #694816, remove comparison of unsigned value < 0
Neal Norwitz <nnorwitz@gmail.com>
parents: 26571
diff changeset
465 if (code >= 0x110000) {
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
466 *index = 0;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
467 } else if (self && get_old_record(self, code)->category_changed==0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
468 /* unassigned in old version */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
469 *index = 0;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
470 }
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
471 else {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
472 *index = decomp_index1[(code>>DECOMP_SHIFT)];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
473 *index = decomp_index2[(*index<<DECOMP_SHIFT)+
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
474 (code&((1<<DECOMP_SHIFT)-1))];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
475 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
476
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
477 /* high byte is number of hex bytes (usually one or two), low byte
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
478 is prefix code (from*/
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
479 *count = decomp_data[*index] >> 8;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
480 *prefix = decomp_data[*index] & 255;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
481
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
482 (*index)++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
483 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
484
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
485 #define SBase 0xAC00
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
486 #define LBase 0x1100
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
487 #define VBase 0x1161
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
488 #define TBase 0x11A7
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
489 #define LCount 19
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
490 #define VCount 21
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
491 #define TCount 28
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
492 #define NCount (VCount*TCount)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
493 #define SCount (LCount*NCount)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
494
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
495 static PyObject*
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
496 nfd_nfkd(PyObject *self, PyObject *input, int k)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
497 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
498 PyObject *result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
499 Py_UNICODE *i, *end, *o;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
500 /* Longest decomposition in Unicode 3.2: U+FDFA */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
501 Py_UNICODE stack[20];
37664
d13ded9165ec Support NFD of very long strings.
Martin v. Löwis <martin@v.loewis.de>
parents: 37556
diff changeset
502 Py_ssize_t space, isize;
d13ded9165ec Support NFD of very long strings.
Martin v. Löwis <martin@v.loewis.de>
parents: 37556
diff changeset
503 int index, prefix, count, stackptr;
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
504 unsigned char prev, cur;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
505
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
506 stackptr = 0;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
507 isize = PyUnicode_GET_SIZE(input);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
508 /* Overallocate atmost 10 characters. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
509 space = (isize > 10 ? 10 : isize) + isize;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
510 result = PyUnicode_FromUnicode(NULL, space);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
511 if (!result)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
512 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
513 i = PyUnicode_AS_UNICODE(input);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
514 end = i + isize;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
515 o = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
516
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
517 while (i < end) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
518 stack[stackptr++] = *i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
519 while(stackptr) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
520 Py_UNICODE code = stack[--stackptr];
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
521 /* Hangul Decomposition adds three characters in
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
522 a single step, so we need atleast that much room. */
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
523 if (space < 3) {
47527
52294af4b663 This reverts r63675 based on the discussion in this thread:
Gregory P. Smith <greg@mad-scientist.com>
parents: 47451
diff changeset
524 Py_ssize_t newsize = PyString_GET_SIZE(result) + 10;
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
525 space += 10;
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
526 if (PyUnicode_Resize(&result, newsize) == -1)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
527 return NULL;
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
528 o = PyUnicode_AS_UNICODE(result) + newsize - space;
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
529 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
530 /* Hangul Decomposition. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
531 if (SBase <= code && code < (SBase+SCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
532 int SIndex = code - SBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
533 int L = LBase + SIndex / NCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
534 int V = VBase + (SIndex % NCount) / TCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
535 int T = TBase + SIndex % TCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
536 *o++ = L;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
537 *o++ = V;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
538 space -= 2;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
539 if (T != TBase) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
540 *o++ = T;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
541 space --;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
542 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
543 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
544 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
545 /* normalization changes */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
546 if (self) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
547 Py_UCS4 value = ((PreviousDBVersion*)self)->normalization(code);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
548 if (value != 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
549 stack[stackptr++] = value;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
550 continue;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
551 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
552 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
553
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
554 /* Other decompositions. */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
555 get_decomp_record(self, code, &index, &prefix, &count);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
556
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
557 /* Copy character if it is not decomposable, or has a
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
558 compatibility decomposition, but we do NFD. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
559 if (!count || (prefix && !k)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
560 *o++ = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
561 space--;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
562 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
563 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
564 /* Copy decomposition onto the stack, in reverse
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
565 order. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
566 while(count) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
567 code = decomp_data[index + (--count)];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
568 stack[stackptr++] = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
569 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
570 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
571 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
572
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
573 /* Drop overallocation. Cannot fail. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
574 PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
575
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
576 /* Sort canonically. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
577 i = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
578 prev = _getrecord_ex(*i)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
579 end = i + PyUnicode_GET_SIZE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
580 for (i++; i < end; i++) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
581 cur = _getrecord_ex(*i)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
582 if (prev == 0 || cur == 0 || prev <= cur) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
583 prev = cur;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
584 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
585 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
586 /* Non-canonical order. Need to switch *i with previous. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
587 o = i - 1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
588 while (1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
589 Py_UNICODE tmp = o[1];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
590 o[1] = o[0];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
591 o[0] = tmp;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
592 o--;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
593 if (o < PyUnicode_AS_UNICODE(result))
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
594 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
595 prev = _getrecord_ex(*o)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
596 if (prev == 0 || prev <= cur)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
597 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
598 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
599 prev = _getrecord_ex(*i)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
600 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
601 return result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
602 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
603
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
604 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
605 find_nfc_index(PyObject *self, struct reindex* nfc, Py_UNICODE code)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
606 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
607 int index;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
608 for (index = 0; nfc[index].start; index++) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
609 int start = nfc[index].start;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
610 if (code < start)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
611 return -1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
612 if (code <= start + nfc[index].count) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
613 int delta = code - start;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
614 return nfc[index].index + delta;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
615 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
616 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
617 return -1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
618 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
619
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
620 static PyObject*
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
621 nfc_nfkc(PyObject *self, PyObject *input, int k)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
622 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
623 PyObject *result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
624 Py_UNICODE *i, *i1, *o, *end;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
625 int f,l,index,index1,comb;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
626 Py_UNICODE code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
627 Py_UNICODE *skipped[20];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
628 int cskipped = 0;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
629
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
630 result = nfd_nfkd(self, input, k);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
631 if (!result)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
632 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
633
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
634 /* We are going to modify result in-place.
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
635 If nfd_nfkd is changed to sometimes return the input,
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
636 this code needs to be reviewed. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
637 assert(result != input);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
638
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
639 i = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
640 end = i + PyUnicode_GET_SIZE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
641 o = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
642
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
643 again:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
644 while (i < end) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
645 for (index = 0; index < cskipped; index++) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
646 if (skipped[index] == i) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
647 /* *i character is skipped.
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
648 Remove from list. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
649 skipped[index] = skipped[cskipped-1];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
650 cskipped--;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
651 i++;
26571
cb8cfadf75a2 Remove C++ comment.
Martin v. Löwis <martin@v.loewis.de>
parents: 26397
diff changeset
652 goto again; /* continue while */
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
653 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
654 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
655 /* Hangul Composition. We don't need to check for <LV,T>
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
656 pairs, since we always have decomposed data. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
657 if (LBase <= *i && *i < (LBase+LCount) &&
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
658 i + 1 < end &&
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
659 VBase <= i[1] && i[1] <= (VBase+VCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
660 int LIndex, VIndex;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
661 LIndex = i[0] - LBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
662 VIndex = i[1] - VBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
663 code = SBase + (LIndex*VCount+VIndex)*TCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
664 i+=2;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
665 if (i < end &&
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
666 TBase <= *i && *i <= (TBase+TCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
667 code += *i-TBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
668 i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
669 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
670 *o++ = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
671 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
672 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
673
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
674 f = find_nfc_index(self, nfc_first, *i);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
675 if (f == -1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
676 *o++ = *i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
677 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
678 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
679 /* Find next unblocked character. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
680 i1 = i+1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
681 comb = 0;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
682 while (i1 < end) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
683 int comb1 = _getrecord_ex(*i1)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
684 if (comb1 && comb == comb1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
685 /* Character is blocked. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
686 i1++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
687 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
688 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
689 l = find_nfc_index(self, nfc_last, *i1);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
690 /* *i1 cannot be combined with *i. If *i1
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
691 is a starter, we don't need to look further.
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
692 Otherwise, record the combining class. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
693 if (l == -1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
694 not_combinable:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
695 if (comb1 == 0)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
696 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
697 comb = comb1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
698 i1++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
699 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
700 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
701 index = f*TOTAL_LAST + l;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
702 index1 = comp_index[index >> COMP_SHIFT];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
703 code = comp_data[(index1<<COMP_SHIFT)+
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
704 (index&((1<<COMP_SHIFT)-1))];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
705 if (code == 0)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
706 goto not_combinable;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
707
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
708 /* Replace the original character. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
709 *i = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
710 /* Mark the second character unused. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
711 skipped[cskipped++] = i1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
712 i1++;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
713 f = find_nfc_index(self, nfc_first, *i);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
714 if (f == -1)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
715 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
716 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
717 *o++ = *i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
718 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
719 if (o != end)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
720 PyUnicode_Resize(&result, o - PyUnicode_AS_UNICODE(result));
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
721 return result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
722 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
723
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
724 PyDoc_STRVAR(unicodedata_normalize__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
725 "normalize(form, unistr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
726 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
727 Return the normal form 'form' for the Unicode string unistr. Valid\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
728 values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
729
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
730 static PyObject*
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
731 unicodedata_normalize(PyObject *self, PyObject *args)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
732 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
733 char *form;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
734 PyObject *input;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
735
32680
d3d426483d71 Fix typo.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 31930
diff changeset
736 if(!PyArg_ParseTuple(args, "sO!:normalize",
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
737 &form, &PyUnicode_Type, &input))
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
738 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
739
31930
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
740 if (PyUnicode_GetSize(input) == 0) {
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
741 /* Special case empty input strings, since resizing
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
742 them later would cause internal errors. */
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
743 Py_INCREF(input);
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
744 return input;
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
745 }
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
746
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
747 if (strcmp(form, "NFC") == 0)
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
748 return nfc_nfkc(self, input, 0);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
749 if (strcmp(form, "NFKC") == 0)
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
750 return nfc_nfkc(self, input, 1);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
751 if (strcmp(form, "NFD") == 0)
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
752 return nfd_nfkd(self, input, 0);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
753 if (strcmp(form, "NFKD") == 0)
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
754 return nfd_nfkd(self, input, 1);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
755 PyErr_SetString(PyExc_ValueError, "invalid normalization form");
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
756 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
757 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
758
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
759 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
760 /* unicode character name tables */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
761
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
762 /* data file generated by Tools/unicode/makeunicodedata.py */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
763 #include "unicodename_db.h"
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
764
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
765 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
766 /* database code (cut and pasted from the unidb package) */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
767
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
768 static unsigned long
16666
e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS
Fredrik Lundh <fredrik@pythonware.com>
parents: 16662
diff changeset
769 _gethash(const char *s, int len, int scale)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
770 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
771 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
772 unsigned long h = 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
773 unsigned long ix;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
774 for (i = 0; i < len; i++) {
37556
9bda5ba41677 Get rid of warnings about using chars as subscripts
Neal Norwitz <nnorwitz@gmail.com>
parents: 36939
diff changeset
775 h = (h * scale) + (unsigned char) toupper(Py_CHARMASK(s[i]));
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
776 ix = h & 0xff000000;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
777 if (ix)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
778 h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
779 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
780 return h;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
781 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
782
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
783 static char *hangul_syllables[][3] = {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
784 { "G", "A", "" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
785 { "GG", "AE", "G" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
786 { "N", "YA", "GG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
787 { "D", "YAE", "GS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
788 { "DD", "EO", "N", },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
789 { "R", "E", "NJ" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
790 { "M", "YEO", "NH" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
791 { "B", "YE", "D" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
792 { "BB", "O", "L" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
793 { "S", "WA", "LG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
794 { "SS", "WAE", "LM" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
795 { "", "OE", "LB" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
796 { "J", "YO", "LS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
797 { "JJ", "U", "LT" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
798 { "C", "WEO", "LP" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
799 { "K", "WE", "LH" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
800 { "T", "WI", "M" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
801 { "P", "YU", "B" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
802 { "H", "EU", "BS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
803 { 0, "YI", "S" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
804 { 0, "I", "SS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
805 { 0, 0, "NG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
806 { 0, 0, "J" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
807 { 0, 0, "C" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
808 { 0, 0, "K" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
809 { 0, 0, "T" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
810 { 0, 0, "P" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
811 { 0, 0, "H" }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
812 };
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
813
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
814 static int
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
815 is_unified_ideograph(Py_UCS4 code)
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
816 {
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
817 return (
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
818 (0x3400 <= code && code <= 0x4DB5) || /* CJK Ideograph Extension A */
36939
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
819 (0x4E00 <= code && code <= 0x9FBB) || /* CJK Ideograph */
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
820 (0x20000 <= code && code <= 0x2A6D6));/* CJK Ideograph Extension B */
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
821 }
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
822
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
823 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
824 _getucname(PyObject *self, Py_UCS4 code, char* buffer, int buflen)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
825 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
826 int offset;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
827 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
828 int word;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
829 unsigned char* w;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
830
36939
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
831 if (code >= 0x110000)
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
832 return 0;
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
833
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
834 if (self) {
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
835 const change_record *old = get_old_record(self, code);
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
836 if (old->category_changed == 0) {
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
837 /* unassigned */
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
838 return 0;
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
839 }
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
840 }
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
841
26374
135bb11215a0 Fix off-by-one error.
Martin v. Löwis <martin@v.loewis.de>
parents: 26370
diff changeset
842 if (SBase <= code && code < SBase+SCount) {
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
843 /* Hangul syllable. */
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
844 int SIndex = code - SBase;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
845 int L = SIndex / NCount;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
846 int V = (SIndex % NCount) / TCount;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
847 int T = SIndex % TCount;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
848
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
849 if (buflen < 27)
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
850 /* Worst case: HANGUL SYLLABLE <10chars>. */
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
851 return 0;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
852 strcpy(buffer, "HANGUL SYLLABLE ");
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
853 buffer += 16;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
854 strcpy(buffer, hangul_syllables[L][0]);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
855 buffer += strlen(hangul_syllables[L][0]);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
856 strcpy(buffer, hangul_syllables[V][1]);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
857 buffer += strlen(hangul_syllables[V][1]);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
858 strcpy(buffer, hangul_syllables[T][2]);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
859 buffer += strlen(hangul_syllables[T][2]);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
860 *buffer = '\0';
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
861 return 1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
862 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
863
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
864 if (is_unified_ideograph(code)) {
26376
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
865 if (buflen < 28)
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
866 /* Worst case: CJK UNIFIED IDEOGRAPH-20000 */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
867 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
868 sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code);
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
869 return 1;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
870 }
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
871
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
872 /* get offset into phrasebook */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
873 offset = phrasebook_offset1[(code>>phrasebook_shift)];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
874 offset = phrasebook_offset2[(offset<<phrasebook_shift) +
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
875 (code&((1<<phrasebook_shift)-1))];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
876 if (!offset)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
877 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
878
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
879 i = 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
880
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
881 for (;;) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
882 /* get word index */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
883 word = phrasebook[offset] - phrasebook_short;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
884 if (word >= 0) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
885 word = (word << 8) + phrasebook[offset+1];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
886 offset += 2;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
887 } else
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
888 word = phrasebook[offset++];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
889 if (i) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
890 if (i > buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
891 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
892 buffer[i++] = ' ';
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
893 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
894 /* copy word string from lexicon. the last character in the
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
895 word has bit 7 set. the last word in a string ends with
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
896 0x80 */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
897 w = lexicon + lexicon_offset[word];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
898 while (*w < 128) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
899 if (i >= buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
900 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
901 buffer[i++] = *w++;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
902 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
903 if (i >= buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
904 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
905 buffer[i++] = *w & 127;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
906 if (*w == 128)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
907 break; /* end of word */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
908 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
909
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
910 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
911 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
912
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
913 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
914 _cmpname(PyObject *self, int code, const char* name, int namelen)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
915 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
916 /* check if code corresponds to the given name */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
917 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
918 char buffer[NAME_MAXLEN];
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
919 if (!_getucname(self, code, buffer, sizeof(buffer)))
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
920 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
921 for (i = 0; i < namelen; i++) {
37556
9bda5ba41677 Get rid of warnings about using chars as subscripts
Neal Norwitz <nnorwitz@gmail.com>
parents: 36939
diff changeset
922 if (toupper(Py_CHARMASK(name[i])) != buffer[i])
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
923 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
924 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
925 return buffer[namelen] == '\0';
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
926 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
927
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
928 static void
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
929 find_syllable(const char *str, int *len, int *pos, int count, int column)
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
930 {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
931 int i, len1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
932 *len = -1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
933 for (i = 0; i < count; i++) {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
934 char *s = hangul_syllables[i][column];
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
935 len1 = strlen(s);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
936 if (len1 <= *len)
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
937 continue;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
938 if (strncmp(str, s, len1) == 0) {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
939 *len = len1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
940 *pos = i;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
941 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
942 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
943 if (*len == -1) {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
944 *len = 0;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
945 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
946 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
947
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
948 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
949 _getcode(PyObject* self, const char* name, int namelen, Py_UCS4* code)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
950 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
951 unsigned int h, v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
952 unsigned int mask = code_size-1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
953 unsigned int i, incr;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
954
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
955 /* Check for hangul syllables. */
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
956 if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) {
36778
cd7d2919178c Remove gcc (4.0.x) warning about uninitialized value by explicitly setting
Thomas Wouters <thomas@python.org>
parents: 35587
diff changeset
957 int len, L = -1, V = -1, T = -1;
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
958 const char *pos = name + 16;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
959 find_syllable(pos, &len, &L, LCount, 0);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
960 pos += len;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
961 find_syllable(pos, &len, &V, VCount, 1);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
962 pos += len;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
963 find_syllable(pos, &len, &T, TCount, 2);
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
964 pos += len;
35587
02a8c0643f5b Patch #1213831: Fix typo in unicodedata._getcode.
Martin v. Löwis <martin@v.loewis.de>
parents: 35016
diff changeset
965 if (L != -1 && V != -1 && T != -1 && pos-name == namelen) {
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
966 *code = SBase + (L*VCount+V)*TCount + T;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
967 return 1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
968 }
26376
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
969 /* Otherwise, it's an illegal syllable name. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
970 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
971 }
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
972
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
973 /* Check for unified ideographs. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
974 if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) {
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
975 /* Four or five hexdigits must follow. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
976 v = 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
977 name += 22;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
978 namelen -= 22;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
979 if (namelen != 4 && namelen != 5)
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
980 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
981 while (namelen--) {
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
982 v *= 16;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
983 if (*name >= '0' && *name <= '9')
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
984 v += *name - '0';
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
985 else if (*name >= 'A' && *name <= 'F')
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
986 v += *name - 'A' + 10;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
987 else
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
988 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
989 name++;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
990 }
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
991 if (!is_unified_ideograph(v))
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
992 return 0;
26376
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
993 *code = v;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
994 return 1;
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
995 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
996
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
997 /* the following is the same as python's dictionary lookup, with
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
998 only minor changes. see the makeunicodedata script for more
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
999 details */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1000
16666
e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS
Fredrik Lundh <fredrik@pythonware.com>
parents: 16662
diff changeset
1001 h = (unsigned int) _gethash(name, namelen, code_magic);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1002 i = (~h) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1003 v = code_hash[i];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1004 if (!v)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1005 return 0;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1006 if (_cmpname(self, v, name, namelen)) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1007 *code = v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1008 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1009 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1010 incr = (h ^ (h >> 3)) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1011 if (!incr)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1012 incr = mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1013 for (;;) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1014 i = (i + incr) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1015 v = code_hash[i];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1016 if (!v)
16662
89c1ccef5ce8 stupid typo (for some reason, this only caused problems on OpenVMS).
Fredrik Lundh <fredrik@pythonware.com>
parents: 16200
diff changeset
1017 return 0;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1018 if (_cmpname(self, v, name, namelen)) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1019 *code = v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1020 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1021 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1022 incr = incr << 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1023 if (incr > mask)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1024 incr = incr ^ code_poly;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1025 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1026 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1027
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1028 static const _PyUnicode_Name_CAPI hashAPI =
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1029 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1030 sizeof(_PyUnicode_Name_CAPI),
23810
01e4b698ff7e _Py prefix is verboten for static entry points
Andrew MacIntyre <andymac@bullseye.apana.org.au>
parents: 22781
diff changeset
1031 _getucname,
16666
e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS
Fredrik Lundh <fredrik@pythonware.com>
parents: 16662
diff changeset
1032 _getcode
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1033 };
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1034
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1035 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1036 /* Python bindings */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1037
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1038 PyDoc_STRVAR(unicodedata_name__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1039 "name(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1040 Returns the name assigned to the Unicode character unichr as a\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1041 string. If no name is defined, default is returned, or, if not\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1042 given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1043
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1044 static PyObject *
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1045 unicodedata_name(PyObject* self, PyObject* args)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1046 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1047 char name[NAME_MAXLEN];
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
1048 Py_UCS4 c;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1049
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1050 PyUnicodeObject* v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1051 PyObject* defobj = NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1052 if (!PyArg_ParseTuple(args, "O!|O:name", &PyUnicode_Type, &v, &defobj))
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1053 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1054
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
1055 c = getuchar(v);
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
1056 if (c == (Py_UCS4)-1)
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
1057 return NULL;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1058
47451
ed471a8009a6 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 47294
diff changeset
1059 if (!_getucname(self, c, name, sizeof(name))) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1060 if (defobj == NULL) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1061 PyErr_SetString(PyExc_ValueError, "no such name");
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1062 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1063 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1064 else {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1065 Py_INCREF(defobj);
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1066 return defobj;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1067 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1068 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1069
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1070 return Py_BuildValue("s", name);
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1071 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1072
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1073 PyDoc_STRVAR(unicodedata_lookup__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1074 "lookup(name)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1075 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1076 Look up character by name. If a character with the\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1077 given name is found, return the corresponding Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1078 character. If not found, KeyError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1079
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1080 static PyObject *
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1081 unicodedata_lookup(PyObject* self, PyObject* args)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1082 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1083 Py_UCS4 code;
42387
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1084 Py_UNICODE str[2];
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1085
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1086 char* name;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1087 int namelen;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1088 if (!PyArg_ParseTuple(args, "s#:lookup", &name, &namelen))
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1089 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1090
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1091 if (!_getcode(self, name, namelen, &code)) {
42387
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1092 PyErr_Format(PyExc_KeyError, "undefined character name '%s'",
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1093 name);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1094 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1095 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1096
42387
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1097 #ifndef Py_UNICODE_WIDE
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1098 if (code >= 0x10000) {
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1099 str[0] = 0xd800 + ((code - 0x10000) >> 10);
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1100 str[1] = 0xdc00 + ((code - 0x10000) & 0x3ff);
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1101 return PyUnicode_FromUnicode(str, 2);
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1102 }
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1103 #endif
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1104 str[0] = (Py_UNICODE) code;
42387
85b7f40e1c1c Bug #1704793: Return UTF-16 pair if unicodedata.lookup cannot
Martin v. Löwis <martin@v.loewis.de>
parents: 42333
diff changeset
1105 return PyUnicode_FromUnicode(str, 1);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1106 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1107
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1108 /* XXX Add doc strings. */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1109
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1110 static PyMethodDef unicodedata_functions[] = {
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1111 {"decimal", unicodedata_decimal, METH_VARARGS, unicodedata_decimal__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1112 {"digit", unicodedata_digit, METH_VARARGS, unicodedata_digit__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1113 {"numeric", unicodedata_numeric, METH_VARARGS, unicodedata_numeric__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1114 {"category", unicodedata_category, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1115 unicodedata_category__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1116 {"bidirectional", unicodedata_bidirectional, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1117 unicodedata_bidirectional__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1118 {"combining", unicodedata_combining, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1119 unicodedata_combining__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1120 {"mirrored", unicodedata_mirrored, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1121 unicodedata_mirrored__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1122 {"east_asian_width", unicodedata_east_asian_width, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1123 unicodedata_east_asian_width__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1124 {"decomposition", unicodedata_decomposition, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1125 unicodedata_decomposition__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1126 {"name", unicodedata_name, METH_VARARGS, unicodedata_name__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1127 {"lookup", unicodedata_lookup, METH_VARARGS, unicodedata_lookup__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1128 {"normalize", unicodedata_normalize, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1129 unicodedata_normalize__doc__},
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1130 {NULL, NULL} /* sentinel */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1131 };
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1132
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1133 static PyTypeObject UCD_Type = {
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1134 /* The ob_type field must be initialized in the module init function
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1135 * to be portable to Windows without using C++. */
42333
6620a510e9a9 PEP 3123: Provide forward compatibility with Python 3.0, while keeping
Martin v. Löwis <martin@v.loewis.de>
parents: 40319
diff changeset
1136 PyVarObject_HEAD_INIT(NULL, 0)
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1137 "unicodedata.UCD", /*tp_name*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1138 sizeof(PreviousDBVersion), /*tp_basicsize*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1139 0, /*tp_itemsize*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1140 /* methods */
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1141 (destructor)PyObject_Del, /*tp_dealloc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1142 0, /*tp_print*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1143 0, /*tp_getattr*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1144 0, /*tp_setattr*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1145 0, /*tp_compare*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1146 0, /*tp_repr*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1147 0, /*tp_as_number*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1148 0, /*tp_as_sequence*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1149 0, /*tp_as_mapping*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1150 0, /*tp_hash*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1151 0, /*tp_call*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1152 0, /*tp_str*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1153 PyObject_GenericGetAttr,/*tp_getattro*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1154 0, /*tp_setattro*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1155 0, /*tp_as_buffer*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1156 Py_TPFLAGS_DEFAULT, /*tp_flags*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1157 0, /*tp_doc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1158 0, /*tp_traverse*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1159 0, /*tp_clear*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1160 0, /*tp_richcompare*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1161 0, /*tp_weaklistoffset*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1162 0, /*tp_iter*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1163 0, /*tp_iternext*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1164 unicodedata_functions, /*tp_methods*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1165 DB_members, /*tp_members*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1166 0, /*tp_getset*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1167 0, /*tp_base*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1168 0, /*tp_dict*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1169 0, /*tp_descr_get*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1170 0, /*tp_descr_set*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1171 0, /*tp_dictoffset*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1172 0, /*tp_init*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1173 0, /*tp_alloc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1174 0, /*tp_new*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1175 0, /*tp_free*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1176 0, /*tp_is_gc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1177 };
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1178
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1179 PyDoc_STRVAR(unicodedata_docstring,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1180 "This module provides access to the Unicode Character Database which\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1181 defines character properties for all Unicode characters. The data in\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1182 this database is based on the UnicodeData.txt file version\n\
49124
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
1183 5.1.0 which is publically available from ftp://ftp.unicode.org/.\n\
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1184 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1185 The module uses the same names and symbols as defined by the\n\
49124
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
1186 UnicodeData File Format 5.1.0 (see\n\
c37c7dbd7880 Issue #3811: The Unicode database was updated to 5.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 47527
diff changeset
1187 http://www.unicode.org/Public/5.1.0/ucd/UCD.html).");
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1188
24363
979ff5d0ae07 Replace DL_IMPORT with PyMODINIT_FUNC and remove "/export:init..." link
Mark Hammond <mhammond@skippinet.com.au>
parents: 23832
diff changeset
1189 PyMODINIT_FUNC
13651
8a063423e0b7 Bunch of minor ANSIfications: 'void initfunc()' -> 'void initfunc(void)',
Thomas Wouters <thomas@python.org>
parents: 12196
diff changeset
1190 initunicodedata(void)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1191 {
22781
ec67fce3fa3c Remove direct manipulation of the module dict.
Fred Drake <fdrake@acm.org>
parents: 22114
diff changeset
1192 PyObject *m, *v;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1193
44350
26384b8180c7 #1629: Renamed Py_Size, Py_Type and Py_Refcnt to Py_SIZE, Py_TYPE and Py_REFCNT. Macros for b/w compatibility are available.
Christian Heimes <christian@cheimes.de>
parents: 42387
diff changeset
1194 Py_TYPE(&UCD_Type) = &PyType_Type;
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1195
16977
0b4c57a25977 Be a bit more strict in setting up the export of the C API for this
Fred Drake <fdrake@acm.org>
parents: 16666
diff changeset
1196 m = Py_InitModule3(
0b4c57a25977 Be a bit more strict in setting up the export of the C API for this
Fred Drake <fdrake@acm.org>
parents: 16666
diff changeset
1197 "unicodedata", unicodedata_functions, unicodedata_docstring);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1198 if (!m)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1199 return;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1200
26397
b27c1cad77ea Add unidata_version. Bump generator version number.
Martin v. Löwis <martin@v.loewis.de>
parents: 26380
diff changeset
1201 PyModule_AddStringConstant(m, "unidata_version", UNIDATA_VERSION);
36928
11f8b5b8ce20 Fix refcounting bug.
Martin v. Löwis <martin@v.loewis.de>
parents: 36927
diff changeset
1202 Py_INCREF(&UCD_Type);
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1203 PyModule_AddObject(m, "UCD", (PyObject*)&UCD_Type);
26397
b27c1cad77ea Add unidata_version. Bump generator version number.
Martin v. Löwis <martin@v.loewis.de>
parents: 26380
diff changeset
1204
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1205 /* Previous versions */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1206 v = new_previous_version("3.2.0", get_change_3_2_0, normalization_3_2_0);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1207 if (v != NULL)
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1208 PyModule_AddObject(m, "ucd_3_2_0", v);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1209
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1210 /* Export C API */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1211 v = PyCObject_FromVoidPtr((void *) &hashAPI, NULL);
22781
ec67fce3fa3c Remove direct manipulation of the module dict.
Fred Drake <fdrake@acm.org>
parents: 22114
diff changeset
1212 if (v != NULL)
ec67fce3fa3c Remove direct manipulation of the module dict.
Fred Drake <fdrake@acm.org>
parents: 22114
diff changeset
1213 PyModule_AddObject(m, "ucnhash_CAPI", v);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1214 }
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1215
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1216 /*
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1217 Local variables:
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1218 c-basic-offset: 4
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
1219 indent-tabs-mode: nil
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1220 End:
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1221 */