Mercurial > cpython-lru_cache-weakref

annotate Modules/unicodedata.c @ 70817:dd775b1561ba

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix markup.
author Georg Brandl <georg@python.org>
date Tue, 14 Jun 2011 21:09:55 +0200
parents 3c9232a684ae
children fe48e2b3dbee 8beaa9a37387
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1 /* ------------------------------------------------------------------------
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
2
59927
65ff526e05eb Merged revisions 79314 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents: 59324
diff changeset
3 unicodedata -- Provides access to the Unicode 5.2 data base.
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
4
59927
65ff526e05eb Merged revisions 79314 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents: 59324
diff changeset
5 Data was extracted from the Unicode 5.2 UnicodeData.txt file.
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
6
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14818
diff changeset
7 Written by Marc-Andre Lemburg (mal@lemburg.com).
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14818
diff changeset
8 Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
64423
e9cd15d539e1 Welcome to the UTF-8 world.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 61466
diff changeset
9 Modified by Martin v. Löwis (martin@v.loewis.de)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
14835
96d31a143857 unicode database compression, step 2:
Fredrik Lundh <fredrik@pythonware.com>
parents: 14818
diff changeset
11 Copyright (c) Corporation for National Research Initiatives.
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
12
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 ------------------------------------------------------------------------ */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
14
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 #include "Python.h"
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
16 #include "ucnhash.h"
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
17 #include "structmember.h"
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
18
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
19 /* character properties */
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
20
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
21 typedef struct {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
22 const unsigned char category; /* index into
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
23 _PyUnicode_CategoryNames */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
24 const unsigned char combining; /* combining class value 0 - 255 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
25 const unsigned char bidirectional; /* index into
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
26 _PyUnicode_BidirectionalNames */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
27 const unsigned char mirrored; /* true if mirrored in bidir mode */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
28 const unsigned char east_asian_width; /* index into
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
29 _PyUnicode_EastAsianWidth */
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
30 const unsigned char normalization_quick_check; /* see is_normalized() */
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
31 } _PyUnicode_DatabaseRecord;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
32
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
33 typedef struct change_record {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
34 /* sequence of fields should be the same as in merge_old_version */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
35 const unsigned char bidir_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
36 const unsigned char category_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
37 const unsigned char decimal_changed;
49125
dc60293b29d5 Merged revisions 66362 via svnmerge from
Martin v. Löwis <martin@v.loewis.de>
parents: 47580
diff changeset
38 const unsigned char mirrored_changed;
56408
6c91d1803182 Merged revisions 75272-75273 via svnmerge from
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 53816
diff changeset
39 const double numeric_changed;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
40 } change_record;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
41
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
42 /* data file generated by Tools/unicode/makeunicodedata.py */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
43 #include "unicodedata_db.h"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
44
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
45 static const _PyUnicode_DatabaseRecord*
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
46 _getrecord_ex(Py_UCS4 code)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
47 {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
48 int index;
28100
54b35da9879e Fix SF bug #694816, remove comparison of unsigned value < 0
Neal Norwitz <nnorwitz@gmail.com>
parents: 26571
diff changeset
49 if (code >= 0x110000)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
50 index = 0;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
51 else {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
52 index = index1[(code>>SHIFT)];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
53 index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
54 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
55
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
56 return &_PyUnicode_Database_Records[index];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
57 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
58
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
59 /* ------------- Previous-version API ------------------------------------- */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
60 typedef struct previous_version {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
61 PyObject_HEAD
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
62 const char *name;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
63 const change_record* (*getrecord)(Py_UCS4);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
64 Py_UCS4 (*normalization)(Py_UCS4);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
65 } PreviousDBVersion;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
66
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
67 #define get_old_record(self, v) ((((PreviousDBVersion*)self)->getrecord)(v))
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
68
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
69 static PyMemberDef DB_members[] = {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
70 {"unidata_version", T_STRING, offsetof(PreviousDBVersion, name), READONLY},
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
71 {NULL}
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
72 };
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
73
40450
cc992d75d5b3 Four months of trunk changes (including a few releases...)
Thomas Wouters <thomas@python.org>
parents: 39610
diff changeset
74 /* forward declaration */
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
75 static PyTypeObject UCD_Type;
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
76 #define UCD_Check(o) (Py_TYPE(o)==&UCD_Type)
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
77
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
78 static PyObject*
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
79 new_previous_version(const char*name, const change_record* (*getrecord)(Py_UCS4),
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
80 Py_UCS4 (*normalization)(Py_UCS4))
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
81 {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
82 PreviousDBVersion *self;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
83 self = PyObject_New(PreviousDBVersion, &UCD_Type);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
84 if (self == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
85 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
86 self->name = name;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
87 self->getrecord = getrecord;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
88 self->normalization = normalization;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
89 return (PyObject*)self;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
90 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
91
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
92
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
93 static Py_UCS4 getuchar(PyUnicodeObject *obj)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
94 {
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
95 Py_UNICODE *v = PyUnicode_AS_UNICODE(obj);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
96
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
97 if (PyUnicode_GET_SIZE(obj) == 1)
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
98 return *v;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
99 #ifndef Py_UNICODE_WIDE
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
100 else if ((PyUnicode_GET_SIZE(obj) == 2) &&
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
101 (0xD800 <= v[0] && v[0] <= 0xDBFF) &&
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
102 (0xDC00 <= v[1] && v[1] <= 0xDFFF))
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
103 return (((v[0] & 0x3FF)<<10) | (v[1] & 0x3FF)) + 0x10000;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
104 #endif
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
105 PyErr_SetString(PyExc_TypeError,
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
106 "need a single Unicode character as parameter");
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
107 return (Py_UCS4)-1;
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
108 }
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
109
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
110 /* --- Module API --------------------------------------------------------- */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
111
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
112 PyDoc_STRVAR(unicodedata_decimal__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
113 "decimal(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
114 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
115 Returns the decimal value assigned to the Unicode character unichr\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
116 as integer. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
117 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
118
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
119 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
120 unicodedata_decimal(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
121 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
122 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
123 PyObject *defobj = NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
124 int have_old = 0;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
125 long rc;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
126 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
127
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
128 if (!PyArg_ParseTuple(args, "O!|O:decimal", &PyUnicode_Type, &v, &defobj))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
129 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
130 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
131 if (c == (Py_UCS4)-1)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
132 return NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
133
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
134 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
135 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
136 if (old->category_changed == 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
137 /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
138 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
139 rc = -1;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
140 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
141 else if (old->decimal_changed != 0xFF) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
142 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
143 rc = old->decimal_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
144 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
145 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
146
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
147 if (!have_old)
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
148 rc = Py_UNICODE_TODECIMAL(c);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
149 if (rc < 0) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
150 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
151 PyErr_SetString(PyExc_ValueError,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
152 "not a decimal");
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
153 return NULL;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
154 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
155 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
156 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
157 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
158 }
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
159 }
44099
f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h
Christian Heimes <christian@cheimes.de>
parents: 42466
diff changeset
160 return PyLong_FromLong(rc);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
161 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
162
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
163 PyDoc_STRVAR(unicodedata_digit__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
164 "digit(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
165 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
166 Returns the digit value assigned to the Unicode character unichr as\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
167 integer. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
168 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
169
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
170 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
171 unicodedata_digit(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
172 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
173 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
174 PyObject *defobj = NULL;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
175 long rc;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
176 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
177
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
178 if (!PyArg_ParseTuple(args, "O!|O:digit", &PyUnicode_Type, &v, &defobj))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
179 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
180 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
181 if (c == (Py_UCS4)-1)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
182 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
183 rc = Py_UNICODE_TODIGIT(c);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
184 if (rc < 0) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
185 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
186 PyErr_SetString(PyExc_ValueError, "not a digit");
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
187 return NULL;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
188 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
189 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
190 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
191 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
192 }
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
193 }
44099
f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h
Christian Heimes <christian@cheimes.de>
parents: 42466
diff changeset
194 return PyLong_FromLong(rc);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
195 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
196
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
197 PyDoc_STRVAR(unicodedata_numeric__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
198 "numeric(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
199 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
200 Returns the numeric value assigned to the Unicode character unichr\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
201 as float. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
202 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
203
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
204 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
205 unicodedata_numeric(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
206 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
207 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
208 PyObject *defobj = NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
209 int have_old = 0;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
210 double rc;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
211 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
212
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
213 if (!PyArg_ParseTuple(args, "O!|O:numeric", &PyUnicode_Type, &v, &defobj))
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
214 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
215 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
216 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
217 return NULL;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
218
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
219 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
220 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
221 if (old->category_changed == 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
222 /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
223 have_old = 1;
38453
29ae5e0fc348 Much-needed merge (using svnmerge.py this time) of trunk changes into p3yk.
Thomas Wouters <thomas@python.org>
parents: 37879
diff changeset
224 rc = -1.0;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
225 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
226 else if (old->decimal_changed != 0xFF) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
227 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
228 rc = old->decimal_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
229 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
230 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
231
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
232 if (!have_old)
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
233 rc = Py_UNICODE_TONUMERIC(c);
38453
29ae5e0fc348 Much-needed merge (using svnmerge.py this time) of trunk changes into p3yk.
Thomas Wouters <thomas@python.org>
parents: 37879
diff changeset
234 if (rc == -1.0) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
235 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
236 PyErr_SetString(PyExc_ValueError, "not a numeric character");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
237 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
238 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
239 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
240 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
241 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
242 }
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
243 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
244 return PyFloat_FromDouble(rc);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
245 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
246
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
247 PyDoc_STRVAR(unicodedata_category__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
248 "category(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
249 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
250 Returns the general category assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
251 unichr as string.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
252
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
253 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
254 unicodedata_category(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
255 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
256 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
257 int index;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
258 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
259
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
260 if (!PyArg_ParseTuple(args, "O!:category",
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
261 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
262 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
263 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
264 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
265 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
266 index = (int) _getrecord_ex(c)->category;
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
267 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
268 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
269 if (old->category_changed != 0xFF)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
270 index = old->category_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
271 }
41944
53963a2f5652 Change category(), bidirectional(), east_asian_width(),
Walter Dörwald <walter@livinglogic.de>
parents: 40450
diff changeset
272 return PyUnicode_FromString(_PyUnicode_CategoryNames[index]);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
273 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
274
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
275 PyDoc_STRVAR(unicodedata_bidirectional__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
276 "bidirectional(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
277 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
278 Returns the bidirectional category assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
279 unichr as string. If no such value is defined, an empty string is\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
280 returned.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
281
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
282 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
283 unicodedata_bidirectional(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
284 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
285 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
286 int index;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
287 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
288
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
289 if (!PyArg_ParseTuple(args, "O!:bidirectional",
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
290 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
291 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
292 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
293 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
294 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
295 index = (int) _getrecord_ex(c)->bidirectional;
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
296 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
297 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
298 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
299 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
300 else if (old->bidir_changed != 0xFF)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
301 index = old->bidir_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
302 }
41944
53963a2f5652 Change category(), bidirectional(), east_asian_width(),
Walter Dörwald <walter@livinglogic.de>
parents: 40450
diff changeset
303 return PyUnicode_FromString(_PyUnicode_BidirectionalNames[index]);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
304 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
305
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
306 PyDoc_STRVAR(unicodedata_combining__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
307 "combining(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
308 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
309 Returns the canonical combining class assigned to the Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
310 character unichr as integer. Returns 0 if no combining class is\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
311 defined.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
312
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
313 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
314 unicodedata_combining(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
315 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
316 PyUnicodeObject *v;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
317 int index;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
318 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
319
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
320 if (!PyArg_ParseTuple(args, "O!:combining",
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
321 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
322 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
323 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
324 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
325 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
326 index = (int) _getrecord_ex(c)->combining;
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
327 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
328 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
329 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
330 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
331 }
44099
f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h
Christian Heimes <christian@cheimes.de>
parents: 42466
diff changeset
332 return PyLong_FromLong(index);
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
333 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
334
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
335 PyDoc_STRVAR(unicodedata_mirrored__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
336 "mirrored(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
337 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
338 Returns the mirrored property assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
339 unichr as integer. Returns 1 if the character has been identified as\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
340 a \"mirrored\" character in bidirectional text, 0 otherwise.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
341
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
342 static PyObject *
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
343 unicodedata_mirrored(PyObject *self, PyObject *args)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
344 {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
345 PyUnicodeObject *v;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
346 int index;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
347 Py_UCS4 c;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
348
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
349 if (!PyArg_ParseTuple(args, "O!:mirrored",
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
350 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
351 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
352 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
353 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
354 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
355 index = (int) _getrecord_ex(c)->mirrored;
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
356 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
357 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
358 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
359 index = 0; /* unassigned */
49125
dc60293b29d5 Merged revisions 66362 via svnmerge from
Martin v. Löwis <martin@v.loewis.de>
parents: 47580
diff changeset
360 else if (old->mirrored_changed != 0xFF)
dc60293b29d5 Merged revisions 66362 via svnmerge from
Martin v. Löwis <martin@v.loewis.de>
parents: 47580
diff changeset
361 index = old->mirrored_changed;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
362 }
44099
f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h
Christian Heimes <christian@cheimes.de>
parents: 42466
diff changeset
363 return PyLong_FromLong(index);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
364 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
365
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
366 PyDoc_STRVAR(unicodedata_east_asian_width__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
367 "east_asian_width(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
368 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
369 Returns the east asian width assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
370 unichr as string.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
371
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
372 static PyObject *
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
373 unicodedata_east_asian_width(PyObject *self, PyObject *args)
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
374 {
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
375 PyUnicodeObject *v;
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
376 int index;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
377 Py_UCS4 c;
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
378
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
379 if (!PyArg_ParseTuple(args, "O!:east_asian_width",
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
380 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
381 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
382 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
383 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
384 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
385 index = (int) _getrecord_ex(c)->east_asian_width;
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
386 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
387 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
388 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
389 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
390 }
41944
53963a2f5652 Change category(), bidirectional(), east_asian_width(),
Walter Dörwald <walter@livinglogic.de>
parents: 40450
diff changeset
391 return PyUnicode_FromString(_PyUnicode_EastAsianWidthNames[index]);
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
392 }
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
393
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
394 PyDoc_STRVAR(unicodedata_decomposition__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
395 "decomposition(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
396 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
397 Returns the character decomposition mapping assigned to the Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
398 character unichr as string. An empty string is returned in case no\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
399 such mapping is defined.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
400
32951
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32680
diff changeset
401 static PyObject *
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
402 unicodedata_decomposition(PyObject *self, PyObject *args)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
403 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
404 PyUnicodeObject *v;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
405 char decomp[256];
67347
3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int
Victor Stinner <victor.stinner@haypocalc.com>
parents: 67088
diff changeset
406 int code, index, count;
3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int
Victor Stinner <victor.stinner@haypocalc.com>
parents: 67088
diff changeset
407 size_t i;
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
408 unsigned int prefix_index;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
409 Py_UCS4 c;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
410
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
411 if (!PyArg_ParseTuple(args, "O!:decomposition",
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
412 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
413 return NULL;
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
414 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
415 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
416 return NULL;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
417
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
418 code = (int)c;
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
419
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
420 if (self && UCD_Check(self)) {
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
421 const change_record *old = get_old_record(self, c);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
422 if (old->category_changed == 0)
41944
53963a2f5652 Change category(), bidirectional(), east_asian_width(),
Walter Dörwald <walter@livinglogic.de>
parents: 40450
diff changeset
423 return PyUnicode_FromString(""); /* unassigned */
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
424 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
425
25852
1f5f6481886d Update to Unicode 3.2 database.
Martin v. Löwis <martin@v.loewis.de>
parents: 24363
diff changeset
426 if (code < 0 || code >= 0x110000)
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
427 index = 0;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
428 else {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
429 index = decomp_index1[(code>>DECOMP_SHIFT)];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
430 index = decomp_index2[(index<<DECOMP_SHIFT)+
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
431 (code&((1<<DECOMP_SHIFT)-1))];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
432 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
433
21112
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
434 /* high byte is number of hex bytes (usually one or two), low byte
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
435 is prefix code (from*/
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
436 count = decomp_data[index] >> 8;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
437
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
438 /* XXX: could allocate the PyString up front instead
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
439 (strlen(prefix) + 5 * count + 1 bytes) */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
440
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
441 /* Based on how index is calculated above and decomp_data is generated
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
442 from Tools/unicode/makeunicodedata.py, it should not be possible
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
443 to overflow decomp_prefix. */
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
444 prefix_index = decomp_data[index] & 255;
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
445 assert(prefix_index < (sizeof(decomp_prefix)/sizeof(*decomp_prefix)));
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
446
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
447 /* copy prefix */
39442
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
448 i = strlen(decomp_prefix[prefix_index]);
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 38453
diff changeset
449 memcpy(decomp, decomp_prefix[prefix_index], i);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
450
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
451 while (count-- > 0) {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
452 if (i)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
453 decomp[i++] = ' ';
67347
3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int
Victor Stinner <victor.stinner@haypocalc.com>
parents: 67088
diff changeset
454 assert(i < sizeof(decomp));
21112
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
455 PyOS_snprintf(decomp + i, sizeof(decomp) - i, "%04X",
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf.
Tim Peters <tim.peters@gmail.com>
parents: 18752
diff changeset
456 decomp_data[++index]);
16090
8ac40d48a581 compress unicode decomposition tables (this saves another 55k)
Fredrik Lundh <fredrik@pythonware.com>
parents: 14835
diff changeset
457 i += strlen(decomp + i);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
458 }
67347
3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int
Victor Stinner <victor.stinner@haypocalc.com>
parents: 67088
diff changeset
459 return PyUnicode_FromStringAndSize(decomp, i);
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
460 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
461
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36939
diff changeset
462 static void
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
463 get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *count)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
464 {
28100
54b35da9879e Fix SF bug #694816, remove comparison of unsigned value < 0
Neal Norwitz <nnorwitz@gmail.com>
parents: 26571
diff changeset
465 if (code >= 0x110000) {
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
466 *index = 0;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
467 } else if (self && UCD_Check(self) &&
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
468 get_old_record(self, code)->category_changed==0) {
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
469 /* unassigned in old version */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
470 *index = 0;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
471 }
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
472 else {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
473 *index = decomp_index1[(code>>DECOMP_SHIFT)];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
474 *index = decomp_index2[(*index<<DECOMP_SHIFT)+
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
475 (code&((1<<DECOMP_SHIFT)-1))];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
476 }
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
477
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
478 /* high byte is number of hex bytes (usually one or two), low byte
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
479 is prefix code (from*/
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
480 *count = decomp_data[*index] >> 8;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
481 *prefix = decomp_data[*index] & 255;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
482
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
483 (*index)++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
484 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
485
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
486 #define SBase 0xAC00
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
487 #define LBase 0x1100
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
488 #define VBase 0x1161
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
489 #define TBase 0x11A7
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
490 #define LCount 19
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
491 #define VCount 21
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
492 #define TCount 28
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
493 #define NCount (VCount*TCount)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
494 #define SCount (LCount*NCount)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
495
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
496 static PyObject*
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
497 nfd_nfkd(PyObject *self, PyObject *input, int k)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
498 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
499 PyObject *result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
500 Py_UNICODE *i, *end, *o;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
501 /* Longest decomposition in Unicode 3.2: U+FDFA */
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
502 Py_UNICODE stack[20];
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36939
diff changeset
503 Py_ssize_t space, isize;
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36939
diff changeset
504 int index, prefix, count, stackptr;
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
505 unsigned char prev, cur;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
506
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
507 stackptr = 0;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
508 isize = PyUnicode_GET_SIZE(input);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
509 /* Overallocate atmost 10 characters. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
510 space = (isize > 10 ? 10 : isize) + isize;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
511 result = PyUnicode_FromUnicode(NULL, space);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
512 if (!result)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
513 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
514 i = PyUnicode_AS_UNICODE(input);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
515 end = i + isize;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
516 o = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
517
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
518 while (i < end) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
519 stack[stackptr++] = *i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
520 while(stackptr) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
521 Py_UNICODE code = stack[--stackptr];
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
522 /* Hangul Decomposition adds three characters in
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
523 a single step, so we need atleast that much room. */
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
524 if (space < 3) {
42013
cd125fe83051 Make identifiers str (not str8) objects throughout.
Martin v. Löwis <martin@v.loewis.de>
parents: 41944
diff changeset
525 Py_ssize_t newsize = PyUnicode_GET_SIZE(result) + 10;
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
526 space += 10;
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
527 if (PyUnicode_Resize(&result, newsize) == -1)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
528 return NULL;
30867
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676.
Martin v. Löwis <martin@v.loewis.de>
parents: 28100
diff changeset
529 o = PyUnicode_AS_UNICODE(result) + newsize - space;
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
530 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
531 /* Hangul Decomposition. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
532 if (SBase <= code && code < (SBase+SCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
533 int SIndex = code - SBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
534 int L = LBase + SIndex / NCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
535 int V = VBase + (SIndex % NCount) / TCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
536 int T = TBase + SIndex % TCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
537 *o++ = L;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
538 *o++ = V;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
539 space -= 2;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
540 if (T != TBase) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
541 *o++ = T;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
542 space --;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
543 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
544 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
545 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
546 /* normalization changes */
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
547 if (self && UCD_Check(self)) {
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
548 Py_UCS4 value = ((PreviousDBVersion*)self)->normalization(code);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
549 if (value != 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
550 stack[stackptr++] = value;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
551 continue;
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
552 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
553 }
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
554
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
555 /* Other decompositions. */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
556 get_decomp_record(self, code, &index, &prefix, &count);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
557
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
558 /* Copy character if it is not decomposable, or has a
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
559 compatibility decomposition, but we do NFD. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
560 if (!count || (prefix && !k)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
561 *o++ = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
562 space--;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
563 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
564 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
565 /* Copy decomposition onto the stack, in reverse
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
566 order. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
567 while(count) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
568 code = decomp_data[index + (--count)];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
569 stack[stackptr++] = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
570 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
571 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
572 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
573
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
574 /* Drop overallocation. Cannot fail. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
575 PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
576
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
577 /* Sort canonically. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
578 i = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
579 prev = _getrecord_ex(*i)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
580 end = i + PyUnicode_GET_SIZE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
581 for (i++; i < end; i++) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
582 cur = _getrecord_ex(*i)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
583 if (prev == 0 || cur == 0 || prev <= cur) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
584 prev = cur;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
585 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
586 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
587 /* Non-canonical order. Need to switch *i with previous. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
588 o = i - 1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
589 while (1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
590 Py_UNICODE tmp = o[1];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
591 o[1] = o[0];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
592 o[0] = tmp;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
593 o--;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
594 if (o < PyUnicode_AS_UNICODE(result))
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
595 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
596 prev = _getrecord_ex(*o)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
597 if (prev == 0 || prev <= cur)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
598 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
599 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
600 prev = _getrecord_ex(*i)->combining;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
601 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
602 return result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
603 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
604
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
605 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
606 find_nfc_index(PyObject *self, struct reindex* nfc, Py_UNICODE code)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
607 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
608 int index;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
609 for (index = 0; nfc[index].start; index++) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
610 int start = nfc[index].start;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
611 if (code < start)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
612 return -1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
613 if (code <= start + nfc[index].count) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
614 int delta = code - start;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
615 return nfc[index].index + delta;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
616 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
617 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
618 return -1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
619 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
620
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
621 static PyObject*
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
622 nfc_nfkc(PyObject *self, PyObject *input, int k)
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
623 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
624 PyObject *result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
625 Py_UNICODE *i, *i1, *o, *end;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
626 int f,l,index,index1,comb;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
627 Py_UNICODE code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
628 Py_UNICODE *skipped[20];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
629 int cskipped = 0;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
630
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
631 result = nfd_nfkd(self, input, k);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
632 if (!result)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
633 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
634
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
635 /* We are going to modify result in-place.
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
636 If nfd_nfkd is changed to sometimes return the input,
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
637 this code needs to be reviewed. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
638 assert(result != input);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
639
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
640 i = PyUnicode_AS_UNICODE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
641 end = i + PyUnicode_GET_SIZE(result);
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
642 o = PyUnicode_AS_UNICODE(result);
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
643
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
644 again:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
645 while (i < end) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
646 for (index = 0; index < cskipped; index++) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
647 if (skipped[index] == i) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
648 /* *i character is skipped.
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
649 Remove from list. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
650 skipped[index] = skipped[cskipped-1];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
651 cskipped--;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
652 i++;
26571
cb8cfadf75a2 Remove C++ comment.
Martin v. Löwis <martin@v.loewis.de>
parents: 26397
diff changeset
653 goto again; /* continue while */
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
654 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
655 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
656 /* Hangul Composition. We don't need to check for <LV,T>
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
657 pairs, since we always have decomposed data. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
658 if (LBase <= *i && *i < (LBase+LCount) &&
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
659 i + 1 < end &&
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
660 VBase <= i[1] && i[1] <= (VBase+VCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
661 int LIndex, VIndex;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
662 LIndex = i[0] - LBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
663 VIndex = i[1] - VBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
664 code = SBase + (LIndex*VCount+VIndex)*TCount;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
665 i+=2;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
666 if (i < end &&
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
667 TBase <= *i && *i <= (TBase+TCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
668 code += *i-TBase;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
669 i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
670 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
671 *o++ = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
672 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
673 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
674
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
675 f = find_nfc_index(self, nfc_first, *i);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
676 if (f == -1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
677 *o++ = *i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
678 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
679 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
680 /* Find next unblocked character. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
681 i1 = i+1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
682 comb = 0;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
683 while (i1 < end) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
684 int comb1 = _getrecord_ex(*i1)->combining;
67184
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
685 if (comb) {
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
686 if (comb1 == 0)
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
687 break;
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
688 if (comb >= comb1) {
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
689 /* Character is blocked. */
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
690 i1++;
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
691 continue;
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
692 }
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
693 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
694 l = find_nfc_index(self, nfc_last, *i1);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
695 /* *i1 cannot be combined with *i. If *i1
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
696 is a starter, we don't need to look further.
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
697 Otherwise, record the combining class. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
698 if (l == -1) {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
699 not_combinable:
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
700 if (comb1 == 0)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
701 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
702 comb = comb1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
703 i1++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
704 continue;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
705 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
706 index = f*TOTAL_LAST + l;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
707 index1 = comp_index[index >> COMP_SHIFT];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
708 code = comp_data[(index1<<COMP_SHIFT)+
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
709 (index&((1<<COMP_SHIFT)-1))];
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
710 if (code == 0)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
711 goto not_combinable;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
712
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
713 /* Replace the original character. */
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
714 *i = code;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
715 /* Mark the second character unused. */
67184
a4ec20cc9066 Merged revisions 87442 via svnmerge from
Alexander Belopolsky <alexander.belopolsky@gmail.com>
parents: 66396
diff changeset
716 assert(cskipped < 20);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
717 skipped[cskipped++] = i1;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
718 i1++;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
719 f = find_nfc_index(self, nfc_first, *i);
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
720 if (f == -1)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
721 break;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
722 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
723 *o++ = *i++;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
724 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
725 if (o != end)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
726 PyUnicode_Resize(&result, o - PyUnicode_AS_UNICODE(result));
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
727 return result;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
728 }
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
729
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
730 /* Return 1 if the input is certainly normalized, 0 if it might not be. */
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
731 static int
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
732 is_normalized(PyObject *self, PyObject *input, int nfc, int k)
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
733 {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
734 Py_UNICODE *i, *end;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
735 unsigned char prev_combining = 0, quickcheck_mask;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
736
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
737 /* An older version of the database is requested, quickchecks must be
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
738 disabled. */
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
739 if (self && UCD_Check(self))
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
740 return 0;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
741
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
742 /* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
743 as described in http://unicode.org/reports/tr15/#Annex8. */
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
744 quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
745
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
746 i = PyUnicode_AS_UNICODE(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
747 end = i + PyUnicode_GET_SIZE(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
748 while (i < end) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
749 const _PyUnicode_DatabaseRecord *record = _getrecord_ex(*i++);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
750 unsigned char combining = record->combining;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
751 unsigned char quickcheck = record->normalization_quick_check;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
752
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
753 if (quickcheck & quickcheck_mask)
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
754 return 0; /* this string might need normalization */
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
755 if (combining && prev_combining > combining)
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
756 return 0; /* non-canonical sort order, not normalized */
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
757 prev_combining = combining;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
758 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
759 return 1; /* certainly normalized */
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
760 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
761
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
762 PyDoc_STRVAR(unicodedata_normalize__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
763 "normalize(form, unistr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
764 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
765 Return the normal form 'form' for the Unicode string unistr. Valid\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
766 values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
767
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
768 static PyObject*
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
769 unicodedata_normalize(PyObject *self, PyObject *args)
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
770 {
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
771 char *form;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
772 PyObject *input;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
773
32680
d3d426483d71 Fix typo.
Hye-Shik Chang <hyeshik@gmail.com>
parents: 31930
diff changeset
774 if(!PyArg_ParseTuple(args, "sO!:normalize",
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
775 &form, &PyUnicode_Type, &input))
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
776 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
777
31930
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
778 if (PyUnicode_GetSize(input) == 0) {
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
779 /* Special case empty input strings, since resizing
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
780 them later would cause internal errors. */
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
781 Py_INCREF(input);
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
782 return input;
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
783 }
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361.
Martin v. Löwis <martin@v.loewis.de>
parents: 30867
diff changeset
784
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
785 if (strcmp(form, "NFC") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
786 if (is_normalized(self, input, 1, 0)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
787 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
788 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
789 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
790 return nfc_nfkc(self, input, 0);
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
791 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
792 if (strcmp(form, "NFKC") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
793 if (is_normalized(self, input, 1, 1)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
794 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
795 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
796 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
797 return nfc_nfkc(self, input, 1);
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
798 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
799 if (strcmp(form, "NFD") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
800 if (is_normalized(self, input, 0, 0)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
801 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
802 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
803 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
804 return nfd_nfkd(self, input, 0);
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
805 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
806 if (strcmp(form, "NFKD") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
807 if (is_normalized(self, input, 0, 1)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
808 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
809 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
810 }
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
811 return nfd_nfkd(self, input, 1);
53554
423f8f9b07b1 Merged revisions 72054 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 51271
diff changeset
812 }
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
813 PyErr_SetString(PyExc_ValueError, "invalid normalization form");
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
814 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
815 }
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
816
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
817 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
818 /* unicode character name tables */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
819
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
820 /* data file generated by Tools/unicode/makeunicodedata.py */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
821 #include "unicodename_db.h"
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
822
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
823 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
824 /* database code (cut and pasted from the unidb package) */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
825
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
826 static unsigned long
16666
e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS
Fredrik Lundh <fredrik@pythonware.com>
parents: 16662
diff changeset
827 _gethash(const char *s, int len, int scale)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
828 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
829 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
830 unsigned long h = 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
831 unsigned long ix;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
832 for (i = 0; i < len; i++) {
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36939
diff changeset
833 h = (h * scale) + (unsigned char) toupper(Py_CHARMASK(s[i]));
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
834 ix = h & 0xff000000;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
835 if (ix)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
836 h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
837 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
838 return h;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
839 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
840
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
841 static char *hangul_syllables[][3] = {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
842 { "G", "A", "" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
843 { "GG", "AE", "G" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
844 { "N", "YA", "GG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
845 { "D", "YAE", "GS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
846 { "DD", "EO", "N", },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
847 { "R", "E", "NJ" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
848 { "M", "YEO", "NH" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
849 { "B", "YE", "D" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
850 { "BB", "O", "L" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
851 { "S", "WA", "LG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
852 { "SS", "WAE", "LM" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
853 { "", "OE", "LB" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
854 { "J", "YO", "LS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
855 { "JJ", "U", "LT" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
856 { "C", "WEO", "LP" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
857 { "K", "WE", "LH" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
858 { "T", "WI", "M" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
859 { "P", "YU", "B" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
860 { "H", "EU", "BS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
861 { 0, "YI", "S" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
862 { 0, "I", "SS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
863 { 0, 0, "NG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
864 { 0, 0, "J" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
865 { 0, 0, "C" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
866 { 0, 0, "K" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
867 { 0, 0, "T" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
868 { 0, 0, "P" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
869 { 0, 0, "H" }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
870 };
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
871
66392
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
872 /* These ranges need to match makeunicodedata.py:cjk_ranges. */
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
873 static int
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
874 is_unified_ideograph(Py_UCS4 code)
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
875 {
66392
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
876 return
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
877 (0x3400 <= code && code <= 0x4DB5) || /* CJK Ideograph Extension A */
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
878 (0x4E00 <= code && code <= 0x9FCB) || /* CJK Ideograph */
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
879 (0x20000 <= code && code <= 0x2A6D6) || /* CJK Ideograph Extension B */
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
880 (0x2A700 <= code && code <= 0x2B734) || /* CJK Ideograph Extension C */
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0.
Martin v. Löwis <martin@v.loewis.de>
parents: 64423
diff changeset
881 (0x2B740 <= code && code <= 0x2B81D); /* CJK Ideograph Extension D */
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
882 }
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
883
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
884 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
885 _getucname(PyObject *self, Py_UCS4 code, char* buffer, int buflen)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
886 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
887 int offset;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
888 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
889 int word;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
890 unsigned char* w;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
891
36939
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
892 if (code >= 0x110000)
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
893 return 0;
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
894
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
895 if (self && UCD_Check(self)) {
36939
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
896 const change_record *old = get_old_record(self, code);
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
897 if (old->category_changed == 0) {
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
898 /* unassigned */
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
899 return 0;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
900 }
36939
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
901 }
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36928
diff changeset
902
26374
135bb11215a0 Fix off-by-one error.
Martin v. Löwis <martin@v.loewis.de>
parents: 26370
diff changeset
903 if (SBase <= code && code < SBase+SCount) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
904 /* Hangul syllable. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
905 int SIndex = code - SBase;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
906 int L = SIndex / NCount;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
907 int V = (SIndex % NCount) / TCount;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
908 int T = SIndex % TCount;
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
909
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
910 if (buflen < 27)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
911 /* Worst case: HANGUL SYLLABLE <10chars>. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
912 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
913 strcpy(buffer, "HANGUL SYLLABLE ");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
914 buffer += 16;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
915 strcpy(buffer, hangul_syllables[L][0]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
916 buffer += strlen(hangul_syllables[L][0]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
917 strcpy(buffer, hangul_syllables[V][1]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
918 buffer += strlen(hangul_syllables[V][1]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
919 strcpy(buffer, hangul_syllables[T][2]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
920 buffer += strlen(hangul_syllables[T][2]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
921 *buffer = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
922 return 1;
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
923 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
924
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
925 if (is_unified_ideograph(code)) {
26376
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
926 if (buflen < 28)
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
927 /* Worst case: CJK UNIFIED IDEOGRAPH-20000 */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
928 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
929 sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code);
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
930 return 1;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
931 }
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
932
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
933 /* get offset into phrasebook */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
934 offset = phrasebook_offset1[(code>>phrasebook_shift)];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
935 offset = phrasebook_offset2[(offset<<phrasebook_shift) +
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
936 (code&((1<<phrasebook_shift)-1))];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
937 if (!offset)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
938 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
939
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
940 i = 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
941
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
942 for (;;) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
943 /* get word index */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
944 word = phrasebook[offset] - phrasebook_short;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
945 if (word >= 0) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
946 word = (word << 8) + phrasebook[offset+1];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
947 offset += 2;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
948 } else
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
949 word = phrasebook[offset++];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
950 if (i) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
951 if (i > buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
952 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
953 buffer[i++] = ' ';
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
954 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
955 /* copy word string from lexicon. the last character in the
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
956 word has bit 7 set. the last word in a string ends with
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
957 0x80 */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
958 w = lexicon + lexicon_offset[word];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
959 while (*w < 128) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
960 if (i >= buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
961 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
962 buffer[i++] = *w++;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
963 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
964 if (i >= buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
965 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
966 buffer[i++] = *w & 127;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
967 if (*w == 128)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
968 break; /* end of word */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
969 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
970
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
971 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
972 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
973
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
974 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
975 _cmpname(PyObject *self, int code, const char* name, int namelen)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
976 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
977 /* check if code corresponds to the given name */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
978 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
979 char buffer[NAME_MAXLEN];
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
980 if (!_getucname(self, code, buffer, sizeof(buffer)))
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
981 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
982 for (i = 0; i < namelen; i++) {
37879
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair
Thomas Wouters <thomas@python.org>
parents: 36939
diff changeset
983 if (toupper(Py_CHARMASK(name[i])) != buffer[i])
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
984 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
985 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
986 return buffer[namelen] == '\0';
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
987 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
988
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
989 static void
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
990 find_syllable(const char *str, int *len, int *pos, int count, int column)
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
991 {
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
992 int i, len1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
993 *len = -1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
994 for (i = 0; i < count; i++) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
995 char *s = hangul_syllables[i][column];
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
996 len1 = strlen(s);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
997 if (len1 <= *len)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
998 continue;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
999 if (strncmp(str, s, len1) == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1000 *len = len1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1001 *pos = i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1002 }
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1003 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1004 if (*len == -1) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1005 *len = 0;
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1006 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1007 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1008
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1009 static int
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1010 _getcode(PyObject* self, const char* name, int namelen, Py_UCS4* code)
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1011 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1012 unsigned int h, v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1013 unsigned int mask = code_size-1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1014 unsigned int i, incr;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1015
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1016 /* Check for hangul syllables. */
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1017 if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1018 int len, L = -1, V = -1, T = -1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1019 const char *pos = name + 16;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1020 find_syllable(pos, &len, &L, LCount, 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1021 pos += len;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1022 find_syllable(pos, &len, &V, VCount, 1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1023 pos += len;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1024 find_syllable(pos, &len, &T, TCount, 2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1025 pos += len;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1026 if (L != -1 && V != -1 && T != -1 && pos-name == namelen) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1027 *code = SBase + (L*VCount+V)*TCount + T;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1028 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1029 }
26376
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1030 /* Otherwise, it's an illegal syllable name. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1031 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1032 }
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1033
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1034 /* Check for unified ideographs. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1035 if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) {
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1036 /* Four or five hexdigits must follow. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1037 v = 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1038 name += 22;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1039 namelen -= 22;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1040 if (namelen != 4 && namelen != 5)
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1041 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1042 while (namelen--) {
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1043 v *= 16;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1044 if (*name >= '0' && *name <= '9')
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1045 v += *name - '0';
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1046 else if (*name >= 'A' && *name <= 'F')
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1047 v += *name - 'A' + 10;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1048 else
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1049 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1050 name++;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1051 }
26380
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
1052 if (!is_unified_ideograph(v))
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph.
Martin v. Löwis <martin@v.loewis.de>
parents: 26379
diff changeset
1053 return 0;
26376
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1054 *code = v;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output.
Martin v. Löwis <martin@v.loewis.de>
parents: 26374
diff changeset
1055 return 1;
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1056 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1057
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1058 /* the following is the same as python's dictionary lookup, with
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1059 only minor changes. see the makeunicodedata script for more
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1060 details */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1061
16666
e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS
Fredrik Lundh <fredrik@pythonware.com>
parents: 16662
diff changeset
1062 h = (unsigned int) _gethash(name, namelen, code_magic);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1063 i = (~h) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1064 v = code_hash[i];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1065 if (!v)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1066 return 0;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1067 if (_cmpname(self, v, name, namelen)) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1068 *code = v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1069 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1070 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1071 incr = (h ^ (h >> 3)) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1072 if (!incr)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1073 incr = mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1074 for (;;) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1075 i = (i + incr) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1076 v = code_hash[i];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1077 if (!v)
16662
89c1ccef5ce8 stupid typo (for some reason, this only caused problems on OpenVMS).
Fredrik Lundh <fredrik@pythonware.com>
parents: 16200
diff changeset
1078 return 0;
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1079 if (_cmpname(self, v, name, namelen)) {
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1080 *code = v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1081 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1082 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1083 incr = incr << 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1084 if (incr > mask)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1085 incr = incr ^ code_poly;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1086 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1087 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1088
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1089 static const _PyUnicode_Name_CAPI hashAPI =
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1090 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1091 sizeof(_PyUnicode_Name_CAPI),
23810
01e4b698ff7e _Py prefix is verboten for static entry points
Andrew MacIntyre <andymac@bullseye.apana.org.au>
parents: 22781
diff changeset
1092 _getucname,
16666
e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS
Fredrik Lundh <fredrik@pythonware.com>
parents: 16662
diff changeset
1093 _getcode
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1094 };
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1095
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1096 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1097 /* Python bindings */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1098
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1099 PyDoc_STRVAR(unicodedata_name__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1100 "name(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1101 Returns the name assigned to the Unicode character unichr as a\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1102 string. If no name is defined, default is returned, or, if not\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1103 given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1104
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1105 static PyObject *
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1106 unicodedata_name(PyObject* self, PyObject* args)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1107 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1108 char name[NAME_MAXLEN];
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
1109 Py_UCS4 c;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1110
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1111 PyUnicodeObject* v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1112 PyObject* defobj = NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1113 if (!PyArg_ParseTuple(args, "O!|O:name", &PyUnicode_Type, &v, &defobj))
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1114 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1115
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
1116 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
1117 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
1118 return NULL;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1119
47458
e4a186437440 Change all functions that expect one unicode character to accept a pair of
Walter Dörwald <walter@livinglogic.de>
parents: 44351
diff changeset
1120 if (!_getucname(self, c, name, sizeof(name))) {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1121 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1122 PyErr_SetString(PyExc_ValueError, "no such name");
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1123 return NULL;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1124 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1125 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1126 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1127 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1128 }
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1129 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1130
41944
53963a2f5652 Change category(), bidirectional(), east_asian_width(),
Walter Dörwald <walter@livinglogic.de>
parents: 40450
diff changeset
1131 return PyUnicode_FromString(name);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1132 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1133
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1134 PyDoc_STRVAR(unicodedata_lookup__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1135 "lookup(name)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1136 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1137 Look up character by name. If a character with the\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1138 given name is found, return the corresponding Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1139 character. If not found, KeyError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1140
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1141 static PyObject *
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1142 unicodedata_lookup(PyObject* self, PyObject* args)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1143 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1144 Py_UCS4 code;
42466
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1145 Py_UNICODE str[2];
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1146
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1147 char* name;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1148 int namelen;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1149 if (!PyArg_ParseTuple(args, "s#:lookup", &name, &namelen))
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1150 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1151
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1152 if (!_getcode(self, name, namelen, &code)) {
42466
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1153 PyErr_Format(PyExc_KeyError, "undefined character name '%s'",
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1154 name);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1155 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1156 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1157
42466
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1158 #ifndef Py_UNICODE_WIDE
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1159 if (code >= 0x10000) {
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1160 str[0] = 0xd800 + ((code - 0x10000) >> 10);
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1161 str[1] = 0xdc00 + ((code - 0x10000) & 0x3ff);
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1162 return PyUnicode_FromUnicode(str, 2);
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1163 }
3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 42334
diff changeset
1164 #endif
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1165 str[0] = (Py_UNICODE) code;
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1166 return PyUnicode_FromUnicode(str, 1);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1167 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1168
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1169 /* XXX Add doc strings. */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1170
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1171 static PyMethodDef unicodedata_functions[] = {
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1172 {"decimal", unicodedata_decimal, METH_VARARGS, unicodedata_decimal__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1173 {"digit", unicodedata_digit, METH_VARARGS, unicodedata_digit__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1174 {"numeric", unicodedata_numeric, METH_VARARGS, unicodedata_numeric__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1175 {"category", unicodedata_category, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1176 unicodedata_category__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1177 {"bidirectional", unicodedata_bidirectional, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1178 unicodedata_bidirectional__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1179 {"combining", unicodedata_combining, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1180 unicodedata_combining__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1181 {"mirrored", unicodedata_mirrored, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1182 unicodedata_mirrored__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1183 {"east_asian_width", unicodedata_east_asian_width, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1184 unicodedata_east_asian_width__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1185 {"decomposition", unicodedata_decomposition, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1186 unicodedata_decomposition__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1187 {"name", unicodedata_name, METH_VARARGS, unicodedata_name__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1188 {"lookup", unicodedata_lookup, METH_VARARGS, unicodedata_lookup__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1189 {"normalize", unicodedata_normalize, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1190 unicodedata_normalize__doc__},
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1191 {NULL, NULL} /* sentinel */
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1192 };
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1193
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1194 static PyTypeObject UCD_Type = {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1195 /* The ob_type field must be initialized in the module init function
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1196 * to be portable to Windows without using C++. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1197 PyVarObject_HEAD_INIT(NULL, 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1198 "unicodedata.UCD", /*tp_name*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1199 sizeof(PreviousDBVersion), /*tp_basicsize*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1200 0, /*tp_itemsize*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1201 /* methods */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1202 (destructor)PyObject_Del, /*tp_dealloc*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1203 0, /*tp_print*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1204 0, /*tp_getattr*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1205 0, /*tp_setattr*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1206 0, /*tp_reserved*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1207 0, /*tp_repr*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1208 0, /*tp_as_number*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1209 0, /*tp_as_sequence*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1210 0, /*tp_as_mapping*/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1211 0, /*tp_hash*/
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1212 0, /*tp_call*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1213 0, /*tp_str*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1214 PyObject_GenericGetAttr,/*tp_getattro*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1215 0, /*tp_setattro*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1216 0, /*tp_as_buffer*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1217 Py_TPFLAGS_DEFAULT, /*tp_flags*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1218 0, /*tp_doc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1219 0, /*tp_traverse*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1220 0, /*tp_clear*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1221 0, /*tp_richcompare*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1222 0, /*tp_weaklistoffset*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1223 0, /*tp_iter*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1224 0, /*tp_iternext*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1225 unicodedata_functions, /*tp_methods*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1226 DB_members, /*tp_members*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1227 0, /*tp_getset*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1228 0, /*tp_base*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1229 0, /*tp_dict*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1230 0, /*tp_descr_get*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1231 0, /*tp_descr_set*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1232 0, /*tp_dictoffset*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1233 0, /*tp_init*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1234 0, /*tp_alloc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1235 0, /*tp_new*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1236 0, /*tp_free*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1237 0, /*tp_is_gc*/
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1238 };
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1239
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1240 PyDoc_STRVAR(unicodedata_docstring,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1241 "This module provides access to the Unicode Character Database which\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1242 defines character properties for all Unicode characters. The data in\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1243 this database is based on the UnicodeData.txt file version\n\
59927
65ff526e05eb Merged revisions 79314 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents: 59324
diff changeset
1244 5.2.0 which is publically available from ftp://ftp.unicode.org/.\n\
34864
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1245 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex
Hye-Shik Chang <hyeshik@gmail.com>
parents: 32951
diff changeset
1246 The module uses the same names and symbols as defined by the\n\
59933
8e99b6000f0d Merged revisions 79320 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents: 59927
diff changeset
1247 UnicodeData File Format 5.2.0 (see\n\
8e99b6000f0d Merged revisions 79320 via svnmerge from
Ezio Melotti <ezio.melotti@gmail.com>
parents: 59927
diff changeset
1248 http://www.unicode.org/reports/tr44/tr44-4.html).");
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1249
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1250
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1251 static struct PyModuleDef unicodedatamodule = {
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1252 PyModuleDef_HEAD_INIT,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1253 "unicodedata",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1254 unicodedata_docstring,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1255 -1,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1256 unicodedata_functions,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1257 NULL,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1258 NULL,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1259 NULL,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1260 NULL
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1261 };
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1262
24363
979ff5d0ae07 Replace DL_IMPORT with PyMODINIT_FUNC and remove "/export:init..." link
Mark Hammond <mhammond@skippinet.com.au>
parents: 23832
diff changeset
1263 PyMODINIT_FUNC
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1264 PyInit_unicodedata(void)
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1265 {
22781
ec67fce3fa3c Remove direct manipulation of the module dict.
Fred Drake <fdrake@acm.org>
parents: 22114
diff changeset
1266 PyObject *m, *v;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1267
44351
86a2679dca46 #1629: Renamed Py_Size, Py_Type and Py_Refcnt to Py_SIZE, Py_TYPE and Py_REFCNT.
Christian Heimes <christian@cheimes.de>
parents: 44099
diff changeset
1268 Py_TYPE(&UCD_Type) = &PyType_Type;
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1269
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1270 m = PyModule_Create(&unicodedatamodule);
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1271 if (!m)
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1272 return NULL;
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1273
26397
b27c1cad77ea Add unidata_version. Bump generator version number.
Martin v. Löwis <martin@v.loewis.de>
parents: 26380
diff changeset
1274 PyModule_AddStringConstant(m, "unidata_version", UNIDATA_VERSION);
36928
11f8b5b8ce20 Fix refcounting bug.
Martin v. Löwis <martin@v.loewis.de>
parents: 36927
diff changeset
1275 Py_INCREF(&UCD_Type);
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1276 PyModule_AddObject(m, "UCD", (PyObject*)&UCD_Type);
26397
b27c1cad77ea Add unidata_version. Bump generator version number.
Martin v. Löwis <martin@v.loewis.de>
parents: 26380
diff changeset
1277
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1278 /* Previous versions */
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1279 v = new_previous_version("3.2.0", get_change_3_2_0, normalization_3_2_0);
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1280 if (v != NULL)
36927
3266a020a0da Avoid forward-declaring the methods array.
Martin v. Löwis <martin@v.loewis.de>
parents: 36924
diff changeset
1281 PyModule_AddObject(m, "ucd_3_2_0", v);
36924
48e0a5e39403 Update Unicode database to Unicode 4.1.
Martin v. Löwis <martin@v.loewis.de>
parents: 36778
diff changeset
1282
16200
3484670c37f2 Move uchhash functionality into unicodedata (after the recent
Fredrik Lundh <fredrik@pythonware.com>
parents: 16091
diff changeset
1283 /* Export C API */
53816
911df82bc25d add a replacement API for PyCObject, PyCapsule #5630
Benjamin Peterson <benjamin@python.org>
parents: 53554
diff changeset
1284 v = PyCapsule_New((void *)&hashAPI, PyUnicodeData_CAPSULE_NAME, NULL);
22781
ec67fce3fa3c Remove direct manipulation of the module dict.
Fred Drake <fdrake@acm.org>
parents: 22114
diff changeset
1285 if (v != NULL)
ec67fce3fa3c Remove direct manipulation of the module dict.
Fred Drake <fdrake@acm.org>
parents: 22114
diff changeset
1286 PyModule_AddObject(m, "ucnhash_CAPI", v);
47580
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API.
Martin v. Löwis <martin@v.loewis.de>
parents: 47458
diff changeset
1287 return m;
11981
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1288 }
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1289
61467
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 59325
diff changeset
1290 /*
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1291 Local variables:
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1292 c-basic-offset: 4
26379
1885b2d6393a Patch #626485: Support Unicode normalization.
Martin v. Löwis <martin@v.loewis.de>
parents: 26376
diff changeset
1293 indent-tabs-mode: nil
26370
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1294 End:
9e489a0df7b5 Patch #626548: Support Hangul syllable names.
Martin v. Löwis <martin@v.loewis.de>
parents: 25852
diff changeset
1295 */