cpython-lru_cache-weakref: Modules/unicodedata.c annotate

annotate Modules/unicodedata.c @ 79691:05c8928362ee

Merge __next__ method link fixes with 3.3.

author	Ezio Melotti <ezio.melotti@gmail.com>
date	Fri, 12 Oct 2012 13:46:37 +0300
parents	c843ec292a8b
children	d8218745c24a

rev	line source
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1 /* ------------------------------------------------------------------------
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	2
73474 6ddda809ea8c #13379: update Unicode version in unicodedata docstrings and comments. Ezio Melotti <ezio.melotti@gmail.com> parents: 72655 diff changeset	3 unicodedata -- Provides access to the Unicode database.
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	4
73474 6ddda809ea8c #13379: update Unicode version in unicodedata docstrings and comments. Ezio Melotti <ezio.melotti@gmail.com> parents: 72655 diff changeset	5 Data was extracted from the UnicodeData.txt file.
6ddda809ea8c #13379: update Unicode version in unicodedata docstrings and comments. Ezio Melotti <ezio.melotti@gmail.com> parents: 72655 diff changeset	6 The current version number is reported in the unidata_version constant.
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	7
14835 96d31a143857 unicode database compression, step 2: Fredrik Lundh <fredrik@pythonware.com> parents: 14818 diff changeset	8 Written by Marc-Andre Lemburg (mal@lemburg.com).
96d31a143857 unicode database compression, step 2: Fredrik Lundh <fredrik@pythonware.com> parents: 14818 diff changeset	9 Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
64423 e9cd15d539e1 Welcome to the UTF-8 world. Florent Xicluna <florent.xicluna@gmail.com> parents: 61466 diff changeset	10 Modified by Martin v. Löwis (martin@v.loewis.de)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	11
14835 96d31a143857 unicode database compression, step 2: Fredrik Lundh <fredrik@pythonware.com> parents: 14818 diff changeset	12 Copyright (c) Corporation for National Research Initiatives.
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	13
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	14 ------------------------------------------------------------------------ */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	15
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	16 #include "Python.h"
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	17 #include "ucnhash.h"
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	18 #include "structmember.h"
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	19
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	20 /* character properties */
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	21
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	22 typedef struct {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	23 const unsigned char category; /* index into
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	24 _PyUnicode_CategoryNames */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	25 const unsigned char combining; /* combining class value 0 - 255 */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	26 const unsigned char bidirectional; /* index into
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	27 _PyUnicode_BidirectionalNames */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	28 const unsigned char mirrored; /* true if mirrored in bidir mode */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	29 const unsigned char east_asian_width; /* index into
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	30 _PyUnicode_EastAsianWidth */
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	31 const unsigned char normalization_quick_check; /* see is_normalized() */
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	32 } _PyUnicode_DatabaseRecord;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	33
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	34 typedef struct change_record {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	35 /* sequence of fields should be the same as in merge_old_version */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	36 const unsigned char bidir_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	37 const unsigned char category_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	38 const unsigned char decimal_changed;
49125 dc60293b29d5 Merged revisions 66362 via svnmerge from Martin v. Löwis <martin@v.loewis.de> parents: 47580 diff changeset	39 const unsigned char mirrored_changed;
56408 6c91d1803182 Merged revisions 75272-75273 via svnmerge from Amaury Forgeot d'Arc <amauryfa@gmail.com> parents: 53816 diff changeset	40 const double numeric_changed;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	41 } change_record;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	42
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	43 /* data file generated by Tools/unicode/makeunicodedata.py */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	44 #include "unicodedata_db.h"
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	45
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	46 static const _PyUnicode_DatabaseRecord*
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	47 _getrecord_ex(Py_UCS4 code)
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	48 {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	49 int index;
28100 54b35da9879e Fix SF bug #694816, remove comparison of unsigned value < 0 Neal Norwitz <nnorwitz@gmail.com> parents: 26571 diff changeset	50 if (code >= 0x110000)
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	51 index = 0;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	52 else {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	53 index = index1[(code>>SHIFT)];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	54 index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	55 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	56
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	57 return &_PyUnicode_Database_Records[index];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	58 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	59
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	60 /* ------------- Previous-version API ------------------------------------- */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	61 typedef struct previous_version {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	62 PyObject_HEAD
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	63 const char *name;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	64 const change_record* (*getrecord)(Py_UCS4);
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	65 Py_UCS4 (*normalization)(Py_UCS4);
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	66 } PreviousDBVersion;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	67
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	68 #define get_old_record(self, v) ((((PreviousDBVersion*)self)->getrecord)(v))
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	69
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	70 static PyMemberDef DB_members[] = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	71 {"unidata_version", T_STRING, offsetof(PreviousDBVersion, name), READONLY},
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	72 {NULL}
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	73 };
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	74
40450 cc992d75d5b3 Four months of trunk changes (including a few releases...) Thomas Wouters <thomas@python.org> parents: 39610 diff changeset	75 /* forward declaration */
36927 3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	76 static PyTypeObject UCD_Type;
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	77 #define UCD_Check(o) (Py_TYPE(o)==&UCD_Type)
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	78
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	79 static PyObject*
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	80 new_previous_version(const charname, const change_record (*getrecord)(Py_UCS4),
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	81 Py_UCS4 (*normalization)(Py_UCS4))
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	82 {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	83 PreviousDBVersion *self;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	84 self = PyObject_New(PreviousDBVersion, &UCD_Type);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	85 if (self == NULL)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	86 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	87 self->name = name;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	88 self->getrecord = getrecord;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	89 self->normalization = normalization;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	90 return (PyObject*)self;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	91 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	92
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	93
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	94 static Py_UCS4 getuchar(PyUnicodeObject *obj)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	95 {
72475 8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	96 if (PyUnicode_READY(obj))
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	97 return (Py_UCS4)-1;
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	98 if (PyUnicode_GET_LENGTH(obj) == 1) {
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	99 if (PyUnicode_READY(obj))
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	100 return (Py_UCS4)-1;
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	101 return PyUnicode_READ_CHAR(obj, 0);
8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	102 }
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	103 PyErr_SetString(PyExc_TypeError,
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	104 "need a single Unicode character as parameter");
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	105 return (Py_UCS4)-1;
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	106 }
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	107
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	108 /* --- Module API --------------------------------------------------------- */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	109
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	110 PyDoc_STRVAR(unicodedata_decimal__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	111 "decimal(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	112 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	113 Returns the decimal value assigned to the Unicode character unichr\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	114 as integer. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	115 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	116
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	117 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	118 unicodedata_decimal(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	119 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	120 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	121 PyObject *defobj = NULL;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	122 int have_old = 0;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	123 long rc;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	124 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	125
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	126 if (!PyArg_ParseTuple(args, "O!\|O:decimal", &PyUnicode_Type, &v, &defobj))
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	127 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	128 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	129 if (c == (Py_UCS4)-1)
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	130 return NULL;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	131
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	132 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	133 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	134 if (old->category_changed == 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	135 /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	136 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	137 rc = -1;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	138 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	139 else if (old->decimal_changed != 0xFF) {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	140 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	141 rc = old->decimal_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	142 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	143 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	144
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	145 if (!have_old)
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	146 rc = Py_UNICODE_TODECIMAL(c);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	147 if (rc < 0) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	148 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	149 PyErr_SetString(PyExc_ValueError,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	150 "not a decimal");
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	151 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	152 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	153 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	154 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	155 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	156 }
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	157 }
44099 f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h Christian Heimes <christian@cheimes.de> parents: 42466 diff changeset	158 return PyLong_FromLong(rc);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	159 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	160
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	161 PyDoc_STRVAR(unicodedata_digit__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	162 "digit(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	163 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	164 Returns the digit value assigned to the Unicode character unichr as\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	165 integer. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	166 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	167
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	168 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	169 unicodedata_digit(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	170 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	171 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	172 PyObject *defobj = NULL;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	173 long rc;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	174 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	175
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	176 if (!PyArg_ParseTuple(args, "O!\|O:digit", &PyUnicode_Type, &v, &defobj))
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	177 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	178 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	179 if (c == (Py_UCS4)-1)
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	180 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	181 rc = Py_UNICODE_TODIGIT(c);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	182 if (rc < 0) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	183 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	184 PyErr_SetString(PyExc_ValueError, "not a digit");
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	185 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	186 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	187 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	188 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	189 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	190 }
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	191 }
44099 f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h Christian Heimes <christian@cheimes.de> parents: 42466 diff changeset	192 return PyLong_FromLong(rc);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	193 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	194
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	195 PyDoc_STRVAR(unicodedata_numeric__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	196 "numeric(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	197 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	198 Returns the numeric value assigned to the Unicode character unichr\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	199 as float. If no such value is defined, default is returned, or, if\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	200 not given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	201
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	202 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	203 unicodedata_numeric(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	204 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	205 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	206 PyObject *defobj = NULL;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	207 int have_old = 0;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	208 double rc;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	209 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	210
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	211 if (!PyArg_ParseTuple(args, "O!\|O:numeric", &PyUnicode_Type, &v, &defobj))
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	212 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	213 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	214 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	215 return NULL;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	216
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	217 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	218 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	219 if (old->category_changed == 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	220 /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	221 have_old = 1;
38453 29ae5e0fc348 Much-needed merge (using svnmerge.py this time) of trunk changes into p3yk. Thomas Wouters <thomas@python.org> parents: 37879 diff changeset	222 rc = -1.0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	223 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	224 else if (old->decimal_changed != 0xFF) {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	225 have_old = 1;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	226 rc = old->decimal_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	227 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	228 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	229
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	230 if (!have_old)
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	231 rc = Py_UNICODE_TONUMERIC(c);
38453 29ae5e0fc348 Much-needed merge (using svnmerge.py this time) of trunk changes into p3yk. Thomas Wouters <thomas@python.org> parents: 37879 diff changeset	232 if (rc == -1.0) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	233 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	234 PyErr_SetString(PyExc_ValueError, "not a numeric character");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	235 return NULL;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	236 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	237 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	238 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	239 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	240 }
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	241 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	242 return PyFloat_FromDouble(rc);
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	243 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	244
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	245 PyDoc_STRVAR(unicodedata_category__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	246 "category(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	247 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	248 Returns the general category assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	249 unichr as string.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	250
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	251 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	252 unicodedata_category(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	253 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	254 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	255 int index;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	256 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	257
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	258 if (!PyArg_ParseTuple(args, "O!:category",
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	259 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	260 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	261 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	262 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	263 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	264 index = (int) _getrecord_ex(c)->category;
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	265 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	266 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	267 if (old->category_changed != 0xFF)
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	268 index = old->category_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	269 }
41944 53963a2f5652 Change category(), bidirectional(), east_asian_width(), Walter Dörwald <walter@livinglogic.de> parents: 40450 diff changeset	270 return PyUnicode_FromString(_PyUnicode_CategoryNames[index]);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	271 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	272
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	273 PyDoc_STRVAR(unicodedata_bidirectional__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	274 "bidirectional(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	275 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	276 Returns the bidirectional category assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	277 unichr as string. If no such value is defined, an empty string is\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	278 returned.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	279
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	280 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	281 unicodedata_bidirectional(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	282 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	283 PyUnicodeObject *v;
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	284 int index;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	285 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	286
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	287 if (!PyArg_ParseTuple(args, "O!:bidirectional",
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	288 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	289 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	290 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	291 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	292 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	293 index = (int) _getrecord_ex(c)->bidirectional;
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	294 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	295 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	296 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	297 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	298 else if (old->bidir_changed != 0xFF)
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	299 index = old->bidir_changed;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	300 }
41944 53963a2f5652 Change category(), bidirectional(), east_asian_width(), Walter Dörwald <walter@livinglogic.de> parents: 40450 diff changeset	301 return PyUnicode_FromString(_PyUnicode_BidirectionalNames[index]);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	302 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	303
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	304 PyDoc_STRVAR(unicodedata_combining__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	305 "combining(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	306 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	307 Returns the canonical combining class assigned to the Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	308 character unichr as integer. Returns 0 if no combining class is\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	309 defined.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	310
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	311 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	312 unicodedata_combining(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	313 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	314 PyUnicodeObject *v;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	315 int index;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	316 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	317
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	318 if (!PyArg_ParseTuple(args, "O!:combining",
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	319 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	320 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	321 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	322 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	323 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	324 index = (int) _getrecord_ex(c)->combining;
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	325 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	326 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	327 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	328 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	329 }
44099 f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h Christian Heimes <christian@cheimes.de> parents: 42466 diff changeset	330 return PyLong_FromLong(index);
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	331 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	332
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	333 PyDoc_STRVAR(unicodedata_mirrored__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	334 "mirrored(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	335 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	336 Returns the mirrored property assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	337 unichr as integer. Returns 1 if the character has been identified as\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	338 a \"mirrored\" character in bidirectional text, 0 otherwise.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	339
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	340 static PyObject *
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	341 unicodedata_mirrored(PyObject self, PyObject args)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	342 {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	343 PyUnicodeObject *v;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	344 int index;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	345 Py_UCS4 c;
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	346
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	347 if (!PyArg_ParseTuple(args, "O!:mirrored",
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	348 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	349 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	350 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	351 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	352 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	353 index = (int) _getrecord_ex(c)->mirrored;
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	354 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	355 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	356 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	357 index = 0; /* unassigned */
49125 dc60293b29d5 Merged revisions 66362 via svnmerge from Martin v. Löwis <martin@v.loewis.de> parents: 47580 diff changeset	358 else if (old->mirrored_changed != 0xFF)
dc60293b29d5 Merged revisions 66362 via svnmerge from Martin v. Löwis <martin@v.loewis.de> parents: 47580 diff changeset	359 index = old->mirrored_changed;
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	360 }
44099 f324631462a2 Cleanup: Replaced most PyInt_ aliases with PyLong_ and disabled the aliases in intobject.h Christian Heimes <christian@cheimes.de> parents: 42466 diff changeset	361 return PyLong_FromLong(index);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	362 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	363
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	364 PyDoc_STRVAR(unicodedata_east_asian_width__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	365 "east_asian_width(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	366 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	367 Returns the east asian width assigned to the Unicode character\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	368 unichr as string.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	369
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	370 static PyObject *
32951 024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	371 unicodedata_east_asian_width(PyObject self, PyObject args)
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	372 {
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	373 PyUnicodeObject *v;
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	374 int index;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	375 Py_UCS4 c;
32951 024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	376
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	377 if (!PyArg_ParseTuple(args, "O!:east_asian_width",
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	378 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	379 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	380 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	381 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	382 return NULL;
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	383 index = (int) _getrecord_ex(c)->east_asian_width;
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	384 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	385 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	386 if (old->category_changed == 0)
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	387 index = 0; /* unassigned */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	388 }
41944 53963a2f5652 Change category(), bidirectional(), east_asian_width(), Walter Dörwald <walter@livinglogic.de> parents: 40450 diff changeset	389 return PyUnicode_FromString(_PyUnicode_EastAsianWidthNames[index]);
32951 024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	390 }
024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	391
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	392 PyDoc_STRVAR(unicodedata_decomposition__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	393 "decomposition(unichr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	394 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	395 Returns the character decomposition mapping assigned to the Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	396 character unichr as string. An empty string is returned in case no\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	397 such mapping is defined.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	398
32951 024ff0ab4588 SF #989185: Drop unicode.iswide() and unicode.width() and add Hye-Shik Chang <hyeshik@gmail.com> parents: 32680 diff changeset	399 static PyObject *
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	400 unicodedata_decomposition(PyObject self, PyObject args)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	401 {
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	402 PyUnicodeObject *v;
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	403 char decomp[256];
67347 3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int Victor Stinner <victor.stinner@haypocalc.com> parents: 67088 diff changeset	404 int code, index, count;
3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int Victor Stinner <victor.stinner@haypocalc.com> parents: 67088 diff changeset	405 size_t i;
39442 654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	406 unsigned int prefix_index;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	407 Py_UCS4 c;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	408
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	409 if (!PyArg_ParseTuple(args, "O!:decomposition",
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	410 &PyUnicode_Type, &v))
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	411 return NULL;
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	412 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	413 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	414 return NULL;
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	415
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	416 code = (int)c;
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	417
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	418 if (self && UCD_Check(self)) {
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	419 const change_record *old = get_old_record(self, c);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	420 if (old->category_changed == 0)
41944 53963a2f5652 Change category(), bidirectional(), east_asian_width(), Walter Dörwald <walter@livinglogic.de> parents: 40450 diff changeset	421 return PyUnicode_FromString(""); /* unassigned */
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	422 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	423
25852 1f5f6481886d Update to Unicode 3.2 database. Martin v. Löwis <martin@v.loewis.de> parents: 24363 diff changeset	424 if (code < 0 \|\| code >= 0x110000)
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	425 index = 0;
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	426 else {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	427 index = decomp_index1[(code>>DECOMP_SHIFT)];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	428 index = decomp_index2[(index<<DECOMP_SHIFT)+
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	429 (code&((1<<DECOMP_SHIFT)-1))];
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	430 }
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	431
21112 38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf. Tim Peters <tim.peters@gmail.com> parents: 18752 diff changeset	432 /* high byte is number of hex bytes (usually one or two), low byte
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	433 is prefix code (from*/
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	434 count = decomp_data[index] >> 8;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	435
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	436 /* XXX: could allocate the PyString up front instead
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	437 (strlen(prefix) + 5 * count + 1 bytes) */
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	438
39442 654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	439 /* Based on how index is calculated above and decomp_data is generated
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	440 from Tools/unicode/makeunicodedata.py, it should not be possible
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	441 to overflow decomp_prefix. */
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	442 prefix_index = decomp_data[index] & 255;
72510 b534b56379cb Use the new Py_ARRAY_LENGTH macro Victor Stinner <victor.stinner@haypocalc.com> parents: 72475 diff changeset	443 assert(prefix_index < Py_ARRAY_LENGTH(decomp_prefix));
39442 654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	444
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	445 /* copy prefix */
39442 654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	446 i = strlen(decomp_prefix[prefix_index]);
654c380cf8b9 Merged revisions 46753-51188 via svnmerge from Thomas Wouters <thomas@python.org> parents: 38453 diff changeset	447 memcpy(decomp, decomp_prefix[prefix_index], i);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	448
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	449 while (count-- > 0) {
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	450 if (i)
8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	451 decomp[i++] = ' ';
67347 3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int Victor Stinner <victor.stinner@haypocalc.com> parents: 67088 diff changeset	452 assert(i < sizeof(decomp));
21112 38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf. Tim Peters <tim.peters@gmail.com> parents: 18752 diff changeset	453 PyOS_snprintf(decomp + i, sizeof(decomp) - i, "%04X",
38d0bd0fa7f2 unicodedata_decomposition(): sprintf -> PyOS_snprintf. Tim Peters <tim.peters@gmail.com> parents: 18752 diff changeset	454 decomp_data[++index]);
16090 8ac40d48a581 compress unicode decomposition tables (this saves another 55k) Fredrik Lundh <fredrik@pythonware.com> parents: 14835 diff changeset	455 i += strlen(decomp + i);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	456 }
67347 3bf4d9f1660a Issue #9566: use Py_ssize_t instead of int Victor Stinner <victor.stinner@haypocalc.com> parents: 67088 diff changeset	457 return PyUnicode_FromStringAndSize(decomp, i);
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	458 }
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	459
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 36939 diff changeset	460 static void
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	461 get_decomp_record(PyObject self, Py_UCS4 code, int index, int prefix, int count)
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	462 {
28100 54b35da9879e Fix SF bug #694816, remove comparison of unsigned value < 0 Neal Norwitz <nnorwitz@gmail.com> parents: 26571 diff changeset	463 if (code >= 0x110000) {
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	464 *index = 0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	465 } else if (self && UCD_Check(self) &&
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	466 get_old_record(self, code)->category_changed==0) {
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	467 /* unassigned in old version */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	468 *index = 0;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	469 }
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	470 else {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	471 *index = decomp_index1[(code>>DECOMP_SHIFT)];
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	472 index = decomp_index2[(index<<DECOMP_SHIFT)+
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	473 (code&((1<<DECOMP_SHIFT)-1))];
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	474 }
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	475
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	476 /* high byte is number of hex bytes (usually one or two), low byte
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	477 is prefix code (from*/
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	478 count = decomp_data[index] >> 8;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	479 prefix = decomp_data[index] & 255;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	480
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	481 (*index)++;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	482 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	483
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	484 #define SBase 0xAC00
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	485 #define LBase 0x1100
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	486 #define VBase 0x1161
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	487 #define TBase 0x11A7
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	488 #define LCount 19
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	489 #define VCount 21
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	490 #define TCount 28
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	491 #define NCount (VCount*TCount)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	492 #define SCount (LCount*NCount)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	493
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	494 static PyObject*
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	495 nfd_nfkd(PyObject self, PyObject input, int k)
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	496 {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	497 PyObject *result;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	498 Py_UCS4 *output;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	499 Py_ssize_t i, o, osize;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	500 int kind;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	501 void *data;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	502 /* Longest decomposition in Unicode 3.2: U+FDFA */
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	503 Py_UCS4 stack[20];
37879 f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 36939 diff changeset	504 Py_ssize_t space, isize;
f731859e9b3b Merge p3yk branch with the trunk up to revision 45595. This breaks a fair Thomas Wouters <thomas@python.org> parents: 36939 diff changeset	505 int index, prefix, count, stackptr;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	506 unsigned char prev, cur;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	507
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	508 stackptr = 0;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	509 isize = PyUnicode_GET_LENGTH(input);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	510 /* Overallocate atmost 10 characters. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	511 space = (isize > 10 ? 10 : isize) + isize;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	512 osize = space;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	513 output = PyMem_Malloc(space * sizeof(Py_UCS4));
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	514 if (!output) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	515 PyErr_NoMemory();
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	516 return NULL;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	517 }
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	518 i = o = 0;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	519 kind = PyUnicode_KIND(input);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	520 data = PyUnicode_DATA(input);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	521
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	522 while (i < isize) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	523 stack[stackptr++] = PyUnicode_READ(kind, data, i++);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	524 while(stackptr) {
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	525 Py_UCS4 code = stack[--stackptr];
30867 04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676. Martin v. Löwis <martin@v.loewis.de> parents: 28100 diff changeset	526 /* Hangul Decomposition adds three characters in
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676. Martin v. Löwis <martin@v.loewis.de> parents: 28100 diff changeset	527 a single step, so we need atleast that much room. */
04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676. Martin v. Löwis <martin@v.loewis.de> parents: 28100 diff changeset	528 if (space < 3) {
77252 588ea940e5e3 Issue #14909: A number of places were using PyMem_Realloc() apis and Kristjan Valur Jonsson <sweskman@gmail.com> parents: 75107 diff changeset	529 Py_UCS4 *new_output;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	530 osize += 10;
30867 04ca4d1f751f Overallocate target buffer for normalization more early. Fixes #834676. Martin v. Löwis <martin@v.loewis.de> parents: 28100 diff changeset	531 space += 10;
77252 588ea940e5e3 Issue #14909: A number of places were using PyMem_Realloc() apis and Kristjan Valur Jonsson <sweskman@gmail.com> parents: 75107 diff changeset	532 new_output = PyMem_Realloc(output, osize*sizeof(Py_UCS4));
588ea940e5e3 Issue #14909: A number of places were using PyMem_Realloc() apis and Kristjan Valur Jonsson <sweskman@gmail.com> parents: 75107 diff changeset	533 if (new_output == NULL) {
588ea940e5e3 Issue #14909: A number of places were using PyMem_Realloc() apis and Kristjan Valur Jonsson <sweskman@gmail.com> parents: 75107 diff changeset	534 PyMem_Free(output);
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	535 PyErr_NoMemory();
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	536 return NULL;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	537 }
77252 588ea940e5e3 Issue #14909: A number of places were using PyMem_Realloc() apis and Kristjan Valur Jonsson <sweskman@gmail.com> parents: 75107 diff changeset	538 output = new_output;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	539 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	540 /* Hangul Decomposition. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	541 if (SBase <= code && code < (SBase+SCount)) {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	542 int SIndex = code - SBase;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	543 int L = LBase + SIndex / NCount;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	544 int V = VBase + (SIndex % NCount) / TCount;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	545 int T = TBase + SIndex % TCount;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	546 output[o++] = L;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	547 output[o++] = V;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	548 space -= 2;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	549 if (T != TBase) {
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	550 output[o++] = T;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	551 space --;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	552 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	553 continue;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	554 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	555 /* normalization changes */
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	556 if (self && UCD_Check(self)) {
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	557 Py_UCS4 value = ((PreviousDBVersion*)self)->normalization(code);
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	558 if (value != 0) {
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	559 stack[stackptr++] = value;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	560 continue;
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	561 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	562 }
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	563
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	564 /* Other decompositions. */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	565 get_decomp_record(self, code, &index, &prefix, &count);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	566
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	567 /* Copy character if it is not decomposable, or has a
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	568 compatibility decomposition, but we do NFD. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	569 if (!count \|\| (prefix && !k)) {
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	570 output[o++] = code;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	571 space--;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	572 continue;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	573 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	574 /* Copy decomposition onto the stack, in reverse
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	575 order. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	576 while(count) {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	577 code = decomp_data[index + (--count)];
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	578 stack[stackptr++] = code;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	579 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	580 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	581 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	582
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	583 result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	584 output, o);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	585 PyMem_Free(output);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	586 if (!result)
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	587 return NULL;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	588 /* result is guaranteed to be ready, as it is compact. */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	589 kind = PyUnicode_KIND(result);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	590 data = PyUnicode_DATA(result);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	591
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	592 /* Sort canonically. */
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	593 i = 0;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	594 prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	595 for (i++; i < PyUnicode_GET_LENGTH(result); i++) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	596 cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	597 if (prev == 0 \|\| cur == 0 \|\| prev <= cur) {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	598 prev = cur;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	599 continue;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	600 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	601 /* Non-canonical order. Need to switch i with previous. /
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	602 o = i - 1;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	603 while (1) {
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	604 Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	605 PyUnicode_WRITE(kind, data, o+1,
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	606 PyUnicode_READ(kind, data, o));
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	607 PyUnicode_WRITE(kind, data, o, tmp);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	608 o--;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	609 if (o < 0)
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	610 break;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	611 prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	612 if (prev == 0 \|\| prev <= cur)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	613 break;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	614 }
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	615 prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	616 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	617 return result;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	618 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	619
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	620 static int
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	621 find_nfc_index(PyObject self, struct reindex nfc, Py_UCS4 code)
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	622 {
72751 d760b345d7bd Fix compilation warnings under 64-bit Windows Antoine Pitrou <solipsis@pitrou.net> parents: 72656 diff changeset	623 unsigned int index;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	624 for (index = 0; nfc[index].start; index++) {
72751 d760b345d7bd Fix compilation warnings under 64-bit Windows Antoine Pitrou <solipsis@pitrou.net> parents: 72656 diff changeset	625 unsigned int start = nfc[index].start;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	626 if (code < start)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	627 return -1;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	628 if (code <= start + nfc[index].count) {
72751 d760b345d7bd Fix compilation warnings under 64-bit Windows Antoine Pitrou <solipsis@pitrou.net> parents: 72656 diff changeset	629 unsigned int delta = code - start;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	630 return nfc[index].index + delta;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	631 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	632 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	633 return -1;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	634 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	635
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	636 static PyObject*
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	637 nfc_nfkc(PyObject self, PyObject input, int k)
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	638 {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	639 PyObject *result;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	640 int kind;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	641 void *data;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	642 Py_UCS4 *output;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	643 Py_ssize_t i, i1, o, len;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	644 int f,l,index,index1,comb;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	645 Py_UCS4 code;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	646 Py_ssize_t skipped[20];
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	647 int cskipped = 0;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	648
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	649 result = nfd_nfkd(self, input, k);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	650 if (!result)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	651 return NULL;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	652 /* result will be "ready". */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	653 kind = PyUnicode_KIND(result);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	654 data = PyUnicode_DATA(result);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	655 len = PyUnicode_GET_LENGTH(result);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	656
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	657 /* We allocate a buffer for the output.
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	658 If we find that we made no changes, we still return
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	659 the NFD result. */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	660 output = PyMem_Malloc(len * sizeof(Py_UCS4));
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	661 if (!output) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	662 PyErr_NoMemory();
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	663 Py_DECREF(result);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	664 return 0;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	665 }
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	666 i = o = 0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	667
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	668 again:
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	669 while (i < len) {
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	670 for (index = 0; index < cskipped; index++) {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	671 if (skipped[index] == i) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	672 /* *i character is skipped.
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	673 Remove from list. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	674 skipped[index] = skipped[cskipped-1];
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	675 cskipped--;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	676 i++;
26571 cb8cfadf75a2 Remove C++ comment. Martin v. Löwis <martin@v.loewis.de> parents: 26397 diff changeset	677 goto again; /* continue while */
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	678 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	679 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	680 /* Hangul Composition. We don't need to check for <LV,T>
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	681 pairs, since we always have decomposed data. */
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	682 code = PyUnicode_READ(kind, data, i);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	683 if (LBase <= code && code < (LBase+LCount) &&
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	684 i + 1 < len &&
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	685 VBase <= PyUnicode_READ(kind, data, i+1) &&
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	686 PyUnicode_READ(kind, data, i+1) <= (VBase+VCount)) {
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	687 int LIndex, VIndex;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	688 LIndex = code - LBase;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	689 VIndex = PyUnicode_READ(kind, data, i+1) - VBase;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	690 code = SBase + (LIndexVCount+VIndex)TCount;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	691 i+=2;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	692 if (i < len &&
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	693 TBase <= PyUnicode_READ(kind, data, i) &&
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	694 PyUnicode_READ(kind, data, i) <= (TBase+TCount)) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	695 code += PyUnicode_READ(kind, data, i)-TBase;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	696 i++;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	697 }
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	698 output[o++] = code;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	699 continue;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	700 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	701
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	702 /* code is still input[i] here */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	703 f = find_nfc_index(self, nfc_first, code);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	704 if (f == -1) {
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	705 output[o++] = code;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	706 i++;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	707 continue;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	708 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	709 /* Find next unblocked character. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	710 i1 = i+1;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	711 comb = 0;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	712 /* output base character for now; might be updated later. */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	713 output[o] = PyUnicode_READ(kind, data, i);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	714 while (i1 < len) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	715 Py_UCS4 code1 = PyUnicode_READ(kind, data, i1);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	716 int comb1 = _getrecord_ex(code1)->combining;
67184 a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	717 if (comb) {
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	718 if (comb1 == 0)
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	719 break;
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	720 if (comb >= comb1) {
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	721 /* Character is blocked. */
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	722 i1++;
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	723 continue;
a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	724 }
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	725 }
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	726 l = find_nfc_index(self, nfc_last, code1);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	727 /* i1 cannot be combined with i. If i1
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	728 is a starter, we don't need to look further.
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	729 Otherwise, record the combining class. */
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	730 if (l == -1) {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	731 not_combinable:
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	732 if (comb1 == 0)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	733 break;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	734 comb = comb1;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	735 i1++;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	736 continue;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	737 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	738 index = f*TOTAL_LAST + l;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	739 index1 = comp_index[index >> COMP_SHIFT];
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	740 code = comp_data[(index1<<COMP_SHIFT)+
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	741 (index&((1<<COMP_SHIFT)-1))];
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	742 if (code == 0)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	743 goto not_combinable;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	744
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	745 /* Replace the original character. */
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	746 output[o] = code;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	747 /* Mark the second character unused. */
67184 a4ec20cc9066 Merged revisions 87442 via svnmerge from Alexander Belopolsky <alexander.belopolsky@gmail.com> parents: 66396 diff changeset	748 assert(cskipped < 20);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	749 skipped[cskipped++] = i1;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	750 i1++;
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	751 f = find_nfc_index(self, nfc_first, output[o]);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	752 if (f == -1)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	753 break;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	754 }
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	755 /* Output character was already written.
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	756 Just advance the indices. */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	757 o++; i++;
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	758 }
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	759 if (o == len) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	760 /* No changes. Return original string. */
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	761 PyMem_Free(output);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	762 return result;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	763 }
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	764 Py_DECREF(result);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	765 result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	766 output, o);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	767 PyMem_Free(output);
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	768 return result;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	769 }
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	770
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	771 /* Return 1 if the input is certainly normalized, 0 if it might not be. */
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	772 static int
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	773 is_normalized(PyObject self, PyObject input, int nfc, int k)
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	774 {
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	775 Py_ssize_t i, len;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	776 int kind;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	777 void *data;
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	778 unsigned char prev_combining = 0, quickcheck_mask;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	779
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	780 /* An older version of the database is requested, quickchecks must be
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	781 disabled. */
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	782 if (self && UCD_Check(self))
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	783 return 0;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	784
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	785 /* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	786 as described in http://unicode.org/reports/tr15/#Annex8. */
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	787 quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	788
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	789 i = 0;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	790 kind = PyUnicode_KIND(input);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	791 data = PyUnicode_DATA(input);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	792 len = PyUnicode_GET_LENGTH(input);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	793 while (i < len) {
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	794 Py_UCS4 ch = PyUnicode_READ(kind, data, i++);
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	795 const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch);
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	796 unsigned char combining = record->combining;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	797 unsigned char quickcheck = record->normalization_quick_check;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	798
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	799 if (quickcheck & quickcheck_mask)
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	800 return 0; /* this string might need normalization */
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	801 if (combining && prev_combining > combining)
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	802 return 0; /* non-canonical sort order, not normalized */
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	803 prev_combining = combining;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	804 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	805 return 1; /* certainly normalized */
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	806 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	807
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	808 PyDoc_STRVAR(unicodedata_normalize__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	809 "normalize(form, unistr)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	810 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	811 Return the normal form 'form' for the Unicode string unistr. Valid\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	812 values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	813
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	814 static PyObject*
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	815 unicodedata_normalize(PyObject self, PyObject args)
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	816 {
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	817 char *form;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	818 PyObject *input;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	819
32680 d3d426483d71 Fix typo. Hye-Shik Chang <hyeshik@gmail.com> parents: 31930 diff changeset	820 if(!PyArg_ParseTuple(args, "sO!:normalize",
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	821 &form, &PyUnicode_Type, &input))
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	822 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	823
72525 40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	824 if (PyUnicode_READY(input) == -1)
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	825 return NULL;
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	826
40cf7204ca2b Port normalization to new API. Martin v. Löwis <martin@v.loewis.de> parents: 72510 diff changeset	827 if (PyUnicode_GET_LENGTH(input) == 0) {
31930 f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361. Martin v. Löwis <martin@v.loewis.de> parents: 30867 diff changeset	828 /* Special case empty input strings, since resizing
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361. Martin v. Löwis <martin@v.loewis.de> parents: 30867 diff changeset	829 them later would cause internal errors. */
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361. Martin v. Löwis <martin@v.loewis.de> parents: 30867 diff changeset	830 Py_INCREF(input);
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361. Martin v. Löwis <martin@v.loewis.de> parents: 30867 diff changeset	831 return input;
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361. Martin v. Löwis <martin@v.loewis.de> parents: 30867 diff changeset	832 }
f8c0ac5bef02 Special case normalization of empty strings. Fixes #924361. Martin v. Löwis <martin@v.loewis.de> parents: 30867 diff changeset	833
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	834 if (strcmp(form, "NFC") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	835 if (is_normalized(self, input, 1, 0)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	836 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	837 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	838 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	839 return nfc_nfkc(self, input, 0);
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	840 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	841 if (strcmp(form, "NFKC") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	842 if (is_normalized(self, input, 1, 1)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	843 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	844 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	845 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	846 return nfc_nfkc(self, input, 1);
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	847 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	848 if (strcmp(form, "NFD") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	849 if (is_normalized(self, input, 0, 0)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	850 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	851 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	852 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	853 return nfd_nfkd(self, input, 0);
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	854 }
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	855 if (strcmp(form, "NFKD") == 0) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	856 if (is_normalized(self, input, 0, 1)) {
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	857 Py_INCREF(input);
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	858 return input;
423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	859 }
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	860 return nfd_nfkd(self, input, 1);
53554 423f8f9b07b1 Merged revisions 72054 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 51271 diff changeset	861 }
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	862 PyErr_SetString(PyExc_ValueError, "invalid normalization form");
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	863 return NULL;
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	864 }
1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	865
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	866 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	867 /* unicode character name tables */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	868
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	869 /* data file generated by Tools/unicode/makeunicodedata.py */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	870 #include "unicodename_db.h"
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	871
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	872 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	873 /* database code (cut and pasted from the unidb package) */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	874
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	875 static unsigned long
16666 e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS Fredrik Lundh <fredrik@pythonware.com> parents: 16662 diff changeset	876 _gethash(const char *s, int len, int scale)
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	877 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	878 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	879 unsigned long h = 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	880 unsigned long ix;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	881 for (i = 0; i < len; i++) {
72655 fe48e2b3dbee Remove all other uses of the C tolower()/toupper() which could break with a Turkish locale. Antoine Pitrou <solipsis@pitrou.net> parents: 68257 diff changeset	882 h = (h * scale) + (unsigned char) Py_TOUPPER(Py_CHARMASK(s[i]));
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	883 ix = h & 0xff000000;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	884 if (ix)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	885 h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	886 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	887 return h;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	888 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	889
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	890 static char *hangul_syllables[][3] = {
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	891 { "G", "A", "" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	892 { "GG", "AE", "G" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	893 { "N", "YA", "GG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	894 { "D", "YAE", "GS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	895 { "DD", "EO", "N", },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	896 { "R", "E", "NJ" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	897 { "M", "YEO", "NH" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	898 { "B", "YE", "D" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	899 { "BB", "O", "L" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	900 { "S", "WA", "LG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	901 { "SS", "WAE", "LM" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	902 { "", "OE", "LB" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	903 { "J", "YO", "LS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	904 { "JJ", "U", "LT" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	905 { "C", "WEO", "LP" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	906 { "K", "WE", "LH" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	907 { "T", "WI", "M" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	908 { "P", "YU", "B" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	909 { "H", "EU", "BS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	910 { 0, "YI", "S" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	911 { 0, "I", "SS" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	912 { 0, 0, "NG" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	913 { 0, 0, "J" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	914 { 0, 0, "C" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	915 { 0, 0, "K" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	916 { 0, 0, "T" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	917 { 0, 0, "P" },
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	918 { 0, 0, "H" }
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	919 };
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	920
66392 d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0. Martin v. Löwis <martin@v.loewis.de> parents: 64423 diff changeset	921 /* These ranges need to match makeunicodedata.py:cjk_ranges. */
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	922 static int
26380 f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	923 is_unified_ideograph(Py_UCS4 code)
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	924 {
66392 d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0. Martin v. Löwis <martin@v.loewis.de> parents: 64423 diff changeset	925 return
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0. Martin v. Löwis <martin@v.loewis.de> parents: 64423 diff changeset	926 (0x3400 <= code && code <= 0x4DB5) \|\| /* CJK Ideograph Extension A */
75107 cd21e6d30a1a update to Unicode 6.1 Benjamin Peterson <benjamin@python.org> parents: 73475 diff changeset	927 (0x4E00 <= code && code <= 0x9FCC) \|\| /* CJK Ideograph */
66392 d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0. Martin v. Löwis <martin@v.loewis.de> parents: 64423 diff changeset	928 (0x20000 <= code && code <= 0x2A6D6) \|\| /* CJK Ideograph Extension B */
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0. Martin v. Löwis <martin@v.loewis.de> parents: 64423 diff changeset	929 (0x2A700 <= code && code <= 0x2B734) \|\| /* CJK Ideograph Extension C */
d9774121a6df Issue #10459: Update CJK character names to Unicode 6.0. Martin v. Löwis <martin@v.loewis.de> parents: 64423 diff changeset	930 (0x2B740 <= code && code <= 0x2B81D); /* CJK Ideograph Extension D */
26380 f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	931 }
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	932
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	933 /* macros used to determine if the given codepoint is in the PUA range that
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	934 * we are using to store aliases and named sequences */
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	935 #define IS_ALIAS(cp) ((cp >= aliases_start) && (cp < aliases_end))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	936 #define IS_NAMED_SEQ(cp) ((cp >= named_sequences_start) && \
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	937 (cp < named_sequences_end))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	938
26380 f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	939 static int
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	940 _getucname(PyObject self, Py_UCS4 code, char buffer, int buflen,
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	941 int with_alias_and_seq)
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	942 {
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	943 /* Find the name associated with the given codepoint.
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	944 * If with_alias_and_seq is 1, check for names in the Private Use Area 15
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	945 * that we are using for aliases and named sequences. */
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	946 int offset;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	947 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	948 int word;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	949 unsigned char* w;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	950
36939 03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	951 if (code >= 0x110000)
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	952 return 0;
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	953
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	954 /* XXX should we just skip all the codepoints in the PUAs here? */
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	955 if (!with_alias_and_seq && (IS_ALIAS(code) \|\| IS_NAMED_SEQ(code)))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	956 return 0;
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	957
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	958 if (self && UCD_Check(self)) {
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	959 /* in 3.2.0 there are no aliases and named sequences */
73047 329b96fe4472 #12753: fix compilation on Windows. Ezio Melotti <ezio.melotti@gmail.com> parents: 73044 diff changeset	960 const change_record *old;
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	961 if (IS_ALIAS(code) \|\| IS_NAMED_SEQ(code))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	962 return 0;
73047 329b96fe4472 #12753: fix compilation on Windows. Ezio Melotti <ezio.melotti@gmail.com> parents: 73044 diff changeset	963 old = get_old_record(self, code);
36939 03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	964 if (old->category_changed == 0) {
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	965 /* unassigned */
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	966 return 0;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	967 }
36939 03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	968 }
03ef987043ce Adjust CJK Ideograph range to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36928 diff changeset	969
26374 135bb11215a0 Fix off-by-one error. Martin v. Löwis <martin@v.loewis.de> parents: 26370 diff changeset	970 if (SBase <= code && code < SBase+SCount) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	971 /* Hangul syllable. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	972 int SIndex = code - SBase;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	973 int L = SIndex / NCount;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	974 int V = (SIndex % NCount) / TCount;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	975 int T = SIndex % TCount;
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	976
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	977 if (buflen < 27)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	978 /* Worst case: HANGUL SYLLABLE <10chars>. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	979 return 0;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	980 strcpy(buffer, "HANGUL SYLLABLE ");
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	981 buffer += 16;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	982 strcpy(buffer, hangul_syllables[L][0]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	983 buffer += strlen(hangul_syllables[L][0]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	984 strcpy(buffer, hangul_syllables[V][1]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	985 buffer += strlen(hangul_syllables[V][1]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	986 strcpy(buffer, hangul_syllables[T][2]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	987 buffer += strlen(hangul_syllables[T][2]);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	988 *buffer = '\0';
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	989 return 1;
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	990 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	991
26380 f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	992 if (is_unified_ideograph(code)) {
26376 0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	993 if (buflen < 28)
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	994 /* Worst case: CJK UNIFIED IDEOGRAPH-20000 */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	995 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	996 sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code);
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	997 return 1;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	998 }
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	999
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1000 /* get offset into phrasebook */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1001 offset = phrasebook_offset1[(code>>phrasebook_shift)];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1002 offset = phrasebook_offset2[(offset<<phrasebook_shift) +
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1003 (code&((1<<phrasebook_shift)-1))];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1004 if (!offset)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1005 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1006
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1007 i = 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1008
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1009 for (;;) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1010 /* get word index */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1011 word = phrasebook[offset] - phrasebook_short;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1012 if (word >= 0) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1013 word = (word << 8) + phrasebook[offset+1];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1014 offset += 2;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1015 } else
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1016 word = phrasebook[offset++];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1017 if (i) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1018 if (i > buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1019 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1020 buffer[i++] = ' ';
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1021 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1022 /* copy word string from lexicon. the last character in the
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1023 word has bit 7 set. the last word in a string ends with
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1024 0x80 */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1025 w = lexicon + lexicon_offset[word];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1026 while (*w < 128) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1027 if (i >= buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1028 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1029 buffer[i++] = *w++;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1030 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1031 if (i >= buflen)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1032 return 0; /* buffer overflow */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1033 buffer[i++] = *w & 127;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1034 if (*w == 128)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1035 break; /* end of word */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1036 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1037
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1038 return 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1039 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1040
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1041 static int
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	1042 _cmpname(PyObject self, int code, const char name, int namelen)
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1043 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1044 /* check if code corresponds to the given name */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1045 int i;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1046 char buffer[NAME_MAXLEN];
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1047 if (!_getucname(self, code, buffer, sizeof(buffer), 1))
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1048 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1049 for (i = 0; i < namelen; i++) {
72655 fe48e2b3dbee Remove all other uses of the C tolower()/toupper() which could break with a Turkish locale. Antoine Pitrou <solipsis@pitrou.net> parents: 68257 diff changeset	1050 if (Py_TOUPPER(Py_CHARMASK(name[i])) != buffer[i])
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1051 return 0;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1052 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1053 return buffer[namelen] == '\0';
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1054 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1055
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1056 static void
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1057 find_syllable(const char str, int len, int *pos, int count, int column)
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1058 {
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1059 int i, len1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1060 *len = -1;
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1061 for (i = 0; i < count; i++) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1062 char *s = hangul_syllables[i][column];
72751 d760b345d7bd Fix compilation warnings under 64-bit Windows Antoine Pitrou <solipsis@pitrou.net> parents: 72656 diff changeset	1063 len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int);
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1064 if (len1 <= *len)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1065 continue;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1066 if (strncmp(str, s, len1) == 0) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1067 *len = len1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1068 *pos = i;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1069 }
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1070 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1071 if (*len == -1) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1072 *len = 0;
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1073 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1074 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1075
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1076 static int
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1077 _check_alias_and_seq(unsigned int cp, Py_UCS4* code, int with_named_seq)
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1078 {
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1079 /* check if named sequences are allowed */
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1080 if (!with_named_seq && IS_NAMED_SEQ(cp))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1081 return 0;
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1082 /* if the codepoint is in the PUA range that we use for aliases,
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1083 * convert it to obtain the right codepoint */
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1084 if (IS_ALIAS(cp))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1085 *code = name_aliases[cp-aliases_start];
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1086 else
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1087 *code = cp;
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1088 return 1;
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1089 }
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1090
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1091 static int
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1092 _getcode(PyObject* self, const char* name, int namelen, Py_UCS4* code,
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1093 int with_named_seq)
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1094 {
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1095 /* Return the codepoint associated with the given name.
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1096 * Named aliases are resolved too (unless self != NULL (i.e. we are using
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1097 * 3.2.0)). If with_named_seq is 1, returns the PUA codepoint that we are
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1098 * using for the named sequence, and the caller must then convert it. */
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1099 unsigned int h, v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1100 unsigned int mask = code_size-1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1101 unsigned int i, incr;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1102
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1103 /* Check for hangul syllables. */
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1104 if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1105 int len, L = -1, V = -1, T = -1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1106 const char *pos = name + 16;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1107 find_syllable(pos, &len, &L, LCount, 0);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1108 pos += len;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1109 find_syllable(pos, &len, &V, VCount, 1);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1110 pos += len;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1111 find_syllable(pos, &len, &T, TCount, 2);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1112 pos += len;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1113 if (L != -1 && V != -1 && T != -1 && pos-name == namelen) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1114 code = SBase + (LVCount+V)*TCount + T;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1115 return 1;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1116 }
26376 0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1117 /* Otherwise, it's an illegal syllable name. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1118 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1119 }
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1120
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1121 /* Check for unified ideographs. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1122 if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) {
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1123 /* Four or five hexdigits must follow. */
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1124 v = 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1125 name += 22;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1126 namelen -= 22;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1127 if (namelen != 4 && namelen != 5)
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1128 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1129 while (namelen--) {
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1130 v *= 16;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1131 if (name >= '0' && name <= '9')
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1132 v += *name - '0';
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1133 else if (name >= 'A' && name <= 'F')
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1134 v += *name - 'A' + 10;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1135 else
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1136 return 0;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1137 name++;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1138 }
26380 f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	1139 if (!is_unified_ideograph(v))
f8a1ba446158 Verify that the code in CJK UNIFIED IDEOGRAPH- actually denotes an ideograph. Martin v. Löwis <martin@v.loewis.de> parents: 26379 diff changeset	1140 return 0;
26376 0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1141 *code = v;
0f2766302ba0 Implement names for CJK unified ideographs. Add name to KeyError output. Martin v. Löwis <martin@v.loewis.de> parents: 26374 diff changeset	1142 return 1;
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1143 }
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1144
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1145 /* the following is the same as python's dictionary lookup, with
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1146 only minor changes. see the makeunicodedata script for more
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1147 details */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1148
16666 e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS Fredrik Lundh <fredrik@pythonware.com> parents: 16662 diff changeset	1149 h = (unsigned int) _gethash(name, namelen, code_magic);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1150 i = (~h) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1151 v = code_hash[i];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1152 if (!v)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1153 return 0;
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1154 if (_cmpname(self, v, name, namelen))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1155 return _check_alias_and_seq(v, code, with_named_seq);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1156 incr = (h ^ (h >> 3)) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1157 if (!incr)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1158 incr = mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1159 for (;;) {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1160 i = (i + incr) & mask;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1161 v = code_hash[i];
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1162 if (!v)
16662 89c1ccef5ce8 stupid typo (for some reason, this only caused problems on OpenVMS). Fredrik Lundh <fredrik@pythonware.com> parents: 16200 diff changeset	1163 return 0;
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1164 if (_cmpname(self, v, name, namelen))
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1165 return _check_alias_and_seq(v, code, with_named_seq);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1166 incr = incr << 1;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1167 if (incr > mask)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1168 incr = incr ^ code_poly;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1169 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1170 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1171
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1172 static const _PyUnicode_Name_CAPI hashAPI =
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1173 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1174 sizeof(_PyUnicode_Name_CAPI),
23810 01e4b698ff7e _Py prefix is verboten for static entry points Andrew MacIntyre <andymac@bullseye.apana.org.au> parents: 22781 diff changeset	1175 _getucname,
16666 e9fcb46c1f2c renamed internal functions to avoid name clashes under OpenVMS Fredrik Lundh <fredrik@pythonware.com> parents: 16662 diff changeset	1176 _getcode
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1177 };
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1178
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1179 /* -------------------------------------------------------------------- */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1180 /* Python bindings */
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1181
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1182 PyDoc_STRVAR(unicodedata_name__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1183 "name(unichr[, default])\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1184 Returns the name assigned to the Unicode character unichr as a\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1185 string. If no name is defined, default is returned, or, if not\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1186 given, ValueError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1187
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1188 static PyObject *
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1189 unicodedata_name(PyObject* self, PyObject* args)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1190 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1191 char name[NAME_MAXLEN];
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	1192 Py_UCS4 c;
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1193
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1194 PyUnicodeObject* v;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1195 PyObject* defobj = NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1196 if (!PyArg_ParseTuple(args, "O!\|O:name", &PyUnicode_Type, &v, &defobj))
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1197 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1198
47458 e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	1199 c = getuchar(v);
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	1200 if (c == (Py_UCS4)-1)
e4a186437440 Change all functions that expect one unicode character to accept a pair of Walter Dörwald <walter@livinglogic.de> parents: 44351 diff changeset	1201 return NULL;
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1202
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1203 if (!_getucname(self, c, name, sizeof(name), 0)) {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1204 if (defobj == NULL) {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1205 PyErr_SetString(PyExc_ValueError, "no such name");
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1206 return NULL;
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1207 }
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1208 else {
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1209 Py_INCREF(defobj);
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1210 return defobj;
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1211 }
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1212 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1213
41944 53963a2f5652 Change category(), bidirectional(), east_asian_width(), Walter Dörwald <walter@livinglogic.de> parents: 40450 diff changeset	1214 return PyUnicode_FromString(name);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1215 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1216
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1217 PyDoc_STRVAR(unicodedata_lookup__doc__,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1218 "lookup(name)\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1219 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1220 Look up character by name. If a character with the\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1221 given name is found, return the corresponding Unicode\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1222 character. If not found, KeyError is raised.");
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1223
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1224 static PyObject *
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1225 unicodedata_lookup(PyObject* self, PyObject* args)
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1226 {
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1227 Py_UCS4 code;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1228
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1229 char* name;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1230 int namelen;
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1231 unsigned int index;
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1232 if (!PyArg_ParseTuple(args, "s#:lookup", &name, &namelen))
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1233 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1234
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1235 if (!_getcode(self, name, namelen, &code, 1)) {
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1236 PyErr_Format(PyExc_KeyError, "undefined character name '%s'", name);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1237 return NULL;
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1238 }
79102 c843ec292a8b Use C-style comments (required for the AIX build slave). Stefan Krah <skrah@bytereef.org> parents: 77252 diff changeset	1239 /* check if code is in the PUA range that we use for named sequences
c843ec292a8b Use C-style comments (required for the AIX build slave). Stefan Krah <skrah@bytereef.org> parents: 77252 diff changeset	1240 and convert it */
73044 a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1241 if (IS_NAMED_SEQ(code)) {
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1242 index = code-named_sequences_start;
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1243 return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND,
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1244 named_sequences[index].seq,
a985d733b3a3 #12753: Add support for Unicode name aliases and named sequences. Ezio Melotti <ezio.melotti@gmail.com> parents: 72751 diff changeset	1245 named_sequences[index].seqlen);
42466 3fc3cedc49ca Merged revisions 56753-56781 via svnmerge from Guido van Rossum <guido@python.org> parents: 42334 diff changeset	1246 }
72475 8beaa9a37387 Implement PEP 393. Martin v. Löwis <martin@v.loewis.de> parents: 68257 diff changeset	1247 return PyUnicode_FromOrdinal(code);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1248 }
3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1249
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1250 /* XXX Add doc strings. */
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1251
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1252 static PyMethodDef unicodedata_functions[] = {
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1253 {"decimal", unicodedata_decimal, METH_VARARGS, unicodedata_decimal__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1254 {"digit", unicodedata_digit, METH_VARARGS, unicodedata_digit__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1255 {"numeric", unicodedata_numeric, METH_VARARGS, unicodedata_numeric__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1256 {"category", unicodedata_category, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1257 unicodedata_category__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1258 {"bidirectional", unicodedata_bidirectional, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1259 unicodedata_bidirectional__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1260 {"combining", unicodedata_combining, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1261 unicodedata_combining__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1262 {"mirrored", unicodedata_mirrored, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1263 unicodedata_mirrored__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1264 {"east_asian_width", unicodedata_east_asian_width, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1265 unicodedata_east_asian_width__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1266 {"decomposition", unicodedata_decomposition, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1267 unicodedata_decomposition__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1268 {"name", unicodedata_name, METH_VARARGS, unicodedata_name__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1269 {"lookup", unicodedata_lookup, METH_VARARGS, unicodedata_lookup__doc__},
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1270 {"normalize", unicodedata_normalize, METH_VARARGS,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1271 unicodedata_normalize__doc__},
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1272 {NULL, NULL} /* sentinel */
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1273 };
d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1274
36927 3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1275 static PyTypeObject UCD_Type = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1276 /* The ob_type field must be initialized in the module init function
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1277 * to be portable to Windows without using C++. */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1278 PyVarObject_HEAD_INIT(NULL, 0)
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1279 "unicodedata.UCD", /tp_name/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1280 sizeof(PreviousDBVersion), /tp_basicsize/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1281 0, /tp_itemsize/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1282 /* methods */
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1283 (destructor)PyObject_Del, /tp_dealloc/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1284 0, /tp_print/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1285 0, /tp_getattr/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1286 0, /tp_setattr/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1287 0, /tp_reserved/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1288 0, /tp_repr/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1289 0, /tp_as_number/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1290 0, /tp_as_sequence/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1291 0, /tp_as_mapping/
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1292 0, /tp_hash/
36927 3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1293 0, /tp_call/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1294 0, /tp_str/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1295 PyObject_GenericGetAttr,/tp_getattro/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1296 0, /tp_setattro/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1297 0, /tp_as_buffer/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1298 Py_TPFLAGS_DEFAULT, /tp_flags/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1299 0, /tp_doc/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1300 0, /tp_traverse/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1301 0, /tp_clear/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1302 0, /tp_richcompare/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1303 0, /tp_weaklistoffset/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1304 0, /tp_iter/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1305 0, /tp_iternext/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1306 unicodedata_functions, /tp_methods/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1307 DB_members, /tp_members/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1308 0, /tp_getset/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1309 0, /tp_base/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1310 0, /tp_dict/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1311 0, /tp_descr_get/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1312 0, /tp_descr_set/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1313 0, /tp_dictoffset/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1314 0, /tp_init/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1315 0, /tp_alloc/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1316 0, /tp_new/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1317 0, /tp_free/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1318 0, /tp_is_gc/
3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1319 };
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	1320
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1321 PyDoc_STRVAR(unicodedata_docstring,
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1322 "This module provides access to the Unicode Character Database which\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1323 defines character properties for all Unicode characters. The data in\n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1324 this database is based on the UnicodeData.txt file version\n\
73474 6ddda809ea8c #13379: update Unicode version in unicodedata docstrings and comments. Ezio Melotti <ezio.melotti@gmail.com> parents: 72655 diff changeset	1325 6.0.0 which is publically available from ftp://ftp.unicode.org/.\n\
34864 34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1326 \n\
34cdfc5dd7ea Fill docstrings for module and functions, extracted from the tex Hye-Shik Chang <hyeshik@gmail.com> parents: 32951 diff changeset	1327 The module uses the same names and symbols as defined by the\n\
73474 6ddda809ea8c #13379: update Unicode version in unicodedata docstrings and comments. Ezio Melotti <ezio.melotti@gmail.com> parents: 72655 diff changeset	1328 UnicodeData File Format 6.0.0 (see\n\
6ddda809ea8c #13379: update Unicode version in unicodedata docstrings and comments. Ezio Melotti <ezio.melotti@gmail.com> parents: 72655 diff changeset	1329 http://www.unicode.org/reports/tr44/tr44-6.html).");
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1330
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1331
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1332 static struct PyModuleDef unicodedatamodule = {
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1333 PyModuleDef_HEAD_INIT,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1334 "unicodedata",
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1335 unicodedata_docstring,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1336 -1,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1337 unicodedata_functions,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1338 NULL,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1339 NULL,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1340 NULL,
f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1341 NULL
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1342 };
94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1343
24363 979ff5d0ae07 Replace DL_IMPORT with PyMODINIT_FUNC and remove "/export:init..." link Mark Hammond <mhammond@skippinet.com.au> parents: 23832 diff changeset	1344 PyMODINIT_FUNC
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1345 PyInit_unicodedata(void)
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1346 {
22781 ec67fce3fa3c Remove direct manipulation of the module dict. Fred Drake <fdrake@acm.org> parents: 22114 diff changeset	1347 PyObject m, v;
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1348
44351 86a2679dca46 #1629: Renamed Py_Size, Py_Type and Py_Refcnt to Py_SIZE, Py_TYPE and Py_REFCNT. Christian Heimes <christian@cheimes.de> parents: 44099 diff changeset	1349 Py_TYPE(&UCD_Type) = &PyType_Type;
36927 3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1350
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1351 m = PyModule_Create(&unicodedatamodule);
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1352 if (!m)
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1353 return NULL;
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1354
26397 b27c1cad77ea Add unidata_version. Bump generator version number. Martin v. Löwis <martin@v.loewis.de> parents: 26380 diff changeset	1355 PyModule_AddStringConstant(m, "unidata_version", UNIDATA_VERSION);
36928 11f8b5b8ce20 Fix refcounting bug. Martin v. Löwis <martin@v.loewis.de> parents: 36927 diff changeset	1356 Py_INCREF(&UCD_Type);
36927 3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1357 PyModule_AddObject(m, "UCD", (PyObject*)&UCD_Type);
26397 b27c1cad77ea Add unidata_version. Bump generator version number. Martin v. Löwis <martin@v.loewis.de> parents: 26380 diff changeset	1358
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	1359 /* Previous versions */
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	1360 v = new_previous_version("3.2.0", get_change_3_2_0, normalization_3_2_0);
48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	1361 if (v != NULL)
36927 3266a020a0da Avoid forward-declaring the methods array. Martin v. Löwis <martin@v.loewis.de> parents: 36924 diff changeset	1362 PyModule_AddObject(m, "ucd_3_2_0", v);
36924 48e0a5e39403 Update Unicode database to Unicode 4.1. Martin v. Löwis <martin@v.loewis.de> parents: 36778 diff changeset	1363
16200 3484670c37f2 Move uchhash functionality into unicodedata (after the recent Fredrik Lundh <fredrik@pythonware.com> parents: 16091 diff changeset	1364 /* Export C API */
53816 911df82bc25d add a replacement API for PyCObject, PyCapsule #5630 Benjamin Peterson <benjamin@python.org> parents: 53554 diff changeset	1365 v = PyCapsule_New((void *)&hashAPI, PyUnicodeData_CAPSULE_NAME, NULL);
22781 ec67fce3fa3c Remove direct manipulation of the module dict. Fred Drake <fdrake@acm.org> parents: 22114 diff changeset	1366 if (v != NULL)
ec67fce3fa3c Remove direct manipulation of the module dict. Fred Drake <fdrake@acm.org> parents: 22114 diff changeset	1367 PyModule_AddObject(m, "ucnhash_CAPI", v);
47580 94f64f11fdd0 Implement PEP 3121: new module initialization and finalization API. Martin v. Löwis <martin@v.loewis.de> parents: 47458 diff changeset	1368 return m;
11981 d7a4a5ae2dae Module unicodedata -- Provides access to the Unicode 3.0 data base. Guido van Rossum <guido@python.org> parents: diff changeset	1369 }
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1370
61467 f197407bdba2 Recorded merge of revisions 81032 via svnmerge from Antoine Pitrou <solipsis@pitrou.net> parents: 59325 diff changeset	1371 /*
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1372 Local variables:
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1373 c-basic-offset: 4
26379 1885b2d6393a Patch #626485: Support Unicode normalization. Martin v. Löwis <martin@v.loewis.de> parents: 26376 diff changeset	1374 indent-tabs-mode: nil
26370 9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1375 End:
9e489a0df7b5 Patch #626548: Support Hangul syllable names. Martin v. Löwis <martin@v.loewis.de> parents: 25852 diff changeset	1376 */

Mercurial > cpython-lru_cache-weakref

annotate Modules/unicodedata.c @ 79691:05c8928362ee