cpython-lru_cache-weakref: Python/fileutils.c annotate

annotate Python/fileutils.c @ 86913:8c9769b17171 3.2

Backout 7d399099334d.

author	Georg Brandl <georg@python.org>
date	Mon, 04 Nov 2013 07:44:29 +0100
parents	c256764e2b3f
children	5bb289e4fb35

rev	line source
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	1 #include "Python.h"
65164 1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	2 #ifdef MS_WINDOWS
1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	3 # include <windows.h>
1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	4 #endif
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	5
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	6 #ifdef HAVE_LANGINFO_H
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	7 #include <locale.h>
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	8 #include <langinfo.h>
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	9 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	10
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	11 #ifdef __APPLE__
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	12 extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	13 #endif
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	14
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	15 #if !defined(__APPLE__) && !defined(MS_WINDOWS)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	16 extern int _Py_normalize_encoding(const char , char , size_t);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	17
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	18 /* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale.
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	19 On these operating systems, nl_langinfo(CODESET) announces an alias of the
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	20 ASCII encoding, whereas mbstowcs() and wcstombs() functions use the
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	21 ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	22 locale.getpreferredencoding() codec. For example, if command line arguments
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	23 are decoded by mbstowcs() and encoded back by os.fsencode(), we get a
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	24 UnicodeEncodeError instead of retrieving the original byte string.
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	25
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	26 The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	27 nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	28 one byte in range 0x80-0xff can be decoded from the locale encoding. The
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	29 workaround is also enabled on error, for example if getting the locale
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	30 failed.
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	31
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	32 Values of locale_is_ascii:
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	33
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	34 1: the workaround is used: _Py_wchar2char() uses
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	35 encode_ascii_surrogateescape() and _Py_char2wchar() uses
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	36 decode_ascii_surrogateescape()
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	37 0: the workaround is not used: _Py_wchar2char() uses wcstombs() and
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	38 _Py_char2wchar() uses mbstowcs()
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	39 -1: unknown, need to call check_force_ascii() to get the value
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	40 */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	41 static int force_ascii = -1;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	42
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	43 static int
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	44 check_force_ascii(void)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	45 {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	46 char *loc;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	47 #if defined(HAVE_LANGINFO_H) && defined(CODESET)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	48 char codeset, *alias;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	49 char encoding[100];
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	50 int is_ascii;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	51 unsigned int i;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	52 char* ascii_aliases[] = {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	53 "ascii",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	54 "646",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	55 "ansi-x3.4-1968",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	56 "ansi-x3-4-1968",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	57 "ansi-x3.4-1986",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	58 "cp367",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	59 "csascii",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	60 "ibm367",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	61 "iso646-us",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	62 "iso-646.irv-1991",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	63 "iso-ir-6",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	64 "us",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	65 "us-ascii",
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	66 NULL
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	67 };
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	68 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	69
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	70 loc = setlocale(LC_CTYPE, NULL);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	71 if (loc == NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	72 goto error;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	73 if (strcmp(loc, "C") != 0) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	74 /* the LC_CTYPE locale is different than C */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	75 return 0;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	76 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	77
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	78 #if defined(HAVE_LANGINFO_H) && defined(CODESET)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	79 codeset = nl_langinfo(CODESET);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	80 if (!codeset \|\| codeset[0] == '\0') {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	81 /* CODESET is not set or empty */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	82 goto error;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	83 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	84 if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding)))
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	85 goto error;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	86
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	87 is_ascii = 0;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	88 for (alias=ascii_aliases; *alias != NULL; alias++) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	89 if (strcmp(encoding, *alias) == 0) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	90 is_ascii = 1;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	91 break;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	92 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	93 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	94 if (!is_ascii) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	95 /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	96 return 0;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	97 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	98
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	99 for (i=0x80; i<0xff; i++) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	100 unsigned char ch;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	101 wchar_t wch;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	102 size_t res;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	103
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	104 ch = (unsigned char)i;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	105 res = mbstowcs(&wch, (char*)&ch, 1);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	106 if (res != (size_t)-1) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	107 /* decoding a non-ASCII character from the locale encoding succeed:
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	108 the locale encoding is not ASCII, force ASCII */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	109 return 1;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	110 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	111 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	112 /* None of the bytes in the range 0x80-0xff can be decoded from the locale
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	113 encoding: the locale encoding is really ASCII */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	114 return 0;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	115 #else
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	116 /* nl_langinfo(CODESET) is not available: always force ASCII */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	117 return 1;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	118 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	119
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	120 error:
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	121 /* if an error occured, force the ASCII encoding */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	122 return 1;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	123 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	124
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	125 static char*
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	126 encode_ascii_surrogateescape(const wchar_t text, size_t error_pos)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	127 {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	128 char result = NULL, out;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	129 size_t len, i;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	130 wchar_t ch;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	131
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	132 if (error_pos != NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	133 *error_pos = (size_t)-1;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	134
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	135 len = wcslen(text);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	136
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	137 result = PyMem_Malloc(len + 1); /* +1 for NUL byte */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	138 if (result == NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	139 return NULL;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	140
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	141 out = result;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	142 for (i=0; i<len; i++) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	143 ch = text[i];
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	144
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	145 if (ch <= 0x7f) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	146 /* ASCII character */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	147 *out++ = (char)ch;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	148 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	149 else if (0xdc80 <= ch && ch <= 0xdcff) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	150 /* UTF-8b surrogate */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	151 *out++ = (char)(ch - 0xdc00);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	152 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	153 else {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	154 if (error_pos != NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	155 *error_pos = i;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	156 PyMem_Free(result);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	157 return NULL;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	158 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	159 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	160 *out = '\0';
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	161 return result;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	162 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	163 #endif /* !defined(__APPLE__) && !defined(MS_WINDOWS) */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	164
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	165 #if !defined(__APPLE__) && (!defined(MS_WINDOWS) \|\| !defined(HAVE_MBRTOWC))
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	166 static wchar_t*
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	167 decode_ascii_surrogateescape(const char arg, size_t size)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	168 {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	169 wchar_t *res;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	170 unsigned char *in;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	171 wchar_t *out;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	172
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	173 res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	174 if (!res)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	175 return NULL;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	176
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	177 in = (unsigned char*)arg;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	178 out = res;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	179 while(*in)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	180 if(*in < 128)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	181 out++ = in++;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	182 else
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	183 out++ = 0xdc00 + in++;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	184 *out = 0;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	185 if (size != NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	186 *size = out - res;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	187 return res;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	188 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	189 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	190
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	191
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	192 /* Decode a byte string from the locale encoding with the
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	193 surrogateescape error handler (undecodable bytes are decoded as characters
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	194 in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	195 character, escape the bytes using the surrogateescape error handler instead
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	196 of decoding them.
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	197
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	198 Use _Py_wchar2char() to encode the character string back to a byte string.
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	199
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	200 Return a pointer to a newly allocated wide character string (use
3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	201 PyMem_Free() to free the memory) and write the number of written wide
3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	202 characters excluding the null character into *size if size is not NULL, or
66094 76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	203 NULL on error (conversion or memory allocation error).
76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	204
76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	205 Conversion errors should never happen, unless there is a bug in the C
76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	206 library. */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	207 wchar_t*
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	208 _Py_char2wchar(const char* arg, size_t *size)
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	209 {
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	210 #ifdef __APPLE__
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	211 wchar_t *wstr;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	212 wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg));
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	213 if (size != NULL) {
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	214 if (wstr != NULL)
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	215 *size = wcslen(wstr);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	216 else
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	217 *size = (size_t)-1;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	218 }
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	219 return wstr;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	220 #else
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	221 wchar_t *res;
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	222 size_t argsize;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	223 size_t count;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	224 unsigned char *in;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	225 wchar_t *out;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	226 #ifdef HAVE_MBRTOWC
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	227 mbstate_t mbs;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	228 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	229
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	230 #ifndef MS_WINDOWS
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	231 if (force_ascii == -1)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	232 force_ascii = check_force_ascii();
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	233
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	234 if (force_ascii) {
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	235 /* force ASCII encoding to workaround mbstowcs() issue */
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	236 res = decode_ascii_surrogateescape(arg, size);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	237 if (res == NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	238 goto oom;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	239 return res;
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	240 }
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	241 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	242
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	243 #ifdef HAVE_BROKEN_MBSTOWCS
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	244 /* Some platforms have a broken implementation of
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	245 * mbstowcs which does not count the characters that
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	246 * would result from conversion. Use an upper bound.
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	247 */
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	248 argsize = strlen(arg);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	249 #else
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	250 argsize = mbstowcs(NULL, arg, 0);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	251 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	252 if (argsize != (size_t)-1) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	253 res = (wchar_t )PyMem_Malloc((argsize+1)sizeof(wchar_t));
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	254 if (!res)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	255 goto oom;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	256 count = mbstowcs(res, arg, argsize+1);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	257 if (count != (size_t)-1) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	258 wchar_t *tmp;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	259 /* Only use the result if it contains no
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	260 surrogate characters. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	261 for (tmp = res; *tmp != 0 &&
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	262 (tmp < 0xd800 \|\| tmp > 0xdfff); tmp++)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	263 ;
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	264 if (*tmp == 0) {
3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	265 if (size != NULL)
3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	266 *size = count;
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	267 return res;
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	268 }
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	269 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	270 PyMem_Free(res);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	271 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	272 /* Conversion failed. Fall back to escaping with surrogateescape. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	273 #ifdef HAVE_MBRTOWC
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	274 /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	275
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	276 /* Overallocate; as multi-byte characters are in the argument, the
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	277 actual output could use less memory. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	278 argsize = strlen(arg) + 1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	279 res = (wchar_t)PyMem_Malloc(argsizesizeof(wchar_t));
66094 76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	280 if (!res)
76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	281 goto oom;
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	282 in = (unsigned char*)arg;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	283 out = res;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	284 memset(&mbs, 0, sizeof mbs);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	285 while (argsize) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	286 size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	287 if (converted == 0)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	288 /* Reached end of string; null char stored. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	289 break;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	290 if (converted == (size_t)-2) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	291 /* Incomplete character. This should never happen,
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	292 since we provide everything that we have -
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	293 unless there is a bug in the C library, or I
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	294 misunderstood how mbrtowc works. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	295 fprintf(stderr, "unexpected mbrtowc result -2\n");
66094 76a1c5e88352 _Py_char2wchar() frees the memory on conversion error Victor Stinner <victor.stinner@haypocalc.com> parents: 66092 diff changeset	296 PyMem_Free(res);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	297 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	298 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	299 if (converted == (size_t)-1) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	300 /* Conversion error. Escape as UTF-8b, and start over
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	301 in the initial shift state. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	302 out++ = 0xdc00 + in++;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	303 argsize--;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	304 memset(&mbs, 0, sizeof mbs);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	305 continue;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	306 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	307 if (out >= 0xd800 && out <= 0xdfff) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	308 /* Surrogate character. Escape the original
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	309 byte sequence with surrogateescape. */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	310 argsize -= converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	311 while (converted--)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	312 out++ = 0xdc00 + in++;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	313 continue;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	314 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	315 /* successfully converted some bytes */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	316 in += converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	317 argsize -= converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	318 out++;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	319 }
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	320 if (size != NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	321 *size = out - res;
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	322 #else /* HAVE_MBRTOWC */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	323 /* Cannot use C locale for escaping; manually escape as if charset
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	324 is ASCII (i.e. escape all bytes > 128. This will still roundtrip
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	325 correctly in the locale's charset, which must be an ASCII superset. */
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	326 res = decode_ascii_surrogateescape(arg, size);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	327 if (res == NULL)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	328 goto oom;
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	329 #endif /* HAVE_MBRTOWC */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	330 return res;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	331 oom:
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	332 fprintf(stderr, "out of memory\n");
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	333 return NULL;
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	334 #endif /* __APPLE__ */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	335 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	336
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	337 /* Encode a (wide) character string to the locale encoding with the
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	338 surrogateescape error handler (characters in range U+DC80..U+DCFF are
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	339 converted to bytes 0x80..0xFF).
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	340
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	341 This function is the reverse of _Py_char2wchar().
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	342
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	343 Return a pointer to a newly allocated byte string (use PyMem_Free() to free
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	344 the memory), or NULL on conversion or memory allocation error.
1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	345
1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	346 If error_pos is not NULL: *error_pos is the index of the invalid character
1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	347 on conversion error, or (size_t)-1 otherwise. */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	348 char*
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	349 _Py_wchar2char(const wchar_t text, size_t error_pos)
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	350 {
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	351 #ifdef __APPLE__
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	352 Py_ssize_t len;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	353 PyObject unicode, bytes = NULL;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	354 char *cpath;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	355
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	356 unicode = PyUnicode_FromWideChar(text, wcslen(text));
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	357 if (unicode == NULL)
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	358 return NULL;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	359
80704 af6fd3ca6de9 Issue #16416: Fix compilation error Victor Stinner <victor.stinner@gmail.com> parents: 80702 diff changeset	360 bytes = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
af6fd3ca6de9 Issue #16416: Fix compilation error Victor Stinner <victor.stinner@gmail.com> parents: 80702 diff changeset	361 PyUnicode_GET_SIZE(unicode),
af6fd3ca6de9 Issue #16416: Fix compilation error Victor Stinner <victor.stinner@gmail.com> parents: 80702 diff changeset	362 "surrogateescape");
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	363 Py_DECREF(unicode);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	364 if (bytes == NULL) {
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	365 PyErr_Clear();
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	366 if (error_pos != NULL)
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	367 *error_pos = (size_t)-1;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	368 return NULL;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	369 }
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	370
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	371 len = PyBytes_GET_SIZE(bytes);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	372 cpath = PyMem_Malloc(len+1);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	373 if (cpath == NULL) {
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	374 PyErr_Clear();
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	375 Py_DECREF(bytes);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	376 if (error_pos != NULL)
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	377 *error_pos = (size_t)-1;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	378 return NULL;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	379 }
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	380 memcpy(cpath, PyBytes_AsString(bytes), len + 1);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	381 Py_DECREF(bytes);
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	382 return cpath;
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	383 #else /* __APPLE__ */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	384 const size_t len = wcslen(text);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	385 char result = NULL, bytes = NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	386 size_t i, size, converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	387 wchar_t c, buf[2];
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	388
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	389 #ifndef MS_WINDOWS
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	390 if (force_ascii == -1)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	391 force_ascii = check_force_ascii();
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	392
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	393 if (force_ascii)
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	394 return encode_ascii_surrogateescape(text, error_pos);
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	395 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	396
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	397 /* The function works in two steps:
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	398 1. compute the length of the output buffer in bytes (size)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	399 2. outputs the bytes */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	400 size = 0;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	401 buf[1] = 0;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	402 while (1) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	403 for (i=0; i < len; i++) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	404 c = text[i];
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	405 if (c >= 0xdc80 && c <= 0xdcff) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	406 /* UTF-8b surrogate */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	407 if (bytes != NULL) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	408 *bytes++ = c - 0xdc00;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	409 size--;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	410 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	411 else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	412 size++;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	413 continue;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	414 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	415 else {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	416 buf[0] = c;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	417 if (bytes != NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	418 converted = wcstombs(bytes, buf, size);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	419 else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	420 converted = wcstombs(NULL, buf, 0);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	421 if (converted == (size_t)-1) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	422 if (result != NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	423 PyMem_Free(result);
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	424 if (error_pos != NULL)
1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	425 *error_pos = i;
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	426 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	427 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	428 if (bytes != NULL) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	429 bytes += converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	430 size -= converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	431 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	432 else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	433 size += converted;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	434 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	435 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	436 if (result != NULL) {
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	437 *bytes = '\0';
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	438 break;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	439 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	440
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	441 size += 1; /* nul byte at the end */
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	442 result = PyMem_Malloc(size);
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	443 if (result == NULL) {
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	444 if (error_pos != NULL)
c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	445 *error_pos = (size_t)-1;
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	446 return NULL;
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	447 }
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	448 bytes = result;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	449 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	450 return result;
80702 c838c9b117f1 Issue #16416: On Mac OS X, operating system data are now always Victor Stinner <victor.stinner@gmail.com> parents: 66094 diff changeset	451 #endif /* __APPLE__ */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	452 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	453
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	454 /* In principle, this should use HAVE__WSTAT, and _wstat
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	455 should be detected by autoconf. However, no current
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	456 POSIX system provides that function, so testing for
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	457 it is pointless.
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	458 Not sure whether the MS_WINDOWS guards are necessary:
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	459 perhaps for cygwin/mingw builds?
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	460 */
65164 1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	461 #if defined(HAVE_STAT) && !defined(MS_WINDOWS)
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	462
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	463 /* Get file status. Encode the path to the locale encoding. */
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	464
65164 1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	465 int
1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	466 _Py_wstat(const wchar_t* path, struct stat *buf)
1f3919af8a53 Fix fileutils for Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65163 diff changeset	467 {
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	468 int err;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	469 char *fname;
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	470 fname = _Py_wchar2char(path, NULL);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	471 if (fname == NULL) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	472 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	473 return -1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	474 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	475 err = stat(fname, buf);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	476 PyMem_Free(fname);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	477 return err;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	478 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	479 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	480
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	481 #ifdef HAVE_STAT
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	482
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	483 /* Call _wstat() on Windows, or encode the path to the filesystem encoding and
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	484 call stat() otherwise. Only fill st_mode attribute on Windows.
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	485
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	486 Return 0 on success, -1 on _wstat() / stat() error or (if PyErr_Occurred())
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	487 unicode error. */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	488
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	489 int
65165 2147cc152227 _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65164 diff changeset	490 _Py_stat(PyObject path, struct stat statbuf)
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	491 {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	492 #ifdef MS_WINDOWS
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	493 int err;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	494 struct _stat wstatbuf;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	495
65165 2147cc152227 _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65164 diff changeset	496 err = _wstat(PyUnicode_AS_UNICODE(path), &wstatbuf);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	497 if (!err)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	498 statbuf->st_mode = wstatbuf.st_mode;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	499 return err;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	500 #else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	501 int ret;
65165 2147cc152227 _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65164 diff changeset	502 PyObject *bytes = PyUnicode_EncodeFSDefault(path);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	503 if (bytes == NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	504 return -1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	505 ret = stat(PyBytes_AS_STRING(bytes), statbuf);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	506 Py_DECREF(bytes);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	507 return ret;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	508 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	509 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	510
81257 c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	511 #endif
c256764e2b3f Issue #16455: On FreeBSD and Solaris, if the locale is C, the Victor Stinner <victor.stinner@gmail.com> parents: 80704 diff changeset	512
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	513 /* Open a file. Use _wfopen() on Windows, encode the path to the locale
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	514 encoding and use fopen() otherwise. */
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	515
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	516 FILE *
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	517 _Py_wfopen(const wchar_t path, const wchar_t mode)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	518 {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	519 #ifndef MS_WINDOWS
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	520 FILE *f;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	521 char *cpath;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	522 char cmode[10];
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	523 size_t r;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	524 r = wcstombs(cmode, mode, 10);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	525 if (r == (size_t)-1 \|\| r >= 10) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	526 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	527 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	528 }
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	529 cpath = _Py_wchar2char(path, NULL);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	530 if (cpath == NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	531 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	532 f = fopen(cpath, cmode);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	533 PyMem_Free(cpath);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	534 return f;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	535 #else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	536 return _wfopen(path, mode);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	537 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	538 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	539
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	540 /* Call _wfopen() on Windows, or encode the path to the filesystem encoding and
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	541 call fopen() otherwise.
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	542
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	543 Return the new file object on success, or NULL if the file cannot be open or
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	544 (if PyErr_Occurred()) on unicode error */
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	545
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	546 FILE*
65165 2147cc152227 _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65164 diff changeset	547 _Py_fopen(PyObject path, const char mode)
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	548 {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	549 #ifdef MS_WINDOWS
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	550 wchar_t wmode[10];
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	551 int usize;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	552
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	553 usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode));
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	554 if (usize == 0)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	555 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	556
65165 2147cc152227 _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65164 diff changeset	557 return _wfopen(PyUnicode_AS_UNICODE(path), wmode);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	558 #else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	559 FILE *f;
65165 2147cc152227 _Py_stat() and _Py_fopen(): avoid PyUnicode_AsWideCharString() on Windows Victor Stinner <victor.stinner@haypocalc.com> parents: 65164 diff changeset	560 PyObject *bytes = PyUnicode_EncodeFSDefault(path);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	561 if (bytes == NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	562 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	563 f = fopen(PyBytes_AS_STRING(bytes), mode);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	564 Py_DECREF(bytes);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	565 return f;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	566 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	567 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	568
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	569 #ifdef HAVE_READLINK
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	570
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	571 /* Read value of symbolic link. Encode the path to the locale encoding, decode
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	572 the result from the locale encoding. */
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	573
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	574 int
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	575 _Py_wreadlink(const wchar_t path, wchar_t buf, size_t bufsiz)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	576 {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	577 char *cpath;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	578 char cbuf[PATH_MAX];
65409 e1dc9861b43f _Py_wreadlink() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65346 diff changeset	579 wchar_t *wbuf;
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	580 int res;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	581 size_t r1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	582
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	583 cpath = _Py_wchar2char(path, NULL);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	584 if (cpath == NULL) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	585 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	586 return -1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	587 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	588 res = (int)readlink(cpath, cbuf, PATH_MAX);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	589 PyMem_Free(cpath);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	590 if (res == -1)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	591 return -1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	592 if (res == PATH_MAX) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	593 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	594 return -1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	595 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	596 cbuf[res] = '\0'; /* buf will be null terminated */
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	597 wbuf = _Py_char2wchar(cbuf, &r1);
65410 6dcba93d6262 _Py_wreadlink(): catch _Py_char2wchar() failure Victor Stinner <victor.stinner@haypocalc.com> parents: 65409 diff changeset	598 if (wbuf == NULL) {
6dcba93d6262 _Py_wreadlink(): catch _Py_char2wchar() failure Victor Stinner <victor.stinner@haypocalc.com> parents: 65409 diff changeset	599 errno = EINVAL;
6dcba93d6262 _Py_wreadlink(): catch _Py_char2wchar() failure Victor Stinner <victor.stinner@haypocalc.com> parents: 65409 diff changeset	600 return -1;
6dcba93d6262 _Py_wreadlink(): catch _Py_char2wchar() failure Victor Stinner <victor.stinner@haypocalc.com> parents: 65409 diff changeset	601 }
65409 e1dc9861b43f _Py_wreadlink() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65346 diff changeset	602 if (bufsiz <= r1) {
e1dc9861b43f _Py_wreadlink() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65346 diff changeset	603 PyMem_Free(wbuf);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	604 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	605 return -1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	606 }
65409 e1dc9861b43f _Py_wreadlink() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65346 diff changeset	607 wcsncpy(buf, wbuf, bufsiz);
e1dc9861b43f _Py_wreadlink() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65346 diff changeset	608 PyMem_Free(wbuf);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	609 return (int)r1;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	610 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	611 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	612
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	613 #ifdef HAVE_REALPATH
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	614
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	615 /* Return the canonicalized absolute pathname. Encode path to the locale
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	616 encoding, decode the result from the locale encoding. */
1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	617
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	618 wchar_t*
65166 8fd5d06462e6 _Py_wrealpath() requires the size of the output buffer Victor Stinner <victor.stinner@haypocalc.com> parents: 65165 diff changeset	619 _Py_wrealpath(const wchar_t *path,
8fd5d06462e6 _Py_wrealpath() requires the size of the output buffer Victor Stinner <victor.stinner@haypocalc.com> parents: 65165 diff changeset	620 wchar_t *resolved_path, size_t resolved_path_size)
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	621 {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	622 char *cpath;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	623 char cresolved_path[PATH_MAX];
65411 df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	624 wchar_t *wresolved_path;
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	625 char *res;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	626 size_t r;
66092 1b5f47e2e0f0 PyUnicode_EncodeFS() raises an exception if _Py_wchar2char() fails Victor Stinner <victor.stinner@haypocalc.com> parents: 65412 diff changeset	627 cpath = _Py_wchar2char(path, NULL);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	628 if (cpath == NULL) {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	629 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	630 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	631 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	632 res = realpath(cpath, cresolved_path);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	633 PyMem_Free(cpath);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	634 if (res == NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	635 return NULL;
65411 df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	636
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	637 wresolved_path = _Py_char2wchar(cresolved_path, &r);
65411 df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	638 if (wresolved_path == NULL) {
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	639 errno = EINVAL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	640 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	641 }
65411 df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	642 if (resolved_path_size <= r) {
df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	643 PyMem_Free(wresolved_path);
df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	644 errno = EINVAL;
df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	645 return NULL;
df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	646 }
df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	647 wcsncpy(resolved_path, wresolved_path, resolved_path_size);
df0e348e7176 _Py_wrealpath() uses _Py_char2wchar() to decode the result, to support Victor Stinner <victor.stinner@haypocalc.com> parents: 65410 diff changeset	648 PyMem_Free(wresolved_path);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	649 return resolved_path;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	650 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	651 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	652
65312 a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	653 /* Get the current directory. size is the buffer size in wide characters
a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	654 including the null character. Decode the path from the locale encoding. */
65168 1c4774b4e98a fileutils.c: document which encodings are used Victor Stinner <victor.stinner@haypocalc.com> parents: 65166 diff changeset	655
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	656 wchar_t*
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	657 _Py_wgetcwd(wchar_t *buf, size_t size)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	658 {
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	659 #ifdef MS_WINDOWS
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	660 return _wgetcwd(buf, size);
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	661 #else
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	662 char fname[PATH_MAX];
65312 a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	663 wchar_t *wname;
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	664 size_t len;
65312 a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	665
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	666 if (getcwd(fname, PATH_MAX) == NULL)
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	667 return NULL;
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	668 wname = _Py_char2wchar(fname, &len);
65312 a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	669 if (wname == NULL)
a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	670 return NULL;
65412 3794c9b50824 Add an optional size argument to _Py_char2wchar() Victor Stinner <victor.stinner@haypocalc.com> parents: 65411 diff changeset	671 if (size <= len) {
65312 a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	672 PyMem_Free(wname);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	673 return NULL;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	674 }
65312 a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	675 wcsncpy(buf, wname, size);
a27fc0ff3678 _Py_wgetcwd() decodes the path using _Py_char2wchar() to support surrogates Victor Stinner <victor.stinner@haypocalc.com> parents: 65168 diff changeset	676 PyMem_Free(wname);
65162 39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	677 return buf;
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	678 #endif
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	679 }
39014f0abc32 Create fileutils.c/.h Victor Stinner <victor.stinner@haypocalc.com> parents: diff changeset	680

Mercurial > cpython-lru_cache-weakref

annotate Python/fileutils.c @ 86913:8c9769b17171 3.2