Mercurial > cpython-lru_cache-weakref

annotate Lib/multiprocessing/reduction.py @ 78069:a8aa77458e12 3.2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add c_size_t and c_ssize_t to the ctypes summary table.
author Antoine Pitrou <solipsis@pitrou.net>
date Thu, 12 Jul 2012 20:31:50 +0200
parents b084b50e4aa6
children 1ac03e071d65
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
1 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
2 # Module to allow connection and socket objects to be transferred
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
3 # between processes
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
4 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
5 # multiprocessing/reduction.py
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
6 #
66885
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
7 # Copyright (c) 2006-2008, R Oudkerk
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
8 # All rights reserved.
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
9 #
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
10 # Redistribution and use in source and binary forms, with or without
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
11 # modification, are permitted provided that the following conditions
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
12 # are met:
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
13 #
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
14 # 1. Redistributions of source code must retain the above copyright
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
15 # notice, this list of conditions and the following disclaimer.
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
16 # 2. Redistributions in binary form must reproduce the above copyright
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
17 # notice, this list of conditions and the following disclaimer in the
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
18 # documentation and/or other materials provided with the distribution.
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
19 # 3. Neither the name of author nor the names of any contributors may be
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
20 # used to endorse or promote products derived from this software
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
21 # without specific prior written permission.
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
22 #
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
23 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
24 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
25 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
26 # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
27 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
28 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
29 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
30 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
31 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
32 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
b084b50e4aa6 9162: fix license in multiprocessing files
R. David Murray <rdmurray@bitdance.com>
parents: 63736
diff changeset
33 # SUCH DAMAGE.
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
34 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
35
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
36 __all__ = []
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
37
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
38 import os
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
39 import sys
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
40 import socket
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
41 import threading
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
42
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
43 import _multiprocessing
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
44 from multiprocessing import current_process
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
45 from multiprocessing.forking import Popen, duplicate, close, ForkingPickler
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
46 from multiprocessing.util import register_after_fork, debug, sub_debug
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
47 from multiprocessing.connection import Client, Listener
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
48
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
49
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
50 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
51 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
52 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
53
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
54 if not(sys.platform == 'win32' or hasattr(_multiprocessing, 'recvfd')):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
55 raise ImportError('pickling of connections not supported')
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
56
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
57 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
58 # Platform specific definitions
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
59 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
60
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
61 if sys.platform == 'win32':
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
62 import _subprocess
63736
fd7f73e12568 Fix #9513 to remove relative imports from multiprocessing.
Brian Curtin <brian.curtin@gmail.com>
parents: 48780
diff changeset
63 from _multiprocessing import win32
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
64
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
65 def send_handle(conn, handle, destination_pid):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
66 process_handle = win32.OpenProcess(
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
67 win32.PROCESS_ALL_ACCESS, False, destination_pid
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
68 )
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
69 try:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
70 new_handle = duplicate(handle, process_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
71 conn.send(new_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
72 finally:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
73 close(process_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
74
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
75 def recv_handle(conn):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
76 return conn.recv()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
77
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
78 else:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
79 def send_handle(conn, handle, destination_pid):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
80 _multiprocessing.sendfd(conn.fileno(), handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
81
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
82 def recv_handle(conn):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
83 return _multiprocessing.recvfd(conn.fileno())
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
84
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
85 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
86 # Support for a per-process server thread which caches pickled handles
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
87 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
88
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
89 _cache = set()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
90
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
91 def _reset(obj):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
92 global _lock, _listener, _cache
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
93 for h in _cache:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
94 close(h)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
95 _cache.clear()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
96 _lock = threading.Lock()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
97 _listener = None
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
98
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
99 _reset(None)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
100 register_after_fork(_reset, _reset)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
101
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
102 def _get_listener():
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
103 global _listener
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
104
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
105 if _listener is None:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
106 _lock.acquire()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
107 try:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
108 if _listener is None:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
109 debug('starting listener and thread for sending handles')
48773
25f06a703f89 Merged revisions 65864 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 47615
diff changeset
110 _listener = Listener(authkey=current_process().authkey)
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
111 t = threading.Thread(target=_serve)
48773
25f06a703f89 Merged revisions 65864 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 47615
diff changeset
112 t.daemon = True
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
113 t.start()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
114 finally:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
115 _lock.release()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
116
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
117 return _listener
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
118
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
119 def _serve():
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
120 from .util import is_exiting, sub_warning
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
121
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
122 while 1:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
123 try:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
124 conn = _listener.accept()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
125 handle_wanted, destination_pid = conn.recv()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
126 _cache.remove(handle_wanted)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
127 send_handle(conn, handle_wanted, destination_pid)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
128 close(handle_wanted)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
129 conn.close()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
130 except:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
131 if not is_exiting():
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
132 import traceback
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
133 sub_warning(
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
134 'thread for sharing handles raised exception :\n' +
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
135 '-'*79 + '\n' + traceback.format_exc() + '-'*79
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
136 )
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
137
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
138 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
139 # Functions to be used for pickling/unpickling objects with handles
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
140 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
141
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
142 def reduce_handle(handle):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
143 if Popen.thread_is_spawning():
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
144 return (None, Popen.duplicate_for_child(handle), True)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
145 dup_handle = duplicate(handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
146 _cache.add(dup_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
147 sub_debug('reducing handle %d', handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
148 return (_get_listener().address, dup_handle, False)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
149
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
150 def rebuild_handle(pickled_data):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
151 address, handle, inherited = pickled_data
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
152 if inherited:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
153 return handle
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
154 sub_debug('rebuilding handle %d', handle)
48773
25f06a703f89 Merged revisions 65864 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 47615
diff changeset
155 conn = Client(address, authkey=current_process().authkey)
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
156 conn.send((handle, os.getpid()))
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
157 new_handle = recv_handle(conn)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
158 conn.close()
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
159 return new_handle
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
160
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
161 #
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
162 # Register `_multiprocessing.Connection` with `ForkingPickler`
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
163 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
164
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
165 def reduce_connection(conn):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
166 rh = reduce_handle(conn.fileno())
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
167 return rebuild_connection, (rh, conn.readable, conn.writable)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
168
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
169 def rebuild_connection(reduced_handle, readable, writable):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
170 handle = rebuild_handle(reduced_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
171 return _multiprocessing.Connection(
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
172 handle, readable=readable, writable=writable
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
173 )
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
174
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
175 ForkingPickler.register(_multiprocessing.Connection, reduce_connection)
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
176
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
177 #
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
178 # Register `socket.socket` with `ForkingPickler`
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
179 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
180
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
181 def fromfd(fd, family, type_, proto=0):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
182 s = socket.fromfd(fd, family, type_, proto)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
183 if s.__class__ is not socket.socket:
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
184 s = socket.socket(_sock=s)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
185 return s
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
186
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
187 def reduce_socket(s):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
188 reduced_handle = reduce_handle(s.fileno())
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
189 return rebuild_socket, (reduced_handle, s.family, s.type, s.proto)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
190
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
191 def rebuild_socket(reduced_handle, family, type_, proto):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
192 fd = rebuild_handle(reduced_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
193 _sock = fromfd(fd, family, type_, proto)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
194 close(fd)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
195 return _sock
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
196
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
197 ForkingPickler.register(socket.socket, reduce_socket)
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
198
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
199 #
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
200 # Register `_multiprocessing.PipeConnection` with `ForkingPickler`
47595
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
201 #
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
202
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
203 if sys.platform == 'win32':
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
204
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
205 def reduce_pipe_connection(conn):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
206 rh = reduce_handle(conn.fileno())
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
207 return rebuild_pipe_connection, (rh, conn.readable, conn.writable)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
208
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
209 def rebuild_pipe_connection(reduced_handle, readable, writable):
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
210 handle = rebuild_handle(reduced_handle)
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
211 return _multiprocessing.PipeConnection(
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
212 handle, readable=readable, writable=writable
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
213 )
5c6cdfcd6eae Merged revisions 64104,64117 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents:
diff changeset
214
48780
99e783ec7781 Issue #3125: Remove copy_reg in multiprocessing and replace it with
Amaury Forgeot d'Arc <amauryfa@gmail.com>
parents: 48773
diff changeset
215 ForkingPickler.register(_multiprocessing.PipeConnection, reduce_pipe_connection)