import os , pickle
from multiprocessing import Pool, Lock, synchronize, get_context, Process
import multiprocessing.queues
import sys
_is_windows= sys .platform == 'win32'
if _is_windows:
import _winapi
def work( q, reply_q) :
print ( "Worker: Main says" , q.get ( ) )
reply_q.put ( 'haha' )
class DupSemLockHandle( object ) :
"""
Picklable wrapper for a handle. Attempts to mirror how PipeConnection objects are pickled using appropriate api
"""
def __init__ ( self , handle, pid= None ) :
if pid is None :
# We just duplicate the handle in the current process and
# let the receiving process steal the handle.
pid = os .getpid ( )
if _is_windows:
proc = _winapi.OpenProcess ( _winapi.PROCESS_DUP_HANDLE , False , pid)
try :
self ._handle = _winapi.DuplicateHandle (
_winapi.GetCurrentProcess ( ) ,
handle, proc, 0 , False , _winapi.DUPLICATE_SAME_ACCESS )
finally :
_winapi.CloseHandle ( proc)
else :
self ._handle = handle
self ._pid = pid
def detach( self ) :
"""
Get the handle, typically from another process
"""
# retrieve handle from process which currently owns it
if self ._pid == os .getpid ( ) :
# The handle has already been duplicated for this process.
return self ._handle
if not _is_windows:
return self ._handle
# We must steal the handle from the process whose pid is self._pid.
proc = _winapi.OpenProcess ( _winapi.PROCESS_DUP_HANDLE , False ,
self ._pid)
try :
return _winapi.DuplicateHandle (
proc, self ._handle, _winapi.GetCurrentProcess ( ) ,
0 , False , _winapi.DUPLICATE_CLOSE_SOURCE | _winapi.DUPLICATE_SAME_ACCESS )
finally :
_winapi.CloseHandle ( proc)
def reduce_lock_connection( self ) :
sl = self ._semlock
dh = DupSemLockHandle( sl.handle )
return rebuild_lock_connection, ( dh, type ( self ) , ( sl.kind , sl.maxvalue , sl.name ) )
def rebuild_lock_connection( dh, t, state) :
handle = dh.detach ( ) # Duplicated handle valid in current process's context
# Create a new instance without calling __init__ because we'll supply the state ourselves
lck = t.__new__ ( t)
lck.__setstate__( ( handle, ) +state)
return lck
# Add our own reduce function to pickle SemLock and it's child classes
synchronize.SemLock .__reduce__ = reduce_lock_connection
class PicklableQueue( multiprocessing.queues .Queue ) :
"""
A picklable Queue that skips the call to context.assert_spawning because it's no longer needed
"""
def __init__ ( self , *args, **kwargs) :
ctx = get_context( )
super ( ) .__init__ ( *args, **kwargs, ctx= ctx)
def __getstate__( self ) :
return ( self ._ignore_epipe, self ._maxsize, self ._reader, self ._writer,
self ._rlock, self ._wlock, self ._sem, self ._opid)
def is_locked( l) :
"""
Returns whether the given lock is acquired or not.
"""
locked = l.acquire ( block= False )
if locked is False :
return True
else :
l.release ( )
return False
if __name__ == '__main__' :
from multiprocessing import set_start_method
set_start_method( 'spawn' )
print ( f'The platform is {sys.platform}.' )
# Example that shows that you can now pickle/unpickle locks and they'll still point towards the same object
l1 = Lock( )
p = pickle .dumps ( l1)
l2 = pickle .loads ( p)
print ( 'before acquiring, l1 locked:' , is_locked( l1) , 'l2 locked' , is_locked( l2) )
l2.acquire ( )
print ( 'after acquiring l1 locked:' , is_locked( l1) , 'l2 locked' , is_locked( l2) )
# Putting a queue to a queue:
q1 = PicklableQueue( )
q1.put ( 'Putting a queue to a queue!' )
q2 = PicklableQueue( )
q2.put ( q1)
print ( q2.get ( ) .get ( ) )
# This should still work:
q = PicklableQueue( )
reply_q = PicklableQueue( )
q.put ( 'It still works!' )
p = Process( target= work, args= ( q, reply_q) )
p.start ( )
print ( reply_q.get ( ) )
p.join ( )
# Example that shows how you can pass a queue to Pool and it will work:
pool = Pool( 8 )
for _ in range ( 8 ) :
q.put ( 'laugh' )
for _ in range ( 8 ) :
pool.apply_async ( work, ( q, reply_q) )
for _ in range ( 8 ) :
print ( reply_q.get ( ) )
pool.close ( )
pool.join ( )
print ( 'All pool-submitted tasks are complete!' )
aW1wb3J0IG9zLCBwaWNrbGUKZnJvbSBtdWx0aXByb2Nlc3NpbmcgaW1wb3J0IFBvb2wsIExvY2ssIHN5bmNocm9uaXplLCBnZXRfY29udGV4dCwgUHJvY2VzcwppbXBvcnQgbXVsdGlwcm9jZXNzaW5nLnF1ZXVlcwppbXBvcnQgc3lzCl9pc193aW5kb3dzPSBzeXMucGxhdGZvcm0gPT0gJ3dpbjMyJwppZiBfaXNfd2luZG93czoKICAgIGltcG9ydCBfd2luYXBpCgpkZWYgd29yayhxLCByZXBseV9xKToKICAgIHByaW50KCJXb3JrZXI6IE1haW4gc2F5cyIsIHEuZ2V0KCkpCiAgICByZXBseV9xLnB1dCgnaGFoYScpCgoKY2xhc3MgRHVwU2VtTG9ja0hhbmRsZShvYmplY3QpOgogICAgIiIiCiAgICBQaWNrbGFibGUgd3JhcHBlciBmb3IgYSBoYW5kbGUuIEF0dGVtcHRzIHRvIG1pcnJvciBob3cgUGlwZUNvbm5lY3Rpb24gb2JqZWN0cyBhcmUgcGlja2xlZCB1c2luZyBhcHByb3ByaWF0ZSBhcGkKICAgICIiIgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBoYW5kbGUsIHBpZD1Ob25lKToKICAgICAgICBpZiBwaWQgaXMgTm9uZToKICAgICAgICAgICAgIyBXZSBqdXN0IGR1cGxpY2F0ZSB0aGUgaGFuZGxlIGluIHRoZSBjdXJyZW50IHByb2Nlc3MgYW5kCiAgICAgICAgICAgICMgbGV0IHRoZSByZWNlaXZpbmcgcHJvY2VzcyBzdGVhbCB0aGUgaGFuZGxlLgogICAgICAgICAgICBwaWQgPSBvcy5nZXRwaWQoKQogICAgICAgIGlmIF9pc193aW5kb3dzOgogICAgICAgICAgICBwcm9jID0gX3dpbmFwaS5PcGVuUHJvY2Vzcyhfd2luYXBpLlBST0NFU1NfRFVQX0hBTkRMRSwgRmFsc2UsIHBpZCkKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgc2VsZi5faGFuZGxlID0gX3dpbmFwaS5EdXBsaWNhdGVIYW5kbGUoCiAgICAgICAgICAgICAgICAgICAgX3dpbmFwaS5HZXRDdXJyZW50UHJvY2VzcygpLAogICAgICAgICAgICAgICAgICAgIGhhbmRsZSwgcHJvYywgMCwgRmFsc2UsIF93aW5hcGkuRFVQTElDQVRFX1NBTUVfQUNDRVNTKQogICAgICAgICAgICBmaW5hbGx5OgogICAgICAgICAgICAgICAgX3dpbmFwaS5DbG9zZUhhbmRsZShwcm9jKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHNlbGYuX2hhbmRsZSA9IGhhbmRsZQogICAgICAgIHNlbGYuX3BpZCA9IHBpZAoKICAgIGRlZiBkZXRhY2goc2VsZik6CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSBoYW5kbGUsIHR5cGljYWxseSBmcm9tIGFub3RoZXIgcHJvY2VzcwogICAgICAgICIiIgogICAgICAgICMgcmV0cmlldmUgaGFuZGxlIGZyb20gcHJvY2VzcyB3aGljaCBjdXJyZW50bHkgb3ducyBpdAogICAgICAgIGlmIHNlbGYuX3BpZCA9PSBvcy5nZXRwaWQoKToKICAgICAgICAgICAgIyBUaGUgaGFuZGxlIGhhcyBhbHJlYWR5IGJlZW4gZHVwbGljYXRlZCBmb3IgdGhpcyBwcm9jZXNzLgogICAgICAgICAgICByZXR1cm4gc2VsZi5faGFuZGxlCgogICAgICAgIGlmIG5vdCBfaXNfd2luZG93czoKICAgICAgICAgICAgcmV0dXJuIHNlbGYuX2hhbmRsZQoKICAgICAgICAjIFdlIG11c3Qgc3RlYWwgdGhlIGhhbmRsZSBmcm9tIHRoZSBwcm9jZXNzIHdob3NlIHBpZCBpcyBzZWxmLl9waWQuCiAgICAgICAgcHJvYyA9IF93aW5hcGkuT3BlblByb2Nlc3MoX3dpbmFwaS5QUk9DRVNTX0RVUF9IQU5ETEUsIEZhbHNlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNlbGYuX3BpZCkKICAgICAgICB0cnk6CiAgICAgICAgICAgIHJldHVybiBfd2luYXBpLkR1cGxpY2F0ZUhhbmRsZSgKICAgICAgICAgICAgICAgIHByb2MsIHNlbGYuX2hhbmRsZSwgX3dpbmFwaS5HZXRDdXJyZW50UHJvY2VzcygpLAogICAgICAgICAgICAgICAgMCwgRmFsc2UsIF93aW5hcGkuRFVQTElDQVRFX0NMT1NFX1NPVVJDRSB8IF93aW5hcGkuRFVQTElDQVRFX1NBTUVfQUNDRVNTKQogICAgICAgIGZpbmFsbHk6CiAgICAgICAgICAgIF93aW5hcGkuQ2xvc2VIYW5kbGUocHJvYykKCgpkZWYgcmVkdWNlX2xvY2tfY29ubmVjdGlvbihzZWxmKToKICAgIHNsID0gc2VsZi5fc2VtbG9jawogICAgZGggPSBEdXBTZW1Mb2NrSGFuZGxlKHNsLmhhbmRsZSkKICAgIHJldHVybiByZWJ1aWxkX2xvY2tfY29ubmVjdGlvbiwgKGRoLCB0eXBlKHNlbGYpLCAoc2wua2luZCwgc2wubWF4dmFsdWUsIHNsLm5hbWUpKQoKCmRlZiByZWJ1aWxkX2xvY2tfY29ubmVjdGlvbihkaCwgdCwgc3RhdGUpOgogICAgaGFuZGxlID0gZGguZGV0YWNoKCkgICMgRHVwbGljYXRlZCBoYW5kbGUgdmFsaWQgaW4gY3VycmVudCBwcm9jZXNzJ3MgY29udGV4dAoKICAgICMgQ3JlYXRlIGEgbmV3IGluc3RhbmNlIHdpdGhvdXQgY2FsbGluZyBfX2luaXRfXyBiZWNhdXNlIHdlJ2xsIHN1cHBseSB0aGUgc3RhdGUgb3Vyc2VsdmVzCiAgICBsY2sgPSB0Ll9fbmV3X18odCkKICAgIGxjay5fX3NldHN0YXRlX18oKGhhbmRsZSwpK3N0YXRlKQogICAgcmV0dXJuIGxjawoKCiMgQWRkIG91ciBvd24gcmVkdWNlIGZ1bmN0aW9uIHRvIHBpY2tsZSBTZW1Mb2NrIGFuZCBpdCdzIGNoaWxkIGNsYXNzZXMKc3luY2hyb25pemUuU2VtTG9jay5fX3JlZHVjZV9fID0gcmVkdWNlX2xvY2tfY29ubmVjdGlvbgoKCmNsYXNzIFBpY2tsYWJsZVF1ZXVlKG11bHRpcHJvY2Vzc2luZy5xdWV1ZXMuUXVldWUpOgogICAgIiIiCiAgICBBIHBpY2tsYWJsZSBRdWV1ZSB0aGF0IHNraXBzIHRoZSBjYWxsIHRvIGNvbnRleHQuYXNzZXJ0X3NwYXduaW5nIGJlY2F1c2UgaXQncyBubyBsb25nZXIgbmVlZGVkCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgKmFyZ3MsICoqa3dhcmdzKToKICAgICAgICBjdHggPSBnZXRfY29udGV4dCgpCiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygqYXJncywgKiprd2FyZ3MsIGN0eD1jdHgpCgogICAgZGVmIF9fZ2V0c3RhdGVfXyhzZWxmKToKCiAgICAgICAgcmV0dXJuIChzZWxmLl9pZ25vcmVfZXBpcGUsIHNlbGYuX21heHNpemUsIHNlbGYuX3JlYWRlciwgc2VsZi5fd3JpdGVyLAogICAgICAgICAgICAgICAgc2VsZi5fcmxvY2ssIHNlbGYuX3dsb2NrLCBzZWxmLl9zZW0sIHNlbGYuX29waWQpCgpkZWYgaXNfbG9ja2VkKGwpOgogICAgIiIiCiAgICBSZXR1cm5zIHdoZXRoZXIgdGhlIGdpdmVuIGxvY2sgaXMgYWNxdWlyZWQgb3Igbm90LgogICAgIiIiCiAgICBsb2NrZWQgPSBsLmFjcXVpcmUoYmxvY2s9RmFsc2UpCiAgICBpZiBsb2NrZWQgaXMgRmFsc2U6CiAgICAgICAgcmV0dXJuIFRydWUKICAgIGVsc2U6CiAgICAgICAgbC5yZWxlYXNlKCkKICAgICAgICByZXR1cm4gRmFsc2UKCgppZiBfX25hbWVfXyA9PSAnX19tYWluX18nOgogICAgZnJvbSBtdWx0aXByb2Nlc3NpbmcgaW1wb3J0IHNldF9zdGFydF9tZXRob2QKICAgIHNldF9zdGFydF9tZXRob2QoJ3NwYXduJykKCiAgICBwcmludChmJ1RoZSBwbGF0Zm9ybSBpcyB7c3lzLnBsYXRmb3JtfS4nKQoKICAgICMgRXhhbXBsZSB0aGF0IHNob3dzIHRoYXQgeW91IGNhbiBub3cgcGlja2xlL3VucGlja2xlIGxvY2tzIGFuZCB0aGV5J2xsIHN0aWxsIHBvaW50IHRvd2FyZHMgdGhlIHNhbWUgb2JqZWN0CiAgICBsMSA9IExvY2soKQogICAgcCA9IHBpY2tsZS5kdW1wcyhsMSkKICAgIGwyID0gcGlja2xlLmxvYWRzKHApCiAgICBwcmludCgnYmVmb3JlIGFjcXVpcmluZywgbDEgbG9ja2VkOicsIGlzX2xvY2tlZChsMSksICdsMiBsb2NrZWQnLCBpc19sb2NrZWQobDIpKQogICAgbDIuYWNxdWlyZSgpCiAgICBwcmludCgnYWZ0ZXIgYWNxdWlyaW5nIGwxIGxvY2tlZDonLCBpc19sb2NrZWQobDEpLCAnbDIgbG9ja2VkJywgaXNfbG9ja2VkKGwyKSkKCiAgICAjIFB1dHRpbmcgYSBxdWV1ZSB0byBhIHF1ZXVlOgogICAgcTEgPSBQaWNrbGFibGVRdWV1ZSgpCiAgICBxMS5wdXQoJ1B1dHRpbmcgYSBxdWV1ZSB0byBhIHF1ZXVlIScpCiAgICBxMiA9IFBpY2tsYWJsZVF1ZXVlKCkKICAgIHEyLnB1dChxMSkKICAgIHByaW50KHEyLmdldCgpLmdldCgpKQoKICAgICMgVGhpcyBzaG91bGQgc3RpbGwgd29yazoKICAgIHEgPSBQaWNrbGFibGVRdWV1ZSgpCiAgICByZXBseV9xID0gUGlja2xhYmxlUXVldWUoKQogICAgcS5wdXQoJ0l0IHN0aWxsIHdvcmtzIScpCiAgICBwID0gUHJvY2Vzcyh0YXJnZXQ9d29yaywgYXJncz0ocSwgcmVwbHlfcSkpCiAgICBwLnN0YXJ0KCkKICAgIHByaW50KHJlcGx5X3EuZ2V0KCkpCiAgICBwLmpvaW4oKQoKICAgICMgRXhhbXBsZSB0aGF0IHNob3dzIGhvdyB5b3UgY2FuIHBhc3MgYSBxdWV1ZSB0byBQb29sIGFuZCBpdCB3aWxsIHdvcms6CiAgICBwb29sID0gUG9vbCg4KQogICAgZm9yIF8gaW4gcmFuZ2UoOCk6CiAgICAgICAgcS5wdXQoJ2xhdWdoJykKICAgIGZvciBfIGluIHJhbmdlKDgpOgogICAgICAgIHBvb2wuYXBwbHlfYXN5bmMod29yaywgKHEsIHJlcGx5X3EpKQogICAgZm9yIF8gaW4gcmFuZ2UoOCk6CiAgICAgICAgcHJpbnQocmVwbHlfcS5nZXQoKSkKICAgIHBvb2wuY2xvc2UoKQogICAgcG9vbC5qb2luKCkKICAgIHByaW50KCdBbGwgcG9vbC1zdWJtaXR0ZWQgdGFza3MgYXJlIGNvbXBsZXRlIScp