asyncio
asyncio copied to clipboard
python
asyncio.create_subprocess_exec with IO redirection hangs if no global loop set
If I set global event loop to None (asyncio.set_event_loop(None)) and explicitly pass everywhere event loop, asyncio.create_subprocess_exec with I/O redirection will hang.
In following example proc.communicate() will never return:
import asyncio.subprocess
async def f(loop):
proc = await asyncio.create_subprocess_exec(
'echo',
stdout=asyncio.subprocess.PIPE,
loop=loop)
await proc.communicate()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(None)
date = loop.run_until_complete(f(loop))
loop.close()
With PYTHONASYNCIODEBUG=x:
Traceback (most recent call last):
File "asyncio_subprocess_hangs_wo_global_loop.py", line 17, in <module>
date = loop.run_until_complete(f(loop))
File "/usr/lib/python3.5/asyncio/base_events.py", line 387, in run_until_complete
return future.result()
File "/usr/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/usr/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "asyncio_subprocess_hangs_wo_global_loop.py", line 9, in f
loop=loop)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/subprocess.py", line 212, in create_subprocess_exec
stderr=stderr, **kwds)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/base_events.py", line 1079, in subprocess_exec
bufsize, **kwargs)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/unix_events.py", line 187, in _make_subprocess_transport
self._child_watcher_callback, transp)
File "/usr/lib/python3.5/asyncio/unix_events.py", line 808, in add_child_handler
self._do_waitpid(pid)
File "/usr/lib/python3.5/asyncio/unix_events.py", line 841, in _do_waitpid
if self._loop.get_debug():
AttributeError: 'NoneType' object has no attribute 'get_debug'
Task was destroyed but it is pending!
source_traceback: Object created at (most recent call last):
File "asyncio_subprocess_hangs_wo_global_loop.py", line 17, in <module>
date = loop.run_until_complete(f(loop))
File "/usr/lib/python3.5/asyncio/base_events.py", line 375, in run_until_complete
self.run_forever()
File "/usr/lib/python3.5/asyncio/base_events.py", line 345, in run_forever
self._run_once()
File "/usr/lib/python3.5/asyncio/base_events.py", line 1304, in _run_once
handle._run()
File "/usr/lib/python3.5/asyncio/events.py", line 125, in _run
self._callback(*self._args)
File "/usr/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "asyncio_subprocess_hangs_wo_global_loop.py", line 9, in f
loop=loop)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/subprocess.py", line 212, in create_subprocess_exec
stderr=stderr, **kwds)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/base_events.py", line 1079, in subprocess_exec
bufsize, **kwargs)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/unix_events.py", line 184, in _make_subprocess_transport
**kwargs)
File "/usr/lib/python3.5/asyncio/base_subprocess.py", line 56, in __init__
self._loop.create_task(self._connect_pipes(waiter))
task: <Task pending coro=<BaseSubprocessTransport._connect_pipes() running at /usr/lib/python3.5/asyncio/base_subprocess.py:171> wait_for=<Future pending cb=[Task._wakeup()] created at /usr/lib/python3.5/asyncio/base_events.py:252> created at /usr/lib/python3.5/asyncio/base_subprocess.py:56>
Future exception was never retrieved
future: <Future finished exception=RuntimeError('Event loop is closed',) created at /usr/lib/python3.5/asyncio/base_events.py:252>
source_traceback: Object created at (most recent call last):
File "asyncio_subprocess_hangs_wo_global_loop.py", line 17, in <module>
date = loop.run_until_complete(f(loop))
File "/usr/lib/python3.5/asyncio/base_events.py", line 375, in run_until_complete
self.run_forever()
File "/usr/lib/python3.5/asyncio/base_events.py", line 345, in run_forever
self._run_once()
File "/usr/lib/python3.5/asyncio/base_events.py", line 1304, in _run_once
handle._run()
File "/usr/lib/python3.5/asyncio/events.py", line 125, in _run
self._callback(*self._args)
File "/usr/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "asyncio_subprocess_hangs_wo_global_loop.py", line 9, in f
loop=loop)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/subprocess.py", line 212, in create_subprocess_exec
stderr=stderr, **kwds)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/base_events.py", line 1079, in subprocess_exec
bufsize, **kwargs)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 105, in __next__
return self.gen.send(None)
File "/usr/lib/python3.5/asyncio/unix_events.py", line 180, in _make_subprocess_transport
waiter = self.create_future()
File "/usr/lib/python3.5/asyncio/base_events.py", line 252, in create_future
return futures.Future(loop=self)
Traceback (most recent call last):
File "/usr/lib/python3.5/asyncio/base_events.py", line 989, in connect_read_pipe
yield from waiter
GeneratorExit
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.5/asyncio/base_subprocess.py", line 171, in _connect_pipes
proc.stdout)
File "/usr/lib/python3.5/asyncio/coroutines.py", line 127, in close
return self.gen.close()
File "/usr/lib/python3.5/asyncio/base_events.py", line 991, in connect_read_pipe
transport.close()
File "/usr/lib/python3.5/asyncio/unix_events.py", line 376, in close
self._close(None)
File "/usr/lib/python3.5/asyncio/unix_events.py", line 404, in _close
self._loop.call_soon(self._call_connection_lost, exc)
File "/usr/lib/python3.5/asyncio/base_events.py", line 497, in call_soon
handle = self._call_soon(callback, args)
File "/usr/lib/python3.5/asyncio/base_events.py", line 506, in _call_soon
self._check_closed()
File "/usr/lib/python3.5/asyncio/base_events.py", line 334, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
Tested on Ubuntu 16.04, with Python 3.5.2.
A similar issue has been reported (#390), and the hanging problem has been fixed in PR #391. However, you still need to attach the event loop to the child watcher explicitly in order to manage subprocesses without relying on the asyncio policy to provide the event loop (specific to Unix):
asyncio.set_event_loop(None)
loop = asyncio.new_event_loop()
asyncio.get_child_watcher().attach_loop(loop)
Also related: #421, see this comment.
@vxgmichel thanks for the information!
Attaching loop to the child watcher helps.
So this is already fixed and will be included in the next Python 3.5 release?