Skip to content

Fix connection reuse for file-like data payloads #10915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 77 commits into from
May 22, 2025
Merged
Changes from 1 commit
Commits
Show all changes
77 commits
Select commit Hold shift + click to select a range
612af32
Ensure writer is cleaned up if reader is already at eof
bdraco May 21, 2025
2a5abcd
still not a good fix
bdraco May 21, 2025
25dfcd8
still not a good fix
bdraco May 21, 2025
af71a7f
signal writer
bdraco May 21, 2025
f5128cc
signal writer
bdraco May 21, 2025
1a021f1
signal writer
bdraco May 21, 2025
2739c6a
cleanup
bdraco May 21, 2025
d65abb2
revert
bdraco May 21, 2025
a7e332f
cleanup
bdraco May 21, 2025
467442e
cleanup
bdraco May 21, 2025
e20652c
tweak
bdraco May 21, 2025
54c5579
tweak
bdraco May 21, 2025
fe8be39
fixes
bdraco May 21, 2025
4a051ad
debug
bdraco May 21, 2025
8b4ca43
debug
bdraco May 21, 2025
fec1397
more fixes
bdraco May 21, 2025
c582931
more fixes
bdraco May 21, 2025
3b75f9a
more fixes
bdraco May 21, 2025
373f157
reduce
bdraco May 21, 2025
99d8f19
reduce
bdraco May 21, 2025
e902724
reduce
bdraco May 21, 2025
653933c
reduce
bdraco May 21, 2025
fdae4f2
fix
bdraco May 21, 2025
f2ae144
all working now
bdraco May 21, 2025
14a1f41
fixes
bdraco May 21, 2025
05f6a3c
tweak
bdraco May 21, 2025
06c5945
tweak
bdraco May 21, 2025
d638b49
tweak
bdraco May 21, 2025
ca9f98c
tweak
bdraco May 21, 2025
2479a36
docs
bdraco May 21, 2025
71dfa14
docs
bdraco May 21, 2025
6dfdde1
docs
bdraco May 21, 2025
5cb1581
cleanup
bdraco May 21, 2025
ad5ec73
cleanup
bdraco May 21, 2025
8e9aaf8
cleanup
bdraco May 21, 2025
df1d173
cleanup
bdraco May 21, 2025
9683667
cleanup
bdraco May 21, 2025
9a3bde6
preen
bdraco May 21, 2025
133ec6c
cover
bdraco May 21, 2025
4ff2c43
cover
bdraco May 21, 2025
e2e14be
fix
bdraco May 21, 2025
a448922
fix
bdraco May 21, 2025
dbb13ad
convert reproducer to test
bdraco May 21, 2025
0330f2f
fix
bdraco May 21, 2025
25693d4
fix
bdraco May 21, 2025
fa19c4b
preen
bdraco May 21, 2025
5d6fac3
preen
bdraco May 21, 2025
698e9e3
cover
bdraco May 21, 2025
a108969
cleanup
bdraco May 21, 2025
b88e788
Merge branch 'master' into wait_for_writer_eof
bdraco May 21, 2025
ff04923
fixes
bdraco May 21, 2025
9f9cce8
Merge remote-tracking branch 'upstream/wait_for_writer_eof' into wait…
bdraco May 21, 2025
d46356e
cover missing line
bdraco May 21, 2025
7504200
cover missing line
bdraco May 21, 2025
61fca77
proper fix
bdraco May 21, 2025
6106242
fixes
bdraco May 21, 2025
d66b363
fixes
bdraco May 21, 2025
01feb88
lit
bdraco May 21, 2025
c121180
fix
bdraco May 21, 2025
dea6011
fix
bdraco May 21, 2025
fb3223b
no cover
bdraco May 21, 2025
249a31c
cover one more
bdraco May 21, 2025
32f01e7
cover one more
bdraco May 21, 2025
897fe6b
fix test
bdraco May 21, 2025
f591d65
cleanup
bdraco May 22, 2025
80cd057
fix test
bdraco May 22, 2025
966dc55
cleanup
bdraco May 22, 2025
86756af
Merge branch 'master' into wait_for_writer_eof
bdraco May 22, 2025
f1c3a4d
fix leak in client functional
bdraco May 22, 2025
6159fc9
fix PyPy
bdraco May 22, 2025
749aa1a
revert
bdraco May 22, 2025
7e08cd6
move test to see why its failing on pypy
bdraco May 22, 2025
b6fd37c
Revert "move test to see why its failing on pypy"
bdraco May 22, 2025
8654141
see if PyPy error moves
bdraco May 22, 2025
d109188
Fix for PyPy not closing file handle if we do not exaust the iterator
bdraco May 22, 2025
f768b7f
Merge branch 'master' into wait_for_writer_eof
bdraco May 22, 2025
042726d
Revert "see if PyPy error moves"
bdraco May 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
signal writer
  • Loading branch information
bdraco committed May 21, 2025
commit af71a7f0a5917aec9db0550ba0fb339e733b94d4
33 changes: 20 additions & 13 deletions aiohttp/client_reqrep.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,11 @@ def check(self, transport: asyncio.Transport) -> None:
_SSL_SCHEMES = frozenset(("https", "wss"))


class _WriterTask(asyncio.Task):

_reader_done: bool = False


# ConnectionKey is a NamedTuple because it is used as a key in a dict
# and a set in the connector. Since a NamedTuple is a tuple it uses
# the fast native tuple __hash__ and __eq__ implementation in CPython.
Expand Down Expand Up @@ -215,7 +220,7 @@ class ClientRequest:
url = URL()
method = "GET"

__writer: Optional["asyncio.Task[None]"] = None # async task for streaming data
__writer: Optional[_WriterTask] = None # async task for streaming data
_continue = None # waiter future for '100 Continue' response

_skip_auto_headers: Optional["CIMultiDict[None]"] = None
Expand Down Expand Up @@ -629,8 +634,11 @@ async def write_bytes(

set_exception(protocol, reraised_exc, underlying_exc)
except asyncio.CancelledError:
# Body hasn't been fully sent, so connection can't be reused.
conn.close()
if self.__writer and self.__writer._reader_done:
conn.release()
else:
# Body hasn't been fully sent, so connection can't be reused.
conn.close()
raise
except Exception as underlying_exc:
set_exception(
Expand Down Expand Up @@ -710,9 +718,9 @@ async def send(self, conn: "Connection") -> "ClientResponse":
# Optimization for Python 3.12, try to write
# bytes immediately to avoid having to schedule
# the task on the event loop.
task = asyncio.Task(coro, loop=self.loop, eager_start=True)
task = _WriterTask(coro, loop=self.loop, eager_start=True)
else:
task = self.loop.create_task(coro)
task = _WriterTask(coro, loop=self.loop)
if task.done():
task = None
else:
Expand Down Expand Up @@ -801,14 +809,14 @@ class ClientResponse(HeadersMixin):

_resolve_charset: Callable[["ClientResponse", bytes], str] = lambda *_: "utf-8"

__writer: Optional["asyncio.Task[None]"] = None
__writer: Optional[_WriterTask] = None

def __init__(
self,
method: str,
url: URL,
*,
writer: "Optional[asyncio.Task[None]]",
writer: "Optional[_WriterTask]",
continue100: Optional["asyncio.Future[bool]"],
timer: Optional[BaseTimerContext],
request_info: RequestInfo,
Expand Down Expand Up @@ -1021,12 +1029,11 @@ async def start(self, connection: "Connection") -> "ClientResponse":

# payload eof handler
if payload.is_eof():
# If payload is already EOF, release the connection
# to ensure it can be reused as cancelling the writer
# will close it and prevent reuse. The writer may still
# be closing its stream or waiting for an executor job
# to finish.
self._connection.release()
# Make sure to tell the writer that the reader is done
# so it doesn't close the connection if it gets cancelled
# because there is still payload being read.
if self.__writer is not None:
self.__writer._reader_done = True
self._response_eof()
else:
payload.on_eof(self._response_eof)
Expand Down
Loading