`Unable to grow response stream IOBuffer` when retrying `TimeoutError`

[kleinschmidt]

I don't have a MWE for this, but I'm polling an HTTP.jl server (with HTTP.get(...; readtimeout=10)) monitoring a task status. When the task status changes, there can be some significant delay in generating the response, which leads to the request timing out. I want to retry these timeouts so I've added an additional retrycheck=(_, ex, args...) -> ex isa HTTP.Exceptions.TimeoutError. But when the request eventually succeeds after some number of retries, I get the following stacktrace:

caused by: TaskFailedException

    nested task error: ArgumentError: Unable to grow response stream IOBuffer 266 large enough for response body size: 286
    Stacktrace:
     [1] readbytes!(http::HTTP.Streams.Stream{HTTP.Messages.Response, HTTP.Connections.Connection{…}}, buf::IOBuffer, n::Int64)
       @ HTTP.Streams ~/.julia/packages/HTTP/4AUPl/src/Streams.jl:302
     [2] readall!(http::HTTP.Streams.Stream{HTTP.Messages.Response, HTTP.Connections.Connection{…}}, buf::IOBuffer)
       @ HTTP.Streams ~/.julia/packages/HTTP/4AUPl/src/Streams.jl:329
     [3] readbody!(stream::HTTP.Streams.Stream{…}, res::HTTP.Messages.Response, buf_or_stream::HTTP.Streams.Stream{…}, lock::ReentrantLock)
       @ HTTP.StreamRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/StreamRequest.jl:164
     [4] readbody(stream::HTTP.Streams.Stream{…}, res::HTTP.Messages.Response, decompress::Nothing, lock::ReentrantLock)
       @ HTTP.StreamRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/StreamRequest.jl:142
     [5] macro expansion
       @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/StreamRequest.jl:54 [inlined]
     [6] (::HTTP.StreamRequest.var"#3#5"{…})()
       @ HTTP.StreamRequest ~/.julia/packages/ConcurrentUtilities/ofY4K/src/ConcurrentUtilities.jl:10
Stacktrace:
 [1] sync_end(c::Channel{Any})
   @ Base ./task.jl:455
 [2] macro expansion
   @ ./task.jl:487 [inlined]
 [3] streamlayer(stream::HTTP.Streams.Stream{…}; iofunction::Nothing, decompress::Nothing, logerrors::Bool, logtag::Nothing, timedout::ConcurrentUtilities.TimedOut{…}, kw::@Kwargs{…})
   @ HTTP.StreamRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/StreamRequest.jl:35
 [4] streamlayer
   @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/StreamRequest.jl:21 [inlined]
 [5] (::HTTP.ExceptionRequest.var"#exceptions#2"{…})(stream::HTTP.Streams.Stream{…}; status_exception::Bool, timedout::ConcurrentUtilities.TimedOut{…}, logerrors::Bool, logtag::Nothing, kw::@Kwargs{…})
   @ HTTP.ExceptionRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/ExceptionRequest.jl:14
 [6] exceptions
   @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/ExceptionRequest.jl:13 [inlined]
 [7] #2
   @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/TimeoutRequest.jl:22 [inlined]
 [8] macro expansion
   @ ~/.julia/packages/ConcurrentUtilities/ofY4K/src/try_with_timeout.jl:92 [inlined]
 [9] (::ConcurrentUtilities.var"#2#4"{HTTP.Messages.Response, Channel{…}, HTTP.TimeoutRequest.var"#2#5"{…}, Timer})()
   @ ConcurrentUtilities ~/.julia/packages/ConcurrentUtilities/ofY4K/src/ConcurrentUtilities.jl:10
Stacktrace:
  [1] try_yieldto(undo::typeof(Base.ensure_rescheduled))
    @ Base ./task.jl:945
  [2] wait()
    @ Base ./task.jl:1009
  [3] wait(c::Base.GenericCondition{ReentrantLock}; first::Bool)
    @ Base ./condition.jl:130
  [4] wait
    @ ./condition.jl:125 [inlined]
  [5] take_unbuffered(c::Channel{HTTP.Messages.Response})
    @ Base ./channels.jl:494
  [6] take!
    @ ./channels.jl:471 [inlined]
  [7] try_with_timeout(f::Function, timeout::Int64, ::Type{HTTP.Messages.Response})
    @ ConcurrentUtilities ~/.julia/packages/ConcurrentUtilities/ofY4K/src/try_with_timeout.jl:99
  [8] (::HTTP.TimeoutRequest.var"#timeouts#3"{…})(stream::HTTP.Streams.Stream{…}; readtimeout::Int64, logerrors::Bool, logtag::Nothing, kw::@Kwargs{…})
    @ HTTP.TimeoutRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/TimeoutRequest.jl:21
  [9] (::HTTP.ConnectionRequest.var"#connections#4"{…})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Nothing, readtimeout::Int64, connect_timeout::Int64, logerrors::Bool, logtag::Nothing, closeimmediately::Bool, kw::@Kwargs{…})
    @ HTTP.ConnectionRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/ConnectionRequest.jl:122
 [10] connections
    @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/ConnectionRequest.jl:60 [inlined]
 [11] (::Base.var"#96#98"{…})(args::HTTP.Messages.Request; kwargs::@Kwargs{…})
    @ Base ./error.jl:296
 [12] (::HTTP.RetryRequest.var"#manageretries#3"{…})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::@Kwargs{…})
    @ HTTP.RetryRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/RetryRequest.jl:75
 [13] manageretries
    @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/RetryRequest.jl:30 [inlined]
 [14] (::HTTP.CookieRequest.var"#managecookies#4"{…})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::@Kwargs{…})
    @ HTTP.CookieRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/CookieRequest.jl:42
 [15] managecookies
    @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/CookieRequest.jl:19 [inlined]
 [16] (::HTTP.HeadersRequest.var"#defaultheaders#2"{…})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Nothing, basicauth::Bool, detect_content_type::Bool, canonicalize_headers::Bool, kw::@Kwargs{…})
    @ HTTP.HeadersRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/HeadersRequest.jl:71
 [17] defaultheaders
    @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/HeadersRequest.jl:14 [inlined]
 [18] (::HTTP.RedirectRequest.var"#redirects#3"{…})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::@Kwargs{…})
    @ HTTP.RedirectRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/RedirectRequest.jl:25
 [19] redirects
    @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/RedirectRequest.jl:14 [inlined]
 [20] (::HTTP.MessageRequest.var"#makerequest#3"{…})(method::String, url::URIs.URI, headers::Vector{…}, body::Vector{…}; copyheaders::Bool, response_stream::Nothing, http_version::HTTP.Strings.HTTPVersion, verbose::Int64, kw::@Kwargs{…})
    @ HTTP.MessageRequest ~/.julia/packages/HTTP/4AUPl/src/clientlayers/MessageRequest.jl:35
 [21] makerequest
    @ ~/.julia/packages/HTTP/4AUPl/src/clientlayers/MessageRequest.jl:24 [inlined]
 [22] request(stack::HTTP.MessageRequest.var"#makerequest#3"{…}, method::String, url::String, h::Vector{…}, b::Vector{…}, q::Nothing; headers::Vector{…}, body::Vector{…}, query::Nothing, kw::@Kwargs{…})
    @ HTTP ~/.julia/packages/HTTP/4AUPl/src/HTTP.jl:457
 [23] #request#20
    @ ~/.julia/packages/HTTP/4AUPl/src/HTTP.jl:315 [inlined]
 [24] request (repeats 2 times)
    @ ~/.julia/packages/HTTP/4AUPl/src/HTTP.jl:313 [inlined]
 [25] get
    @ ~/.julia/packages/HTTP/4AUPl/src/HTTP.jl:518 [inlined]

What is suspicious is that the previous responses have 266 byte bodies, while the new response is 286 (these are just JSON payloads; one field changes from null to a timestamp).

I don't have an MWE but when I remove the readtimeout (or set it long enough that the read doesn't timeout), everything works fine. My hunch is that somehow the IOBuffer for the response body is being-reused between requests, but I haven't gotten any farther than that (moving the timeout retry outside of HTTP.get using an additional Base.retry solved my immediate problem).