From 23ff7166e0e209a1c430423800c70b70e5b745c4 Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Wed, 10 Feb 2021 16:40:29 -0500 Subject: [PATCH] s3io.Reader: reissue request on ERRCONNRESET If a "connection error" reset is encountered while reading a s3 object attempt to restart the connection and resume read at the current offset. This solves a bug found when trying to ingest several s3 hosted log files: several files will stop ingesting with the error "connection reset by peer". There seems to be a curious behavior of the s3 service that happens when a single session maintains numerous long-running download connections to various objects in a bucket- the service appears to reset connections at random. See: https://github.com/aws/aws-sdk-go/issues/1242 Closes #2005 --- pkg/s3io/reader.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/s3io/reader.go b/pkg/s3io/reader.go index 064d63a988..565ddd4070 100644 --- a/pkg/s3io/reader.go +++ b/pkg/s3io/reader.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "syscall" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" @@ -67,6 +68,7 @@ func (r *Reader) Read(p []byte) (int, error) { if r.offset >= r.size { return 0, io.EOF } +request: if r.body == nil { body, err := r.makeRequest(r.offset, r.size-r.offset) if err != nil { @@ -75,6 +77,18 @@ func (r *Reader) Read(p []byte) (int, error) { r.body = body } n, err := r.body.Read(p) + + // If the error is result of a connection reset set the body to nil and + // attempt to restart the connection at the current offset. This a curious + // behavior of the s3 service that seems to happen when a single session + // maintains numerous long-running download connections to various objects + // in a bucket. Set body to nil and reissue the request at the current + // offset. + // See: https://github.com/aws/aws-sdk-go/issues/1242 + if errors.Is(err, syscall.ECONNRESET) { + r.body = nil + goto request + } if err == io.EOF { err = nil }