From 30bb6c714fc7a9b60ec6c2da45094b6eed107a40 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 31 Mar 2021 09:01:41 +0100 Subject: [PATCH 1/4] Bump go-cleanhttp to 0.5.2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9c0ae09b7..b89a575b1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/cheggaaa/pb v1.0.27 github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.7.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.0 + github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-safetemp v1.0.0 github.com/hashicorp/go-version v1.1.0 github.com/klauspost/compress v1.11.2 diff --git a/go.sum b/go.sum index 99c5ea66b..8aeb2b9c3 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= From 7a21f4cd77f96439e9d00edf2e2ffab4923b9d34 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 31 Mar 2021 09:01:56 +0100 Subject: [PATCH 2/4] add test --- get_http_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/get_http_test.go b/get_http_test.go index 6d7fb90ac..2907b35a3 100644 --- a/get_http_test.go +++ b/get_http_test.go @@ -298,6 +298,23 @@ func TestHttpGetter_file(t *testing.T) { assertContents(t, dst, "Hello\n") } +// TestHttpGetter_http2server tests that http.Request is not reused +// between HEAD & GET, which would lead to race condition in HTTP/2. +// This test is only meaningful for the race detector (go test -race). +func TestHttpGetter_http2server(t *testing.T) { + g := new(HttpGetter) + src, err := url.Parse("https://releases.hashicorp.com/terraform/0.14.0/terraform_0.14.0_SHA256SUMS") + if err != nil { + t.Fatal(err) + } + dst := tempTestFile(t) + + err = g.GetFile(dst, src) + if err != nil { + t.Fatal(err) + } +} + func TestHttpGetter_auth(t *testing.T) { ln := testHttpServer(t) defer ln.Close() From 487a1fa499a112473646b26320f821758e7dcd27 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 31 Mar 2021 09:03:53 +0100 Subject: [PATCH 3/4] test for race conditions in CI --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8cb6ea314..6a29dca3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,9 @@ commands: type: string platform: type: string + govet: + type: string + default: "" steps: - run: name: "Run go tests" @@ -22,7 +25,7 @@ commands: PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname) echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" echo $PACKAGE_NAMES - << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES + << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -race -vet=<< parameters.govet >> -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES jobs: linux-tests: @@ -140,6 +143,9 @@ jobs: - run-gotests: cmd: "./gotestsum.exe" platform: "win" + # Otherwise gcc is required for race detector + # See https://github.com/golang/go/issues/27089 + govet: "off" # Save coverage report parts - persist_to_workspace: From 3cca945b8b947e79d3ea1fc653e251c6670f8e4b Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Wed, 31 Mar 2021 09:09:04 +0100 Subject: [PATCH 4/4] Avoid reusing the same http.Request --- get_http.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/get_http.go b/get_http.go index b0229d763..8c2278d6a 100644 --- a/get_http.go +++ b/get_http.go @@ -181,7 +181,6 @@ func (g *HttpGetter) GetFile(dst string, src *url.URL) error { if fi, err := f.Stat(); err == nil { if _, err = f.Seek(0, io.SeekEnd); err == nil { currentFileSize = fi.Size() - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", currentFileSize)) if currentFileSize >= headResp.ContentLength { // file already present return nil @@ -191,7 +190,17 @@ func (g *HttpGetter) GetFile(dst string, src *url.URL) error { } } } - req.Method = "GET" + + req, err = http.NewRequest("GET", src.String(), nil) + if err != nil { + return err + } + if g.Header != nil { + req.Header = g.Header.Clone() + } + if currentFileSize > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", currentFileSize)) + } resp, err := g.Client.Do(req) if err != nil {