Migrate weaveworks/common packages used by Loki

CHANGELOG.md

@@ -131,6 +131,7 @@
 * [ENHANCEMENT] Lifecycler: allow instances to register in ascending order of ids in case of spread minimizing token generation strategy. #326
 * [ENHANCEMENT] Remove dependency on `github.com/weaveworks/common` package by migrating code to a corresponding package in `github.com/grafana/dskit`. #342
 * [ENHANCEMENT] Add ability to pass TLS certificates and keys inline when configuring server-side TLS. #349
+* [ENHANCEMENT] Migrate `github.com/weaveworks/common/aws` and `github.com/weaveworks/common/test` packages to corresponding packages in `github.com/grafana/dskit`. #356
 * [BUGFIX] spanlogger: Support multiple tenant IDs. #59
 * [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85
 * [BUGFIX] Ring: `ring_member_ownership_percent` and `ring_tokens_owned` metrics are not updated on scale down. #109

aws/config.go

@@ -0,0 +1,69 @@
+// Provenance-includes-location: https://github.com/weaveworks/common/blob/main/aws/config.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Weaveworks Ltd.
+
+package aws
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+// ConfigFromURL returns AWS config from given URL. It expects escaped
+// AWS Access key ID & Secret Access Key to be encoded in the URL. It
+// also expects region specified as a host (letting AWS generate full
+// endpoint) or fully valid endpoint with dummy region assumed (e.g
+// for URLs to emulated services).
+func ConfigFromURL(awsURL *url.URL) (*aws.Config, error) {
+	config := aws.NewConfig().
+		// Use a custom http.Client with the golang defaults but also specifying
+		// MaxIdleConnsPerHost because of a bug in golang https://github.com/golang/go/issues/13801
+		// where MaxIdleConnsPerHost does not work as expected.
+		WithHTTPClient(&http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				DialContext: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+					DualStack: true,
+				}).DialContext,
+				MaxIdleConns:          100,
+				IdleConnTimeout:       90 * time.Second,
+				MaxIdleConnsPerHost:   100,
+				TLSHandshakeTimeout:   3 * time.Second,
+				ExpectContinueTimeout: 1 * time.Second,
+			},
+		})
+
+	if awsURL.User != nil {
+		username := awsURL.User.Username()
+		password, _ := awsURL.User.Password()
+
+		// We request at least the username or password being set to enable the static credentials.
+		if username != "" || password != "" {
+			config = config.WithCredentials(credentials.NewStaticCredentials(username, password, ""))
+		}
+	}
+
+	if strings.Contains(awsURL.Host, ".") {
+		region := os.Getenv("AWS_REGION")
+		if region == "" {
+			region = "dummy"
+		}
+		if awsURL.Scheme == "https" {
+			return config.WithEndpoint(fmt.Sprintf("https://%s", awsURL.Host)).WithRegion(region), nil
+		}
+		return config.WithEndpoint(fmt.Sprintf("http://%s", awsURL.Host)).WithRegion(region), nil
+	}
+
+	// Let AWS generate default endpoint based on region passed as a host in URL.
+	return config.WithRegion(awsURL.Host), nil
+}

aws/config_test.go

@@ -0,0 +1,89 @@
+// Provenance-includes-location: https://github.com/weaveworks/common/blob/main/aws/config_test.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Weaveworks Ltd.
+
+package aws
+
+import (
+	"net/url"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAWSConfigFromURL(t *testing.T) {
+	for i, tc := range []struct {
+		url            string
+		expectedKey    string
+		expectedSecret string
+		expectedRegion string
+		expectedEp     string
+	}{
+		{
+			"s3://abc:[email protected]:4569",
+			"abc",
+			"123",
+			"dummy",
+			"http://s3.default.svc.cluster.local:4569",
+		},
+		{
+			"s3://@us-east-1/test-bucket",
+			"",
+			"",
+			"us-east-1",
+			"",
+		},
+		{
+			"dynamodb://user:[email protected]:8000/cortex",
+			"user",
+			"pass",
+			"dummy",
+			"http://dynamodb.default.svc.cluster.local:8000",
+		},
+		{
+			// No credentials.
+			"s3://s3.default.svc.cluster.local:4569",
+			"",
+			"",
+			"dummy",
+			"http://s3.default.svc.cluster.local:4569",
+		},
+		{
+			"s3://keyWithEscapedSlashAtTheEnd%2F:%24%2C%26%2C%2B%2C%27%2C%2F%2C%3A%2C%3B%2C%3D%2C%3F%2C%40@eu-west-2/bucket1",
+			"keyWithEscapedSlashAtTheEnd/",
+			"$,&,+,',/,:,;,=,?,@",
+			"eu-west-2",
+			"",
+		},
+	} {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			parsedURL, err := url.Parse(tc.url)
+			require.NoError(t, err)
+
+			cfg, err := ConfigFromURL(parsedURL)
+			require.NoError(t, err)
+
+			if tc.expectedKey == "" && tc.expectedSecret == "" {
+				assert.Nil(t, cfg.Credentials)
+			} else {
+				require.NotNil(t, cfg.Credentials)
+				val, err := cfg.Credentials.Get()
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectedKey, val.AccessKeyID)
+				assert.Equal(t, tc.expectedSecret, val.SecretAccessKey)
+			}
+
+			require.NotNil(t, cfg.Region)
+			assert.Equal(t, tc.expectedRegion, *cfg.Region)
+
+			if tc.expectedEp != "" {
+				require.NotNil(t, cfg.Endpoint)
+				assert.Equal(t, tc.expectedEp, *cfg.Endpoint)
+			} else {
+				assert.Nil(t, cfg.Endpoint)
+			}
+		})
+	}
+}

go.mod

@@ -6,8 +6,10 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/alicebob/miniredis v2.5.0+incompatible
 	github.com/armon/go-metrics v0.3.10
+	github.com/aws/aws-sdk-go v1.44.321
 	github.com/cespare/xxhash v1.1.0
 	github.com/cristalhq/hedgedhttp v0.7.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/felixge/httpsnoop v1.0.3
 	github.com/go-kit/log v0.2.1
@@ -29,6 +31,7 @@ require (
 	github.com/opentracing-contrib/go-stdlib v1.0.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 	github.com/prometheus/client_golang v1.15.1
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.44.0
@@ -62,7 +65,6 @@ require (
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -84,7 +86,6 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/onsi/gomega v1.24.0 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect

go.sum

@@ -409,6 +409,8 @@ github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go v1.44.321 h1:iXwFLxWjZPjYqjPq0EcCs46xX7oDLEELte1+BzgpKk8=
+github.com/aws/aws-sdk-go v1.44.321/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -659,6 +661,9 @@ github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -964,6 +969,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
@@ -1098,13 +1104,15 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

test/diff.go

@@ -0,0 +1,28 @@
+// Provenance-includes-location: https://github.com/weaveworks/common/blob/main/test/diff.go
+// Provenance-includes-license: Apache-2.0
+// Provenance-includes-copyright: Weaveworks Ltd.
+
+package test
+
+import (
+	"github.com/davecgh/go-spew/spew"
+	"github.com/pmezard/go-difflib/difflib"
+)
+
+// Diff diffs two arbitrary data structures, giving human-readable output.
+func Diff(want, have interface{}) string {
+	config := spew.NewDefaultConfig()
+	// Set ContinueOnMethod to true if you cannot see a difference and
+	// want to look beyond the String() method
+	config.ContinueOnMethod = false
+	config.SortKeys = true
+	config.SpewKeys = true
+	text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
+		A:        difflib.SplitLines(config.Sdump(want)),
+		B:        difflib.SplitLines(config.Sdump(have)),
+		FromFile: "want",
+		ToFile:   "have",
+		Context:  3,
+	})
+	return "\n" + text
+}