From 119d6397974b8044ba2e15c087f7d4d51cd9600b Mon Sep 17 00:00:00 2001 From: Wayback Archiver <66856220+waybackarchiver@users.noreply.github.com> Date: Sat, 16 Jul 2022 17:15:23 +0000 Subject: [PATCH] Context leak detection --- go.mod | 8 +++++--- go.sum | 14 ++++++++++---- pooling/pooling_test.go | 12 ++++++++++++ publish/publish_test.go | 15 +++++++++++++++ service/service.go | 28 ++++++++++------------------ service/utils_test.go | 8 ++++++++ 6 files changed, 60 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 8a96f7e8..8a57fdee 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896 github.com/wabarc/go-anonfile v0.1.0 github.com/wabarc/go-catbox v0.1.0 - github.com/wabarc/helper v0.0.0-20220320101038-dbb2ea36249b + github.com/wabarc/helper v0.0.0-20220717024606-0f088ee08afa github.com/wabarc/imgbb v1.0.0 github.com/wabarc/ipfs-pinner v1.1.1-0.20220126131044-16299c0dd43d github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792 @@ -45,7 +45,7 @@ require ( github.com/wabarc/telegra.ph v0.0.0-20220501011455-b0b8c35c6d09 github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5 go.etcd.io/bbolt v1.3.6 - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 + golang.org/x/net v0.0.0-20220708220712-1185a9018129 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3 maunium.net/go/mautrix v0.10.12 @@ -69,6 +69,7 @@ require ( github.com/dghubble/sling v1.3.0 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/dop251/goja v0.0.0-20211211112501-fb27c91c26ed // indirect + github.com/fortytw2/leaktest v1.3.0 // indirect github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/gobwas/httphead v0.1.0 // indirect @@ -130,11 +131,12 @@ require ( github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect github.com/ybbus/httpretry v1.0.1 // indirect golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect - golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect mvdan.cc/xurls/v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 9fa9540d..0b9d64f1 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= @@ -711,8 +713,9 @@ github.com/wabarc/helper v0.0.0-20210614160629-1a5ba5e551eb/go.mod h1:TuTZtoiOu9 github.com/wabarc/helper v0.0.0-20210701193643-e0fe0a807cb9/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E= github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ= github.com/wabarc/helper v0.0.0-20211225065210-3d35291efe54/go.mod h1:uS6mimKlWkGvEZXkJ6JoW7LYnnB2JP6dLU9q7pgDaWQ= -github.com/wabarc/helper v0.0.0-20220320101038-dbb2ea36249b h1:eBee82cXn3HoLWopCPBatja684Ff/Y/uH+U3Va8IY3M= github.com/wabarc/helper v0.0.0-20220320101038-dbb2ea36249b/go.mod h1:JGo6GZ+m/xGV63ZiNH0apOrWczV/f8w2JzjoTOg8u58= +github.com/wabarc/helper v0.0.0-20220717024606-0f088ee08afa h1:uYKwuBwCMI1E3huXnl6YFo30oa58UTcS1Tih9R1sI04= +github.com/wabarc/helper v0.0.0-20220717024606-0f088ee08afa/go.mod h1:3yNPJN7DuTY7R2Ol4r/G7W2dO4XYywqdHk1GoErCmqs= github.com/wabarc/imgbb v1.0.0 h1:vNud9qY4dRGp1nikb+GHIrc/sE3WOdNq5gfT/vhWEUQ= github.com/wabarc/imgbb v1.0.0/go.mod h1:piz6QXkVfFpQbaQHx70/rPDwaICkW5Ew3lShTzbnVt0= github.com/wabarc/ipfs-pinner v1.1.1-0.20220126131044-16299c0dd43d h1:rSjZNYtr/5ep62nmcAF31v8zUGb0scD3bov/vAzYUpc= @@ -889,8 +892,9 @@ golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= +golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1006,8 +1010,9 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1260,8 +1265,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pooling/pooling_test.go b/pooling/pooling_test.go index 76ff4611..8e20e4c3 100644 --- a/pooling/pooling_test.go +++ b/pooling/pooling_test.go @@ -10,11 +10,14 @@ import ( "testing" "time" + "github.com/wabarc/helper" "github.com/wabarc/logger" "github.com/wabarc/wayback/config" ) func TestRoll(t *testing.T) { + defer helper.CheckTest(t) + var err error parser := config.NewParser() if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -25,6 +28,7 @@ func TestRoll(t *testing.T) { c := 2 p := New(context.Background(), c) p.timeout = 10 * time.Millisecond + defer helper.CheckContext(p.context, t) if l := len(p.resource); l != c { t.Fatalf("The length of pool got %d instead of %d", l, c) @@ -61,6 +65,8 @@ func TestRoll(t *testing.T) { } func TestTimeout(t *testing.T) { + defer helper.CheckTest(t) + var err error parser := config.NewParser() if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -107,6 +113,8 @@ func TestTimeout(t *testing.T) { } func TestMaxRetries(t *testing.T) { + defer helper.CheckTest(t) + var err error parser := config.NewParser() if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -136,6 +144,8 @@ func TestMaxRetries(t *testing.T) { } func TestFallback(t *testing.T) { + defer helper.CheckTest(t) + var err error parser := config.NewParser() if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -168,6 +178,8 @@ func TestFallback(t *testing.T) { } func TestClose(t *testing.T) { + defer helper.CheckTest(t) + var err error parser := config.NewParser() if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { diff --git a/publish/publish_test.go b/publish/publish_test.go index be483cc9..8128c238 100644 --- a/publish/publish_test.go +++ b/publish/publish_test.go @@ -63,6 +63,8 @@ func unsetAllEnv() { } func TestPublishToChannelFromTelegram(t *testing.T) { + defer helper.CheckTest(t) + unsetAllEnv() setTelegramEnv() config.Opts, _ = config.NewParser().ParseEnvironmentVariables() @@ -103,10 +105,13 @@ func TestPublishToChannelFromTelegram(t *testing.T) { ctx := context.WithValue(context.Background(), FlagTelegram, bot) ctx = context.WithValue(ctx, PubBundle{}, bundleExample) + defer helper.CheckContext(ctx, t) To(ctx, collects, FlagTelegram.String()) } func TestPublishTootFromMastodon(t *testing.T) { + defer helper.CheckTest(t) + unsetAllEnv() setMastodonEnv() @@ -142,10 +147,14 @@ func TestPublishTootFromMastodon(t *testing.T) { mstdn := NewMastodon(nil) ctx := context.WithValue(context.Background(), FlagMastodon, mstdn.client) + ctx = context.WithValue(ctx, PubBundle{}, bundleExample) + defer helper.CheckContext(ctx, t) To(ctx, collects, FlagMastodon.String()) } func TestPublishTweetFromTwitter(t *testing.T) { + defer helper.CheckTest(t) + unsetAllEnv() setTwitterEnv() config.Opts, _ = config.NewParser().ParseEnvironmentVariables() @@ -170,6 +179,8 @@ func TestPublishTweetFromTwitter(t *testing.T) { twi := NewTwitter(twitter.NewClient(httpClient)) ctx := context.WithValue(context.Background(), FlagTwitter, twi.client) + ctx = context.WithValue(ctx, PubBundle{}, bundleExample) + defer helper.CheckContext(ctx, t) To(ctx, collects, FlagTwitter.String()) } @@ -178,6 +189,8 @@ func TestPublishToIRCChannelFromIRC(t *testing.T) { } func TestPublishToMatrixRoomFromMatrix(t *testing.T) { + defer helper.CheckTest(t) + unsetAllEnv() setMatrixEnv() @@ -204,5 +217,7 @@ func TestPublishToMatrixRoomFromMatrix(t *testing.T) { mat := NewMatrix(nil) ctx := context.WithValue(context.Background(), "matrix", mat.client) + ctx = context.WithValue(ctx, PubBundle{}, bundleExample) + defer helper.CheckContext(ctx, t) To(ctx, collects, "matrix") } diff --git a/service/service.go b/service/service.go index 7e36b54e..af0db9df 100644 --- a/service/service.go +++ b/service/service.go @@ -25,26 +25,18 @@ func Wayback(ctx context.Context, urls []*url.URL, do func(cols []wayback.Collec var rdx reduxer.Reduxer go func() { - go func() { - var err error - cols, rdx, err = wayback.Wayback(ctx, urls...) - if err != nil { - done <- errors.Wrap(err, "wayback failed") - return - } - defer rdx.Flush() - // push collects to the Meilisearch - if meili != nil { - go meili.push(cols) - } - done <- do(cols, rdx) - }() - - // Block until context is finished. - select { - case <-ctx.Done(): + var err error + cols, rdx, err = wayback.Wayback(ctx, urls...) + if err != nil { + done <- errors.Wrap(err, "wayback failed") return } + defer rdx.Flush() + // push collects to the Meilisearch + if meili != nil { + meili.push(cols) + } + done <- do(cols, rdx) }() select { diff --git a/service/utils_test.go b/service/utils_test.go index 8551420a..5c62609b 100644 --- a/service/utils_test.go +++ b/service/utils_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/wabarc/helper" "github.com/wabarc/logger" "github.com/wabarc/wayback" "github.com/wabarc/wayback/config" @@ -19,6 +20,8 @@ import ( ) func TestMatchURL(t *testing.T) { + defer helper.CheckTest(t) + parser := config.NewParser() var err error if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -69,6 +72,8 @@ func TestMatchURL(t *testing.T) { } func TestExcludeURL(t *testing.T) { + defer helper.CheckTest(t) + parser := config.NewParser() var err error if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -110,6 +115,8 @@ func TestExcludeURL(t *testing.T) { } func TestWayback(t *testing.T) { + defer helper.CheckTest(t) + parser := config.NewParser() var err error if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil { @@ -120,6 +127,7 @@ func TestWayback(t *testing.T) { u, _ := url.Parse("https://example.com/") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() + defer helper.CheckContext(ctx, t) urls := []*url.URL{u} do := func(cols []wayback.Collect, rdx reduxer.Reduxer) error {