diff --git a/README.md b/README.md index 073917ebce4a..087220e68f06 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -

+< +p align="center">

diff --git a/pkg/cmd/internal/issues/formatter_unit.go b/pkg/cmd/internal/issues/formatter_unit.go index 21cefcbc7318..52d9a46b3b26 100644 --- a/pkg/cmd/internal/issues/formatter_unit.go +++ b/pkg/cmd/internal/issues/formatter_unit.go @@ -67,20 +67,29 @@ var UnitTestFormatter = IssueFormatter{ r.CodeBlock("", data.CondensedMessage.Digest(50)) } - r.Collapsed("Help", func() { - if data.HelpCommand != nil { - data.HelpCommand(r) + if len(data.Parameters) != 0 { + params := make([]string, 0, len(data.Parameters)) + for name := range data.Parameters { + params = append(params, name) } + sort.Strings(params) - if len(data.Parameters) != 0 { - r.Escaped("Parameters in this failure:\n") - for _, p := range data.Parameters { - r.Escaped("\n- ") - r.Escaped(p) - r.Escaped("\n") + r.P(func() { + r.Escaped("Parameters: ") + separator := "" + for _, name := range params { + r.Escaped(separator) + r.Code(fmt.Sprintf("%s=%s", name, data.Parameters[name])) + separator = ", " } - } - }) + }) + } + + if data.HelpCommand != nil { + r.Collapsed("Help", func() { + data.HelpCommand(r) + }) + } if len(data.RelatedIssues) > 0 { r.Collapsed("Same failure on other branches", func() { diff --git a/pkg/cmd/internal/issues/issues.go b/pkg/cmd/internal/issues/issues.go index c8eade867ba5..afb3bce5c362 100644 --- a/pkg/cmd/internal/issues/issues.go +++ b/pkg/cmd/internal/issues/issues.go @@ -138,12 +138,32 @@ func newPoster(client *github.Client, opts *Options) *poster { } } +// parameters returns the parameters to be displayed in the failure +// report. It adds the default parameters (currently, TAGS and +// GOFLAGS) to the list of parameters passed by the caller. +func (p *poster) parameters(extraParams map[string]string) map[string]string { + ps := map[string]string{} + for name, value := range extraParams { + ps[name] = value + } + + if p.Tags != "" { + ps["TAGS"] = p.Tags + } + if p.Goflags != "" { + ps["GOFLAGS"] = p.Goflags + } + + return ps +} + // Options configures the issue poster. type Options struct { Token string // GitHub API token Org string Repo string SHA string + BuildTypeID string BuildID string ServerURL string Branch string @@ -163,6 +183,7 @@ func DefaultOptionsFromEnv() *Options { githubRepoEnv = "GITHUB_REPO" githubAPITokenEnv = "GITHUB_API_TOKEN" teamcityVCSNumberEnv = "BUILD_VCS_NUMBER" + teamcityBuildTypeIDEnv = "TC_BUILDTYPE_ID" teamcityBuildIDEnv = "TC_BUILD_ID" teamcityServerURLEnv = "TC_SERVER_URL" teamcityBuildBranchEnv = "TC_BUILD_BRANCH" @@ -179,6 +200,7 @@ func DefaultOptionsFromEnv() *Options { // at least it'll be obvious that something went wrong (as an // issue will be posted pointing at that SHA). SHA: maybeEnv(teamcityVCSNumberEnv, "8548987813ff9e1b8a9878023d3abfc6911c16db"), + BuildTypeID: maybeEnv(teamcityBuildTypeIDEnv, "BUILDTYPE_ID-not-found-in-env"), BuildID: maybeEnv(teamcityBuildIDEnv, "NOTFOUNDINENV"), ServerURL: maybeEnv(teamcityServerURLEnv, "https://server-url-not-found-in-env.com"), Branch: maybeEnv(teamcityBuildBranchEnv, "branch-not-found-in-env"), @@ -214,8 +236,9 @@ type TemplateData struct { PostRequest // This is foo/bar instead of github.com/cockroachdb/cockroach/pkg/foo/bar. PackageNameShort string - // GOFLAGS=-foo TAGS=-race etc. - Parameters []string + // Parameters includes relevant test or build parameters, such as + // build tags or cluster configuration + Parameters map[string]string // The message, garnished with helpers that allow extracting the useful // bots. CondensedMessage CondensedMessage @@ -246,7 +269,7 @@ func (p *poster) templateData( } return TemplateData{ PostRequest: req, - Parameters: p.parameters(), + Parameters: p.parameters(req.ExtraParams), CondensedMessage: CondensedMessage(req.Message), Branch: p.Branch, Commit: p.SHA, @@ -370,39 +393,27 @@ func (p *poster) post(origCtx context.Context, formatter IssueFormatter, req Pos func (p *poster) teamcityURL(tab, fragment string) *url.URL { options := url.Values{} - options.Add("buildId", p.BuildID) - options.Add("tab", tab) + options.Add("buildTab", tab) u, err := url.Parse(p.ServerURL) if err != nil { log.Fatal(err) } u.Scheme = "https" - u.Path = "viewLog.html" + u.Path = fmt.Sprintf("buildConfiguration/%s/%s", p.BuildTypeID, p.BuildID) u.RawQuery = options.Encode() u.Fragment = fragment return u } func (p *poster) teamcityBuildLogURL() *url.URL { - return p.teamcityURL("buildLog", "") + return p.teamcityURL("log", "") } func (p *poster) teamcityArtifactsURL(artifacts string) *url.URL { return p.teamcityURL("artifacts", artifacts) } -func (p *poster) parameters() []string { - var ps []string - if p.Tags != "" { - ps = append(ps, "TAGS="+p.Tags) - } - if p.Goflags != "" { - ps = append(ps, "GOFLAGS="+p.Goflags) - } - return ps -} - // A PostRequest contains the information needed to create an issue about a // test failure. type PostRequest struct { @@ -412,6 +423,9 @@ type PostRequest struct { TestName string // The test output. Message string + // ExtraParams contains the parameters to be included in a failure + // report, other than the defaults (git branch, test flags). + ExtraParams map[string]string // A path to the test artifacts relative to the artifacts root. If nonempty, // allows the poster formatter to construct a direct URL to this directory. Artifacts string diff --git a/pkg/cmd/internal/issues/issues_test.go b/pkg/cmd/internal/issues/issues_test.go index ba834cd0ec8c..f99bf597fb42 100644 --- a/pkg/cmd/internal/issues/issues_test.go +++ b/pkg/cmd/internal/issues/issues_test.go @@ -37,15 +37,16 @@ func TestPost(t *testing.T) { ) opts := Options{ - Token: "intentionally-unset", - Org: "cockroachdb", - Repo: "cockroach", - SHA: "abcd123", - BuildID: "8008135", - ServerURL: "https://teamcity.example.com", - Branch: "release-0.1", - Tags: "deadlock", - Goflags: "race", + Token: "intentionally-unset", + Org: "cockroachdb", + Repo: "cockroach", + SHA: "abcd123", + BuildTypeID: "nightly123", + BuildID: "8008135", + ServerURL: "https://teamcity.example.com", + Branch: "release-0.1", + Tags: "deadlock", + Goflags: "race", } type testCase struct { @@ -308,6 +309,7 @@ test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTe MentionOnCreate: []string{"@cockroachdb/idonotexistbecausethisisatest"}, HelpCommand: repro, ExtraLabels: []string{"release-blocker"}, + ExtraParams: map[string]string{"ROACHTEST_cloud": "gce"}, } require.NoError(t, p.post(context.Background(), UnitTestFormatter, req)) @@ -349,11 +351,19 @@ func TestPostEndToEnd(t *testing.T) { unset := setEnv(env) defer unset() + params := map[string]string{ + "GOFLAGS": "-race_test", + "ROACHTEST_cloud": "test", + "ROACHTEST_cpu": "2", + } + req := PostRequest{ PackageName: "github.com/cockroachdb/cockroach/pkg/foo/bar", TestName: "TestFooBarBaz", Message: "I'm a message", ExtraLabels: []string{"release-blocker"}, + ExtraParams: params, + HelpCommand: UnitTestHelpCommand(""), } require.NoError(t, Post(context.Background(), UnitTestFormatter, req)) diff --git a/pkg/cmd/internal/issues/render.go b/pkg/cmd/internal/issues/render.go index 7be1e12eaccb..49b84283a6e8 100644 --- a/pkg/cmd/internal/issues/render.go +++ b/pkg/cmd/internal/issues/render.go @@ -72,6 +72,14 @@ func (r *Renderer) Escaped(txt string) { r.printf("%s", html.EscapeString(txt)) } +// Code renders a word or phrase as code. Instead of using backticks +// here (Markdown), we rely on HTML tags since that works even if the +// this function is called within the context of an HTML tag (such as +// a paragraph). +func (r *Renderer) Code(txt string) { + r.HTML("code", func() { r.Escaped(txt) }) +} + // CodeBlock renders a code block. func (r *Renderer) CodeBlock(typ string, txt string) { r.nl() diff --git a/pkg/cmd/internal/issues/testdata/post/failure-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-matching-and-related-issue.txt index 482628aeb007..37fbef1a38bc 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-matching-and-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-matching-and-related-issue.txt @@ -5,21 +5,20 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] createComment owner=cockroachdb repo=cockroach issue=30: -storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` :12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8] ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -34,6 +33,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-matching-issue.txt index 0e97e5342989..adc040f65fef 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-matching-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-matching-issue.txt @@ -5,21 +5,20 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [] createComment owner=cockroachdb repo=cockroach issue=30: -storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` :12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8] ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

@@ -28,6 +27,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-no-issue.txt index 4b777f1cfe6a..b4ec076615b4 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-no-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-no-issue.txt @@ -10,21 +10,20 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: TestReplicateQueueRebalance failed -storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` :12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8] ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

/cc @cockroachdb/idonotexistbecausethisisatest @@ -34,6 +33,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-related-issue.txt index 1b1cf7193743..09477004126d 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-related-issue.txt @@ -10,21 +10,20 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: TestReplicateQueueRebalance failed -storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` :12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8] ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -40,6 +39,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-and-related-issue.txt index 10c73db67dd2..1775db92a1d5 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-and-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-and-related-issue.txt @@ -5,23 +5,22 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "cmd/roachtest: some-roachtest failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] createComment owner=cockroachdb repo=cockroach issue=30: -cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` boom ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See: [FooBar README](https://github.com/cockroachdb/cockroach) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -36,6 +35,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-issue.txt index b806f3d71aa7..82c5609224b9 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-with-url-matching-issue.txt @@ -5,23 +5,22 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "cmd/roachtest: some-roachtest failed" -label:branch-release-0.1: [] createComment owner=cockroachdb repo=cockroach issue=30: -cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` boom ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See: [FooBar README](https://github.com/cockroachdb/cockroach) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

@@ -30,6 +29,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-with-url-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-with-url-no-issue.txt index 48520a0f5078..807375e93953 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-with-url-no-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-with-url-no-issue.txt @@ -10,23 +10,22 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel cmd/roachtest: some-roachtest failed -cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` boom ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See: [FooBar README](https://github.com/cockroachdb/cockroach) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

/cc @cockroachdb/idonotexistbecausethisisatest @@ -36,6 +35,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=cmd%2Froachtest%3A+some-roachtest+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=cmd%2Froachtest%3A+some-roachtest+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/failure-with-url-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/failure-with-url-related-issue.txt index 51e1ba799994..6d859cd529cb 100644 --- a/pkg/cmd/internal/issues/testdata/post/failure-with-url-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/failure-with-url-related-issue.txt @@ -10,23 +10,22 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel cmd/roachtest: some-roachtest failed -cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +cmd/roachtest.some-roachtest [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` boom ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See: [FooBar README](https://github.com/cockroachdb/cockroach) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -42,6 +41,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=cmd%2Froachtest%3A+some-roachtest+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=cmd%2Froachtest.some-roachtest+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0Aboom%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%0ASee%3A+%5BFooBar+README%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%29%0A%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Asome-roachtest.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=cmd%2Froachtest%3A+some-roachtest+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/fatal-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/fatal-matching-and-related-issue.txt index 004ec35201b8..8fd88a0d9b6d 100644 --- a/pkg/cmd/internal/issues/testdata/post/fatal-matching-and-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/fatal-matching-and-related-issue.txt @@ -5,7 +5,7 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestGossipHandlesReplacedNode failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] createComment owner=cockroachdb repo=cockroach issue=30: -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -29,15 +29,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -52,6 +51,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/fatal-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/fatal-matching-issue.txt index 319492ec8ce1..77c97f3af244 100644 --- a/pkg/cmd/internal/issues/testdata/post/fatal-matching-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/fatal-matching-issue.txt @@ -5,7 +5,7 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestGossipHandlesReplacedNode failed" -label:branch-release-0.1: [] createComment owner=cockroachdb repo=cockroach issue=30: -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -29,15 +29,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

@@ -46,6 +45,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/fatal-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/fatal-no-issue.txt index de61eaf7e504..6cf71b62e919 100644 --- a/pkg/cmd/internal/issues/testdata/post/fatal-no-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/fatal-no-issue.txt @@ -10,7 +10,7 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: TestGossipHandlesReplacedNode failed -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -34,15 +34,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

/cc @cockroachdb/idonotexistbecausethisisatest @@ -52,6 +51,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/fatal-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/fatal-related-issue.txt index c03c6ebbd900..f0bfe36b7281 100644 --- a/pkg/cmd/internal/issues/testdata/post/fatal-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/fatal-related-issue.txt @@ -10,7 +10,7 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: TestGossipHandlesReplacedNode failed -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -34,15 +34,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -58,6 +57,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0AF170517+07%3A33%3A43.763059+69575+storage%2Freplica.go%3A1360++%5Bn3%2Cs3%2Cr1%2F3%3A%2FM%7Bin-ax%7D%5D+something+bad+happened%3A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/panic-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/panic-matching-and-related-issue.txt index 8c7497fee58f..40c99de982e7 100644 --- a/pkg/cmd/internal/issues/testdata/post/panic-matching-and-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/panic-matching-and-related-issue.txt @@ -5,7 +5,7 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestGossipHandlesReplacedNode failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] createComment owner=cockroachdb repo=cockroach issue=30: -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -30,15 +30,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -53,6 +52,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/panic-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/panic-matching-issue.txt index 97d57c1f70ec..064724125eba 100644 --- a/pkg/cmd/internal/issues/testdata/post/panic-matching-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/panic-matching-issue.txt @@ -5,7 +5,7 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestGossipHandlesReplacedNode failed" -label:branch-release-0.1: [] createComment owner=cockroachdb repo=cockroach issue=30: -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -30,15 +30,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

@@ -47,6 +46,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/panic-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/panic-no-issue.txt index 0541fec61974..037441dd0c9a 100644 --- a/pkg/cmd/internal/issues/testdata/post/panic-no-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/panic-no-issue.txt @@ -10,7 +10,7 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: TestGossipHandlesReplacedNode failed -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -35,15 +35,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

/cc @cockroachdb/idonotexistbecausethisisatest @@ -53,6 +52,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/panic-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/panic-related-issue.txt index c000253a2b2d..ad1496a25f4b 100644 --- a/pkg/cmd/internal/issues/testdata/post/panic-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/panic-related-issue.txt @@ -10,7 +10,7 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: TestGossipHandlesReplacedNode failed -storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.TestGossipHandlesReplacedNode [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Fatal error: @@ -35,15 +35,14 @@ logging something ```

+

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -59,6 +58,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestGossipHandlesReplacedNode+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0AFatal+error%3A%0A%0A%60%60%60%0Apanic%3A+something+bad+happened%3A%0A%0Afoo%0Abar%0A%0A%60%60%60%0AStack%3A+%0A%0A%60%60%60%0Agoroutine+12+%5Brunning%5D%3A%0A++doing+something%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3ELog+preceding+fatal+error%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A%60%60%60%0Alogging+something%0A%60%60%60%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestGossipHandlesReplacedNode.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestGossipHandlesReplacedNode+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt index 0e81034cde49..80bc56df36c1 100644 --- a/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-and-related-issue.txt @@ -5,7 +5,7 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] createComment owner=cockroachdb repo=cockroach issue=30: -sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Random syntax error: @@ -38,15 +38,14 @@ Schema: test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 --- FAIL: TestRandomSyntaxSQLSmith (300.69s) ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -61,6 +60,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt index 84b9e174c2f6..ccdb869f92c4 100644 --- a/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-matching-issue.txt @@ -5,7 +5,7 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "sql/tests: TestRandomSyntaxSQLSmith failed" -label:branch-release-0.1: [] createComment owner=cockroachdb repo=cockroach issue=30: -sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Random syntax error: @@ -38,15 +38,14 @@ Schema: test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 --- FAIL: TestRandomSyntaxSQLSmith (300.69s) ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

@@ -55,6 +54,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt index 0af1c1ad41b3..73e22f9169fe 100644 --- a/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-no-issue.txt @@ -10,7 +10,7 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel sql/tests: TestRandomSyntaxSQLSmith failed -sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Random syntax error: @@ -43,15 +43,14 @@ Schema: test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 --- FAIL: TestRandomSyntaxSQLSmith (300.69s) ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

/cc @cockroachdb/idonotexistbecausethisisatest @@ -61,6 +60,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=sql%2Ftests%3A+TestRandomSyntaxSQLSmith+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=sql%2Ftests%3A+TestRandomSyntaxSQLSmith+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt index 6e0cf1cd2260..f98810a4211d 100644 --- a/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/rsg-crash-related-issue.txt @@ -10,7 +10,7 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel sql/tests: TestRandomSyntaxSQLSmith failed -sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +sql/tests.TestRandomSyntaxSQLSmith [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): Random syntax error: @@ -43,15 +43,14 @@ Schema: test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTestRandomSyntaxSQLSmith460792454 --- FAIL: TestRandomSyntaxSQLSmith (300.69s) ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -67,6 +66,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=sql%2Ftests%3A+TestRandomSyntaxSQLSmith+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=sql%2Ftests.TestRandomSyntaxSQLSmith+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0ARandom+syntax+error%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A755%3A+Crash+detected%3A+server+panic%3A+pq%3A+internal+error%3A+something+bad%0A%60%60%60%0AQuery%3A%0A%0A%60%60%60%0A%09%09SELECT%0A%09%09%09foo%0A%09%09FROM%0A%09%09%09bar%0A%09%09LIMIT%0A%09%09%0933%3A%3A%3AINT8%3B%0A%60%60%60%0ASchema%3A%0A%0A%60%60%60%0A++++rsg_test.go%3A575%3A+To+reproduce%2C+use+schema%3A%0A++++rsg_test.go%3A577%3A+%0A++++++++%09CREATE+TABLE+table1+%28col1_0+BOOL%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A577%3A+%0A++++++++%0A++++++++CREATE+TYPE+greeting+AS+ENUM+%28%27hello%27%2C+%27howdy%27%2C+%27hi%27%2C+%27good+day%27%2C+%27morning%27%29%3B%0A++++++++%3B%0A++++rsg_test.go%3A579%3A+%0A++++rsg_test.go%3A580%3A+--+test+log+scope+end+--%0Atest+logs+left+over+in%3A+%2Fgo%2Fsrc%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fartifacts%2FlogTestRandomSyntaxSQLSmith460792454%0A---+FAIL%3A+TestRandomSyntaxSQLSmith+%28300.69s%29%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestRandomSyntaxSQLSmith.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=sql%2Ftests%3A+TestRandomSyntaxSQLSmith+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-and-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-and-related-issue.txt index a4f57fe00dbf..d868c42a5d56 100644 --- a/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-and-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-and-related-issue.txt @@ -5,21 +5,20 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: kv/splits/nodes=3/quiesce=true failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}] createComment owner=cockroachdb repo=cockroach issue=30: -storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) with [artifacts](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) with [artifacts](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` The test failed on branch=master, cloud=gce: ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -34,6 +33,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-issue.txt b/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-issue.txt index b2cebfb67d4a..c66297ccdd34 100644 --- a/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/with-artifacts-matching-issue.txt @@ -5,21 +5,20 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label: searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: kv/splits/nodes=3/quiesce=true failed" -label:branch-release-0.1: [] createComment owner=cockroachdb repo=cockroach issue=30: -storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) with [artifacts](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) with [artifacts](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` The test failed on branch=master, cloud=gce: ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

@@ -28,6 +27,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/with-artifacts-no-issue.txt b/pkg/cmd/internal/issues/testdata/post/with-artifacts-no-issue.txt index f744a7e8dcb2..372cf51fef4e 100644 --- a/pkg/cmd/internal/issues/testdata/post/with-artifacts-no-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/with-artifacts-no-issue.txt @@ -10,21 +10,20 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: kv/splits/nodes=3/quiesce=true failed -storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) with [artifacts](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) with [artifacts](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` The test failed on branch=master, cloud=gce: ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

/cc @cockroachdb/idonotexistbecausethisisatest @@ -34,6 +33,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+failed ---- ---- diff --git a/pkg/cmd/internal/issues/testdata/post/with-artifacts-related-issue.txt b/pkg/cmd/internal/issues/testdata/post/with-artifacts-related-issue.txt index de19952eca4b..a5984e196f78 100644 --- a/pkg/cmd/internal/issues/testdata/post/with-artifacts-related-issue.txt +++ b/pkg/cmd/internal/issues/testdata/post/with-artifacts-related-issue.txt @@ -10,21 +10,20 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel storage: kv/splits/nodes=3/quiesce=true failed -storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) with [artifacts](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): +storage.kv/splits/nodes=3/quiesce=true [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) with [artifacts](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=artifacts#/kv/splits/nodes=3/quiesce=true) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123): ``` The test failed on branch=master, cloud=gce: ``` +

Parameters: GOFLAGS=race +, ROACHTEST_cloud=gce +, TAGS=deadlock +

Help

See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM) -Parameters in this failure: - -- TAGS=deadlock - -- GOFLAGS=race

Same failure on other branches @@ -40,6 +39,6 @@ Parameters in this failure: -Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+failed +Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+with+%5Bartifacts%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dartifacts%23%2Fkv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0AThe+test+failed+on+branch%3Dmaster%2C+cloud%3Dgce%3A%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2Akv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+kv%2Fsplits%2Fnodes%3D3%2Fquiesce%3Dtrue+failed ---- ---- diff --git a/pkg/cmd/roachtest/BUILD.bazel b/pkg/cmd/roachtest/BUILD.bazel index 29d88e8a0bf0..633d79096a12 100644 --- a/pkg/cmd/roachtest/BUILD.bazel +++ b/pkg/cmd/roachtest/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "roachtest_lib", srcs = [ "cluster.go", + "github.go", "main.go", "monitor.go", "slack.go", @@ -26,6 +27,7 @@ go_library( "//pkg/internal/team", "//pkg/roachprod", "//pkg/roachprod/config", + "//pkg/roachprod/errors", "//pkg/roachprod/install", "//pkg/roachprod/logger", "//pkg/roachprod/vm", @@ -62,6 +64,7 @@ go_test( size = "small", srcs = [ "cluster_test.go", + "github_test.go", "main_test.go", "test_registry_test.go", "test_test.go", @@ -75,6 +78,7 @@ go_test( "//pkg/cmd/roachtest/test", "//pkg/internal/team", "//pkg/roachprod/logger", + "//pkg/roachprod/vm", "//pkg/testutils", "//pkg/util/quotapool", "//pkg/util/stop", diff --git a/pkg/cmd/roachtest/cluster.go b/pkg/cmd/roachtest/cluster.go index 334b64e944c0..444d05c5bd6c 100644 --- a/pkg/cmd/roachtest/cluster.go +++ b/pkg/cmd/roachtest/cluster.go @@ -792,9 +792,9 @@ func (f *clusterFactory) clusterMock(cfg clusterConfig) *clusterImpl { // NOTE: setTest() needs to be called before a test can use this cluster. func (f *clusterFactory) newCluster( ctx context.Context, cfg clusterConfig, setStatus func(string), teeOpt logger.TeeOptType, -) (*clusterImpl, error) { +) (*clusterImpl, *vm.CreateOpts, error) { if ctx.Err() != nil { - return nil, errors.Wrap(ctx.Err(), "newCluster") + return nil, nil, errors.Wrap(ctx.Err(), "newCluster") } if overrideFlagset != nil && overrideFlagset.Changed("nodes") { @@ -805,9 +805,9 @@ func (f *clusterFactory) newCluster( // For tests, use a mock cluster. c := f.clusterMock(cfg) if err := f.r.registerCluster(c); err != nil { - return nil, err + return nil, nil, err } - return c, nil + return c, nil, nil } if cfg.localCluster { @@ -827,7 +827,10 @@ func (f *clusterFactory) newCluster( // that each create attempt gets a unique cluster name. createVMOpts, providerOpts, err := cfg.spec.RoachprodOpts("", cfg.useIOBarrier) if err != nil { - return nil, err + // We must release the allocation because cluster creation is not possible at this point. + cfg.alloc.Release() + + return nil, nil, err } if cfg.spec.Cloud != spec.Local { providerOptsContainer.SetProviderOpts(cfg.spec.Cloud, providerOpts) @@ -875,18 +878,18 @@ func (f *clusterFactory) newCluster( err = roachprod.Create(ctx, l, cfg.username, cfg.spec.NodeCount, createVMOpts, providerOptsContainer) if err == nil { if err := f.r.registerCluster(c); err != nil { - return nil, err + return nil, nil, err } c.status("idle") l.Close() - return c, nil + return c, &createVMOpts, nil } if errors.HasType(err, (*roachprod.ClusterAlreadyExistsError)(nil)) { // If the cluster couldn't be created because it existed already, bail. // In reality when this is hit is when running with the `local` flag // or a destroy from the previous iteration failed. - return nil, err + return nil, nil, err } l.PrintfCtx(ctx, "cluster creation failed, cleaning up in case it was partially created: %s", err) @@ -898,7 +901,7 @@ func (f *clusterFactory) newCluster( if i >= maxAttempts { // Here we have to release the alloc, as we are giving up. cfg.alloc.Release() - return nil, err + return nil, nil, err } // Try again to create the cluster. } @@ -1076,13 +1079,13 @@ func (c *clusterImpl) Node(i int) option.NodeListOption { // FetchLogs downloads the logs from the cluster using `roachprod get`. // The logs will be placed in the test's artifacts dir. -func (c *clusterImpl) FetchLogs(ctx context.Context, t test.Test) error { +func (c *clusterImpl) FetchLogs(ctx context.Context, l *logger.Logger) error { if c.spec.NodeCount == 0 { // No nodes can happen during unit tests and implies nothing to do. return nil } - t.L().Printf("fetching logs\n") + l.Printf("fetching logs\n") c.status("fetching logs") // Don't hang forever if we can't fetch the logs. @@ -1093,14 +1096,14 @@ func (c *clusterImpl) FetchLogs(ctx context.Context, t test.Test) error { } if err := c.Get(ctx, c.l, "logs" /* src */, path /* dest */); err != nil { - t.L().Printf("failed to fetch logs: %v", err) + l.Printf("failed to fetch logs: %v", err) if ctx.Err() != nil { return errors.Wrap(err, "cluster.FetchLogs") } } if err := c.RunE(ctx, c.All(), "mkdir -p logs/redacted && ./cockroach debug merge-logs --redact logs/*.log > logs/redacted/combined.log"); err != nil { - t.L().Printf("failed to redact logs: %v", err) + l.Printf("failed to redact logs: %v", err) if ctx.Err() != nil { return err } @@ -1152,17 +1155,17 @@ func (c *clusterImpl) CopyRoachprodState(ctx context.Context) error { // the first available node. They can be visualized via: // // `COCKROACH_DEBUG_TS_IMPORT_FILE=tsdump.gob ./cockroach start-single-node --insecure --store=$(mktemp -d)` -func (c *clusterImpl) FetchTimeseriesData(ctx context.Context, t test.Test) error { +func (c *clusterImpl) FetchTimeseriesData(ctx context.Context, l *logger.Logger) error { return contextutil.RunWithTimeout(ctx, "fetch tsdata", 5*time.Minute, func(ctx context.Context) error { node := 1 for ; node <= c.spec.NodeCount; node++ { - db, err := c.ConnE(ctx, t.L(), node) + db, err := c.ConnE(ctx, l, node) if err == nil { err = db.Ping() db.Close() } if err != nil { - t.L().Printf("node %d not responding to SQL, trying next one", node) + l.Printf("node %d not responding to SQL, trying next one", node) continue } break @@ -1181,7 +1184,7 @@ func (c *clusterImpl) FetchTimeseriesData(ctx context.Context, t test.Test) erro ); err != nil { return errors.Wrap(err, "cluster.FetchTimeseriesData") } - db, err := c.ConnE(ctx, t.L(), node) + db, err := c.ConnE(ctx, l, node) if err != nil { return err } @@ -1217,13 +1220,13 @@ COCKROACH_DEBUG_TS_IMPORT_FILE=tsdump.gob cockroach start-single-node --insecure // FetchDebugZip downloads the debug zip from the cluster using `roachprod ssh`. // The logs will be placed in the test's artifacts dir. -func (c *clusterImpl) FetchDebugZip(ctx context.Context, t test.Test) error { +func (c *clusterImpl) FetchDebugZip(ctx context.Context, l *logger.Logger) error { if c.spec.NodeCount == 0 { // No nodes can happen during unit tests and implies nothing to do. return nil } - t.L().Printf("fetching debug zip\n") + l.Printf("fetching debug zip\n") c.status("fetching debug zip") // Don't hang forever if we can't fetch the debug zip. @@ -1244,7 +1247,7 @@ func (c *clusterImpl) FetchDebugZip(ctx context.Context, t test.Test) error { si := strconv.Itoa(i) cmd := []string{"./cockroach", "debug", "zip", "--exclude-files='*.log,*.txt,*.pprof'", "--url", "{pgurl:" + si + "}", zipName} if err := c.RunE(ctx, c.All(), cmd...); err != nil { - t.L().Printf("./cockroach debug zip failed: %v", err) + l.Printf("./cockroach debug zip failed: %v", err) if i < c.spec.NodeCount { continue } @@ -1319,7 +1322,7 @@ WHERE t.status NOT IN ('RANGE_CONSISTENT', 'RANGE_INDETERMINATE')`) } // FailOnReplicaDivergence fails the test if -// crdb_internal.check_consistency(true, '', '') indicates that any ranges' +// crdb_internal.check_consistency(true, ”, ”) indicates that any ranges' // replicas are inconsistent with each other. It uses the first node that // is up to run the query. func (c *clusterImpl) FailOnReplicaDivergence(ctx context.Context, t *testImpl) { @@ -1364,14 +1367,14 @@ func (c *clusterImpl) FailOnReplicaDivergence(ctx context.Context, t *testImpl) // FetchDmesg grabs the dmesg logs if possible. This requires being able to run // `sudo dmesg` on the remote nodes. -func (c *clusterImpl) FetchDmesg(ctx context.Context, t test.Test) error { +func (c *clusterImpl) FetchDmesg(ctx context.Context, l *logger.Logger) error { if c.spec.NodeCount == 0 || c.IsLocal() { // No nodes can happen during unit tests and implies nothing to do. // Also, don't grab dmesg on local runs. return nil } - t.L().Printf("fetching dmesg\n") + l.Printf("fetching dmesg\n") c.status("fetching dmesg") // Don't hang forever. @@ -1395,7 +1398,7 @@ func (c *clusterImpl) FetchDmesg(ctx context.Context, t test.Test) error { if result.Err != nil { // Store `Run` errors to return later (after copying files from successful nodes). combinedDmesgError = errors.CombineErrors(combinedDmesgError, result.Err) - t.L().Printf("running dmesg failed on node %d: %v", result.Node, result.Err) + l.Printf("running dmesg failed on node %d: %v", result.Node, result.Err) } else { // Only run `Get` on successful nodes to avoid pseudo-failure on `Get` caused by an earlier failure on `Run`. successfulNodes = append(successfulNodes, int(result.Node)) @@ -1404,7 +1407,7 @@ func (c *clusterImpl) FetchDmesg(ctx context.Context, t test.Test) error { // Get dmesg files from successful nodes only. if err := c.Get(ctx, c.l, name /* src */, path /* dest */, successfulNodes); err != nil { - t.L().Printf("getting dmesg files failed: %v", err) + l.Printf("getting dmesg files failed: %v", err) return errors.Wrap(err, "cluster.FetchDmesg") } @@ -1415,14 +1418,14 @@ func (c *clusterImpl) FetchDmesg(ctx context.Context, t test.Test) error { // FetchJournalctl grabs the journalctl logs if possible. This requires being // able to run `sudo journalctl` on the remote nodes. -func (c *clusterImpl) FetchJournalctl(ctx context.Context, t test.Test) error { +func (c *clusterImpl) FetchJournalctl(ctx context.Context, l *logger.Logger) error { if c.spec.NodeCount == 0 || c.IsLocal() { // No nodes can happen during unit tests and implies nothing to do. // Also, don't grab journalctl on local runs. return nil } - t.L().Printf("fetching journalctl\n") + l.Printf("fetching journalctl\n") c.status("fetching journalctl") // Don't hang forever. @@ -1446,7 +1449,7 @@ func (c *clusterImpl) FetchJournalctl(ctx context.Context, t test.Test) error { if result.Err != nil { // Store `Run` errors to return later (after copying files from successful nodes). combinedJournalctlError = errors.CombineErrors(combinedJournalctlError, result.Err) - t.L().Printf("running journalctl failed on node %d: %v", result.Node, result.Err) + l.Printf("running journalctl failed on node %d: %v", result.Node, result.Err) } else { // Only run `Get` on successful nodes to avoid pseudo-failure on `Get` caused by an earlier failure on `Run`. successfulNodes = append(successfulNodes, int(result.Node)) @@ -1455,7 +1458,7 @@ func (c *clusterImpl) FetchJournalctl(ctx context.Context, t test.Test) error { // Get files from successful nodes only. if err := c.Get(ctx, c.l, name /* src */, path /* dest */, successfulNodes); err != nil { - t.L().Printf("getting files failed: %v", err) + l.Printf("getting files failed: %v", err) return errors.Wrap(err, "cluster.FetchJournalctl") } @@ -1465,7 +1468,7 @@ func (c *clusterImpl) FetchJournalctl(ctx context.Context, t test.Test) error { } // FetchCores fetches any core files on the cluster. -func (c *clusterImpl) FetchCores(ctx context.Context, t test.Test) error { +func (c *clusterImpl) FetchCores(ctx context.Context, l *logger.Logger) error { if c.spec.NodeCount == 0 || c.IsLocal() { // No nodes can happen during unit tests and implies nothing to do. // Also, don't grab dmesg on local runs. @@ -1477,11 +1480,11 @@ func (c *clusterImpl) FetchCores(ctx context.Context, t test.Test) error { // from having the cores, but we should push them straight into a temp // bucket on S3 instead. OTOH, the ROI of this may be low; I don't know // of a recent example where we've wanted the Core dumps. - t.L().Printf("skipped fetching cores\n") + l.Printf("skipped fetching cores\n") return nil } - t.L().Printf("fetching cores\n") + l.Printf("fetching cores\n") c.status("fetching cores") // Don't hang forever. The core files can be large, so we give a generous diff --git a/pkg/cmd/roachtest/cluster/BUILD.bazel b/pkg/cmd/roachtest/cluster/BUILD.bazel index b74f36a4689f..5e9c10ca7fb3 100644 --- a/pkg/cmd/roachtest/cluster/BUILD.bazel +++ b/pkg/cmd/roachtest/cluster/BUILD.bazel @@ -12,7 +12,6 @@ go_library( deps = [ "//pkg/cmd/roachtest/option", "//pkg/cmd/roachtest/spec", - "//pkg/cmd/roachtest/test", "//pkg/roachprod/install", "//pkg/roachprod/logger", "@com_github_cockroachdb_errors//:errors", diff --git a/pkg/cmd/roachtest/cluster/cluster_interface.go b/pkg/cmd/roachtest/cluster/cluster_interface.go index 22b3af7e1b4e..fcfbdb7a26c4 100644 --- a/pkg/cmd/roachtest/cluster/cluster_interface.go +++ b/pkg/cmd/roachtest/cluster/cluster_interface.go @@ -17,7 +17,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" - "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/roachprod/logger" ) @@ -129,6 +128,7 @@ type Cluster interface { GitClone( ctx context.Context, l *logger.Logger, src, dest, branch string, node option.NodeListOption, ) error - FetchTimeseriesData(ctx context.Context, t test.Test) error + + FetchTimeseriesData(ctx context.Context, l *logger.Logger) error RefetchCertsFromNode(ctx context.Context, node int) error } diff --git a/pkg/cmd/roachtest/github.go b/pkg/cmd/roachtest/github.go new file mode 100644 index 000000000000..e9d019900f0a --- /dev/null +++ b/pkg/cmd/roachtest/github.go @@ -0,0 +1,179 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "context" + "fmt" + "os" + + "github.com/cockroachdb/cockroach/pkg/cmd/internal/issues" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/internal/team" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" + "github.com/cockroachdb/cockroach/pkg/roachprod/vm" +) + +type githubIssues struct { + disable bool + l *logger.Logger + cluster *clusterImpl + vmCreateOpts *vm.CreateOpts + issuePoster func(ctx context.Context, formatter issues.IssueFormatter, req issues.PostRequest) error + teamLoader func() (team.Map, error) +} + +type issueCategory int + +const ( + otherErr issueCategory = iota + clusterCreationErr + sshErr +) + +func newGithubIssues( + disable bool, c *clusterImpl, vmCreateOpts *vm.CreateOpts, l *logger.Logger, +) *githubIssues { + + return &githubIssues{ + disable: disable, + vmCreateOpts: vmCreateOpts, + cluster: c, + l: l, + issuePoster: issues.Post, + teamLoader: team.DefaultLoadTeams, + } +} + +func roachtestPrefix(p string) string { + return "ROACHTEST_" + p +} + +func (g *githubIssues) shouldPost(t test.Test) bool { + opts := issues.DefaultOptionsFromEnv() + return !g.disable && opts.CanPost() && + opts.IsReleaseBranch() && + t.Spec().(*registry.TestSpec).Run != nil && + // NB: check NodeCount > 0 to avoid posting issues from this pkg's unit tests. + t.Spec().(*registry.TestSpec).Cluster.NodeCount > 0 +} + +func (g *githubIssues) createPostRequest( + t test.Test, cat issueCategory, message string, +) issues.PostRequest { + var mention []string + var projColID int + + issueOwner := t.Spec().(*registry.TestSpec).Owner + issueName := t.Name() + + messagePrefix := "" + // Overrides to shield eng teams from potential flakes + if cat == clusterCreationErr { + issueOwner = registry.OwnerDevInf + issueName = "cluster_creation" + messagePrefix = fmt.Sprintf("test %s was skipped due to ", t.Name()) + } else if cat == sshErr { + issueOwner = registry.OwnerTestEng + issueName = "ssh_problem" + messagePrefix = fmt.Sprintf("test %s failed due to ", t.Name()) + } + + teams, err := g.teamLoader() + if err != nil { + t.Fatalf("could not load teams: %v", err) + } + + if sl, ok := teams.GetAliasesForPurpose(ownerToAlias(issueOwner), team.PurposeRoachtest); ok { + for _, alias := range sl { + mention = append(mention, "@"+string(alias)) + } + projColID = teams[sl[0]].TriageColumnID + } + + branch := os.Getenv("TC_BUILD_BRANCH") + if branch == "" { + branch = "" + } + + artifacts := fmt.Sprintf("/%s", t.Name()) + + // Issues posted from roachtest are identifiable as such and + // they are also release blockers (this label may be removed + // by a human upon closer investigation). + spec := t.Spec().(*registry.TestSpec) + labels := []string{"O-roachtest"} + if !spec.NonReleaseBlocker { + labels = append(labels, "release-blocker") + } + + clusterParams := map[string]string{ + roachtestPrefix("cloud"): spec.Cluster.Cloud, + roachtestPrefix("cpu"): fmt.Sprintf("%d", spec.Cluster.CPUs), + roachtestPrefix("ssd"): fmt.Sprintf("%d", spec.Cluster.SSDs), + } + + // These params can be probabilistically set, so we pass them here to + // show what their actual values are in the posted issue. + if g.vmCreateOpts != nil { + clusterParams[roachtestPrefix("fs")] = g.vmCreateOpts.SSDOpts.FileSystem + clusterParams[roachtestPrefix("localSSD")] = fmt.Sprintf("%v", g.vmCreateOpts.SSDOpts.UseLocalSSD) + } + + if g.cluster != nil { + clusterParams[roachtestPrefix("encrypted")] = fmt.Sprintf("%v", g.cluster.encAtRest) + } + + return issues.PostRequest{ + MentionOnCreate: mention, + ProjectColumnID: projColID, + PackageName: "roachtest", + TestName: issueName, + Message: messagePrefix + message, + Artifacts: artifacts, + ExtraLabels: labels, + ExtraParams: clusterParams, + HelpCommand: func(renderer *issues.Renderer) { + issues.HelpCommandAsLink( + "roachtest README", + "https://github.com/cockroachdb/cockroach/blob/master/pkg/cmd/roachtest/README.md", + )(renderer) + issues.HelpCommandAsLink( + "How To Investigate (internal)", + "https://cockroachlabs.atlassian.net/l/c/SSSBr8c7", + )(renderer) + }, + } +} + +func (g *githubIssues) MaybePost(t *testImpl, message string) error { + if !g.shouldPost(t) { + return nil + } + + cat := otherErr + + // Overrides to shield eng teams from potential flakes + firstFailure := t.firstFailure() + if failureContainsError(firstFailure, errClusterProvisioningFailed) { + cat = clusterCreationErr + } else if failureContainsError(firstFailure, rperrors.ErrSSH255) { + cat = sshErr + } + + return g.issuePoster( + context.Background(), + issues.UnitTestFormatter, + g.createPostRequest(t, cat, message), + ) +} diff --git a/pkg/cmd/roachtest/github_test.go b/pkg/cmd/roachtest/github_test.go new file mode 100644 index 000000000000..3342d136853b --- /dev/null +++ b/pkg/cmd/roachtest/github_test.go @@ -0,0 +1,224 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/internal/team" + "github.com/cockroachdb/cockroach/pkg/roachprod/vm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + teamsYaml = `cockroachdb/unowned: + aliases: + cockroachdb/rfc-prs: other + triage_column_id: 0 +cockroachdb/test-eng: + triage_column_id: 14041337 +cockroachdb/dev-inf: + triage_column_id: 10210759` + + validTeamsFn = func() (team.Map, error) { return loadYamlTeams(teamsYaml) } + invalidTeamsFn = func() (team.Map, error) { return loadYamlTeams("invalid yaml") } +) + +func loadYamlTeams(yaml string) (team.Map, error) { + return team.LoadTeams(strings.NewReader(yaml)) +} + +func prefixAll(params map[string]string) map[string]string { + updated := make(map[string]string) + + for k, v := range params { + updated[roachtestPrefix(k)] = v + } + + return updated +} + +func TestShouldPost(t *testing.T) { + testCases := []struct { + disableIssues bool + nodeCount int + envGithubAPIToken string + envTcBuildBranch string + expected bool + }{ + /* Cases 1 - 4 verify that issues are not posted if any of the relevant criteria checks fail */ + // disable + {true, 1, "token", "master", false}, + // nodeCount + {false, 0, "token", "master", false}, + // apiToken + {false, 1, "", "master", false}, + // branch + {false, 1, "token", "", false}, + {false, 1, "token", "master", true}, + } + + reg, _ := makeTestRegistry(spec.GCE, "", "", false) + for _, c := range testCases { + t.Setenv("GITHUB_API_TOKEN", c.envGithubAPIToken) + t.Setenv("TC_BUILD_BRANCH", c.envTcBuildBranch) + + clusterSpec := reg.MakeClusterSpec(c.nodeCount) + testSpec := ®istry.TestSpec{ + Name: "githubPost", + Owner: OwnerUnitTest, + Cluster: clusterSpec, + // `shouldPost` explicitly checks to ensure that the run function is defined + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {}, + } + + ti := &testImpl{ + spec: testSpec, + l: nilLogger(), + } + + github := &githubIssues{ + disable: c.disableIssues, + } + + require.Equal(t, c.expected, github.shouldPost(ti)) + } +} + +func TestCreatePostRequest(t *testing.T) { + testCases := []struct { + nonReleaseBlocker bool + clusterCreationFailed bool + loadTeamsFailed bool + localSSD bool + category issueCategory + expectedPost bool + expectedParams map[string]string + }{ + {true, false, false, false, otherErr, true, + prefixAll(map[string]string{ + "cloud": "gce", + "encrypted": "false", + "fs": "ext4", + "ssd": "0", + "cpu": "4", + "localSSD": "false", + }), + }, + {true, false, false, true, clusterCreationErr, true, + prefixAll(map[string]string{ + "cloud": "gce", + "encrypted": "false", + "fs": "ext4", + "ssd": "0", + "cpu": "4", + "localSSD": "true", + }), + }, + // Assert that release-blocker label exists when !nonReleaseBlocker + // Also ensure that in the event of a failed cluster creation, + // nil `vmOptions` and `clusterImpl` are not dereferenced + {false, true, false, false, sshErr, true, + prefixAll(map[string]string{ + "cloud": "gce", + "ssd": "0", + "cpu": "4", + }), + }, + //Simulate failure loading TEAMS.yaml + {true, false, true, false, otherErr, false, nil}, + } + + reg, _ := makeTestRegistry(spec.GCE, "", "", false) + + for _, c := range testCases { + clusterSpec := reg.MakeClusterSpec(1) + + testSpec := ®istry.TestSpec{ + Name: "github_test", + Owner: OwnerUnitTest, + Cluster: clusterSpec, + NonReleaseBlocker: c.nonReleaseBlocker, + } + + ti := &testImpl{ + spec: testSpec, + l: nilLogger(), + } + + testClusterImpl := &clusterImpl{spec: clusterSpec} + vo := vm.DefaultCreateOpts() + vmOpts := &vo + + if c.clusterCreationFailed { + testClusterImpl = nil + vmOpts = nil + } else if !c.localSSD { + // The default is true set in `vm.DefaultCreateOpts` + vmOpts.SSDOpts.UseLocalSSD = false + } + + teamLoadFn := validTeamsFn + + if c.loadTeamsFailed { + teamLoadFn = invalidTeamsFn + } + + github := &githubIssues{ + vmCreateOpts: vmOpts, + cluster: testClusterImpl, + l: nilLogger(), + teamLoader: teamLoadFn, + } + + if c.loadTeamsFailed { + // Assert that if TEAMS.yaml cannot be loaded then function panics. + assert.Panics(t, func() { github.createPostRequest(ti, c.category, "message") }) + } else { + req := github.createPostRequest(ti, c.category, "message") + + if c.expectedParams != nil { + require.Equal(t, c.expectedParams, req.ExtraParams) + } + + require.Contains(t, req.ExtraLabels, "O-roachtest") + + if !c.nonReleaseBlocker { + require.Contains(t, req.ExtraLabels, "release-blocker") + } + + expectedTeam := "@cockroachdb/unowned" + expectedName := "github_test" + expectedMessagePrefix := "" + + if c.category == clusterCreationErr { + expectedTeam = "@cockroachdb/dev-inf" + expectedName = "cluster_creation" + expectedMessagePrefix = "test github_test was skipped due to " + } else if c.category == sshErr { + expectedTeam = "@cockroachdb/test-eng" + expectedName = "ssh_problem" + expectedMessagePrefix = "test github_test failed due to " + } + + require.Contains(t, req.MentionOnCreate, expectedTeam) + require.Equal(t, expectedName, req.TestName) + require.True(t, strings.HasPrefix(req.Message, expectedMessagePrefix), req.Message) + } + } +} diff --git a/pkg/cmd/roachtest/main.go b/pkg/cmd/roachtest/main.go index 5b33569e8c0d..20ef7d4fe5df 100644 --- a/pkg/cmd/roachtest/main.go +++ b/pkg/cmd/roachtest/main.go @@ -335,7 +335,7 @@ runner itself. if errors.Is(err, errTestsFailed) { code = ExitCodeTestsFailed } - if errors.Is(err, errClusterProvisioningFailed) { + if errors.Is(err, errSomeClusterProvisioningFailed) { code = ExitCodeClusterProvisioningFailed } // Cobra has already printed the error message. @@ -373,8 +373,15 @@ func runTests(register func(registry.Registry), cfg cliCfg) error { filter := registry.NewTestFilter(cfg.args) clusterType := roachprodCluster + bindTo := "" if local { clusterType = localCluster + + // This will suppress the annoying "Allow incoming network connections" popup from + // OSX when running a roachtest + bindTo = "localhost" + + fmt.Printf("--local specified. Binding http listener to localhost only") if cfg.parallelism != 1 { fmt.Printf("--local specified. Overriding --parallelism to 1.\n") cfg.parallelism = 1 @@ -389,7 +396,7 @@ func runTests(register func(registry.Registry), cfg cliCfg) error { keepClustersOnTestFailure: cfg.debugEnabled, clusterID: cfg.clusterID, } - if err := runner.runHTTPServer(cfg.httpPort, os.Stdout); err != nil { + if err := runner.runHTTPServer(cfg.httpPort, os.Stdout, bindTo); err != nil { return err } diff --git a/pkg/cmd/roachtest/test/test_interface.go b/pkg/cmd/roachtest/test/test_interface.go index d0ada18dccbb..7dedf4be366f 100644 --- a/pkg/cmd/roachtest/test/test_interface.go +++ b/pkg/cmd/roachtest/test/test_interface.go @@ -34,6 +34,7 @@ type Test interface { VersionsBinaryOverride() map[string]string Skip(args ...interface{}) Skipf(format string, args ...interface{}) + Error(args ...interface{}) Errorf(string, ...interface{}) FailNow() Fatal(args ...interface{}) diff --git a/pkg/cmd/roachtest/test_impl.go b/pkg/cmd/roachtest/test_impl.go index 374676518893..a7b9a3a9b141 100644 --- a/pkg/cmd/roachtest/test_impl.go +++ b/pkg/cmd/roachtest/test_impl.go @@ -11,13 +11,10 @@ package main import ( - "bytes" "context" "fmt" "io" - // For the debug http handlers. - _ "net/http/pprof" - "runtime" + "os" "strings" "time" @@ -27,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/version" + "github.com/cockroachdb/errors" "github.com/petermattis/goid" ) @@ -41,6 +39,18 @@ type testStatus struct { progress float64 } +// Holds all error information from a single invocation of t.{Fatal,Error}{,f} to +// preserve any structured errors +// e.g. t.Fatalf("foo %s %s %s", "hello", err1, err2) would mean that +// failure.errors == [err1, err2], with all args (including the non error "hello") +// being captured in the squashedErr +type failure struct { + // This is the single error created from variadic args passed to t.{Fatal,Error}{,f} + squashedErr error + // errors are all the `errors` present in the variadic args + errors []error +} + type testImpl struct { spec *registry.TestSpec @@ -69,23 +79,26 @@ type testImpl struct { mu struct { syncutil.RWMutex - done bool - failed bool - timeout bool // if failed == true, this indicates whether the test timed out + done bool + // cancel, if set, is called from the t.Fatal() family of functions when the // test is being marked as failed (i.e. when the failed field above is also // set). This is used to cancel the context passed to t.spec.Run(), so async // test goroutines can be notified. - cancel func() - failLoc struct { - file string - line int - } - failureMsg string + cancel func() + + // failures added via addFailures, in order + // A test can have multiple calls to t.Fail()/Error(), with each call + // referencing 0+ errors. failure captures all the errors + failures []failure + // status is a map from goroutine id to status set by that goroutine. A // special goroutine is indicated by runnerID; that one provides the test's // "main status". status map[int64]testStatus + + // TODO(test-eng): this should just be an in-mem (ring) buffer attached to + // `t.L()`. output []byte } // Map from version to path to the cockroach binary to be used when @@ -97,6 +110,10 @@ type testImpl struct { versionsBinaryOverride map[string]string } +func newFailure(squashedErr error, errs []error) failure { + return failure{squashedErr: squashedErr, errors: errs} +} + // BuildVersion exposes the build version of the cluster // in this test. func (t *testImpl) BuildVersion() *version.Version { @@ -235,6 +252,17 @@ func (t *testImpl) Skipf(format string, args ...interface{}) { panic(errTestFatal) } +// collectErrors extracts any arg that is an error +func collectErrors(args []interface{}) []error { + var errs []error + for _, a := range args { + if err, ok := a.(error); ok { + errs = append(errs, err) + } + } + return errs +} + // Fatal marks the test as failed, prints the args to t.L(), and calls // panic(errTestFatal). It can be called multiple times. // @@ -244,147 +272,88 @@ func (t *testImpl) Skipf(format string, args ...interface{}) { // ATTENTION: Since this calls panic(errTestFatal), it should only be called // from a test's closure. The test runner itself should never call this. func (t *testImpl) Fatal(args ...interface{}) { - t.markFailedInner("" /* format */, args...) + t.addFailure("", args...) panic(errTestFatal) } // Fatalf is like Fatal, but takes a format string. func (t *testImpl) Fatalf(format string, args ...interface{}) { - t.markFailedInner(format, args...) + t.addFailure(format, args...) panic(errTestFatal) } // FailNow implements the TestingT interface. func (t *testImpl) FailNow() { - t.Fatal() + t.addFailure("FailNow called") + panic(errTestFatal) } -// Errorf implements the TestingT interface. -func (t *testImpl) Errorf(format string, args ...interface{}) { - t.markFailedInner(format, args...) +// Error implements the TestingT interface +func (t *testImpl) Error(args ...interface{}) { + t.addFailure("", args...) } -func (t *testImpl) markFailedInner(format string, args ...interface{}) { - // Skip two frames: our own and the caller. - if format != "" { - t.printfAndFail(2 /* skip */, format, args...) - } else { - t.printAndFail(2 /* skip */, args...) - } +// Errorf implements the TestingT interface. +func (t *testImpl) Errorf(format string, args ...interface{}) { + t.addFailure(format, args...) } -func (t *testImpl) printAndFail(skip int, args ...interface{}) { - var msg string - if len(args) == 1 { - // If we were passed only an error, then format it with "%+v" in order to - // get any stack traces. - if err, ok := args[0].(error); ok { - msg = fmt.Sprintf("%+v", err) +// We take the first error from each failure which is the +// "squashed" error that contains all information of a failure +func formatFailure(b *strings.Builder, reportFailures ...failure) { + for i, failure := range reportFailures { + if i > 0 { + fmt.Fprintln(b) } + file, line, fn, ok := errors.GetOneLineSource(failure.squashedErr) + if !ok { + file, line, fn = "", 0, "unknown" + } + fmt.Fprintf(b, "(%s:%d).%s: %v", file, line, fn, failure.squashedErr) } - if msg == "" { - msg = fmt.Sprint(args...) - } - t.failWithMsg(t.decorate(skip+1, msg)) } -func (t *testImpl) printfAndFail(skip int, format string, args ...interface{}) { +func (t *testImpl) addFailure(format string, args ...interface{}) { if format == "" { - panic(fmt.Sprintf("invalid empty format. args: %s", args)) + format = strings.Repeat(" %v", len(args))[1:] } - t.failWithMsg(t.decorate(skip+1, fmt.Sprintf(format, args...))) -} + reportFailure := newFailure(errors.NewWithDepthf(2, format, args...), collectErrors(args)) -func (t *testImpl) failWithMsg(msg string) { t.mu.Lock() defer t.mu.Unlock() - prefix := "" - if t.mu.failed { - prefix = "[not the first failure] " - // NB: the first failure is not always the relevant one due to: - // https://github.com/cockroachdb/cockroach/issues/44436 - // - // So we chain all failures together in the order in which we see - // them. - msg = "\n" + msg + t.mu.failures = append(t.mu.failures, reportFailure) + + var b strings.Builder + formatFailure(&b, reportFailure) + msg := b.String() + + t.L().Printf("test failure #%d: %s", len(t.mu.failures), msg) + // Also dump the verbose error (incl. all stack traces) to a log file, in case + // we need it. The stacks are sometimes helpful, but we don't want them in the + // main log as they are highly verbose. + { + cl, err := t.L().ChildLogger( + fmt.Sprintf("failure_%d", len(t.mu.failures)), + logger.QuietStderr, logger.QuietStdout, + ) + if err == nil { + // We don't actually log through this logger since it adds an unrelated + // file:line caller (namely ours). The error already has stack traces + // so it's better to write only it to the file to avoid confusion. + path := cl.File.Name() + cl.Close() // we just wanted the filename + _ = os.WriteFile(path, []byte(fmt.Sprintf("%+v", reportFailure.squashedErr)), 0644) + } } - t.L().Printf("%stest failure: %s", prefix, msg) - t.mu.failed = true - t.mu.failureMsg += msg t.mu.output = append(t.mu.output, msg...) + t.mu.output = append(t.mu.output, '\n') if t.mu.cancel != nil { t.mu.cancel() } } -// Args: -// skip: The number of stack frames to exclude from the result. 0 means that -// the caller will be the first frame identified. 1 means the caller's caller -// will be the first, etc. -func (t *testImpl) decorate(skip int, s string) string { - // Skip two extra frames to account for this function and runtime.Callers - // itself. - var pc [50]uintptr - n := runtime.Callers(2+skip, pc[:]) - if n == 0 { - panic("zero callers found") - } - - buf := new(bytes.Buffer) - frames := runtime.CallersFrames(pc[:n]) - sep := "\t" - runnerFound := false - for { - if runnerFound { - break - } - - frame, more := frames.Next() - if !more { - break - } - if frame.Function == t.runner { - runnerFound = true - - // Handle the special case of the runner function being the caller of - // t.Fatal(). In that case, that's the line to be used for issue creation. - if t.mu.failLoc.file == "" { - t.mu.failLoc.file = frame.File - t.mu.failLoc.line = frame.Line - } - } - if !t.mu.failed && !runnerFound { - // Keep track of the highest stack frame that is lower than the t.runner - // stack frame. This is used to determine the author of that line of code - // and issue assignment. - t.mu.failLoc.file = frame.File - t.mu.failLoc.line = frame.Line - } - file := frame.File - if index := strings.LastIndexByte(file, '/'); index >= 0 { - file = file[index+1:] - } - fmt.Fprintf(buf, "%s%s:%d", sep, file, frame.Line) - sep = "," - } - buf.WriteString(": ") - - lines := strings.Split(s, "\n") - if l := len(lines); l > 1 && lines[l-1] == "" { - lines = lines[:l-1] - } - for i, line := range lines { - if i > 0 { - buf.WriteString("\n\t\t") - } - buf.WriteString(line) - } - buf.WriteByte('\n') - return buf.String() -} - func (t *testImpl) duration() time.Duration { return t.end.Sub(t.start) } @@ -392,13 +361,39 @@ func (t *testImpl) duration() time.Duration { func (t *testImpl) Failed() bool { t.mu.RLock() defer t.mu.RUnlock() - return t.mu.failed + return t.failedRLocked() +} + +func (t *testImpl) failedRLocked() bool { + return len(t.mu.failures) > 0 +} + +func (t *testImpl) firstFailure() failure { + t.mu.RLock() + defer t.mu.RUnlock() + if len(t.mu.failures) <= 0 { + return failure{} + } + return t.mu.failures[0] } -func (t *testImpl) FailureMsg() string { +func (t *testImpl) failureMsg() string { t.mu.RLock() defer t.mu.RUnlock() - return t.mu.failureMsg + var b strings.Builder + formatFailure(&b, t.mu.failures...) + return b.String() +} + +// failureContainsError returns true if any of the errors in a given failure +// matches the reference error +func failureContainsError(f failure, refError error) bool { + for _, err := range f.errors { + if errors.Is(err, refError) { + return true + } + } + return errors.Is(f.squashedErr, refError) } func (t *testImpl) ArtifactsDir() string { diff --git a/pkg/cmd/roachtest/test_runner.go b/pkg/cmd/roachtest/test_runner.go index 20e4c1a2e001..e8bd89de3a7c 100644 --- a/pkg/cmd/roachtest/test_runner.go +++ b/pkg/cmd/roachtest/test_runner.go @@ -29,14 +29,13 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/cockroach/pkg/cmd/internal/issues" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" - "github.com/cockroachdb/cockroach/pkg/internal/team" "github.com/cockroachdb/cockroach/pkg/roachprod/config" "github.com/cockroachdb/cockroach/pkg/roachprod/logger" + "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/quotapool" @@ -50,8 +49,15 @@ import ( "github.com/petermattis/goid" ) -var errTestsFailed = fmt.Errorf("some tests failed") -var errClusterProvisioningFailed = fmt.Errorf("some clusters could not be created") +var ( + errTestsFailed = fmt.Errorf("some tests failed") + + // reference error used by main.go at the end of a run of tests + errSomeClusterProvisioningFailed = fmt.Errorf("some clusters could not be created") + + // reference error used when cluster creation fails for a test + errClusterProvisioningFailed = fmt.Errorf("cluster could not be created") +) // testRunner runs tests. type testRunner struct { @@ -104,8 +110,10 @@ type testRunner struct { // newTestRunner constructs a testRunner. // // cr: The cluster registry with which all clusters will be registered. The -// caller provides this as the caller needs to be able to shut clusters down -// on Ctrl+C. +// +// caller provides this as the caller needs to be able to shut clusters down +// on Ctrl+C. +// // buildVersion: The version of the Cockroach binary against which tests will run. func newTestRunner( cr *clusterRegistry, stopper *stop.Stopper, buildVersion version.Version, @@ -169,9 +177,11 @@ type testOpts struct { // tests: The tests to run. // count: How many times to run each test selected by filter. // parallelism: How many workers to use for running tests. Tests are run -// locally (although generally they run against remote roachprod clusters). -// parallelism bounds the maximum number of tests that run concurrently. Note -// that the concurrency is also affected by cpuQuota. +// +// locally (although generally they run against remote roachprod clusters). +// parallelism bounds the maximum number of tests that run concurrently. Note +// that the concurrency is also affected by cpuQuota. +// // clusterOpt: Options for the clusters to use by tests. // lopt: Options for logging. func (r *testRunner) Run( @@ -304,7 +314,7 @@ func (r *testRunner) Run( if r.numClusterErrs > 0 { shout(ctx, l, lopt.stdout, "%d clusters could not be created", r.numClusterErrs) - return errClusterProvisioningFailed + return errSomeClusterProvisioningFailed } if len(r.status.fail) > 0 { @@ -343,7 +353,7 @@ func defaultClusterAllocator( alloc *quotapool.IntAlloc, artifactsDir string, wStatus *workerStatus, - ) (*clusterImpl, error) { + ) (*clusterImpl, *vm.CreateOpts, error) { wStatus.SetStatus("creating cluster") defer wStatus.SetStatus("") @@ -353,7 +363,7 @@ func defaultClusterAllocator( logPath := filepath.Join(artifactsDir, runnerLogsDir, "cluster-create", existingClusterName+".log") clusterL, err := logger.RootLogger(logPath, lopt.tee) if err != nil { - return nil, err + return nil, nil, err } defer clusterL.Close() opt := attachOpt{ @@ -362,7 +372,8 @@ func defaultClusterAllocator( skipWipe: r.config.skipClusterWipeOnAttach, } lopt.l.PrintfCtx(ctx, "Attaching to existing cluster %s for test %s", existingClusterName, t.Name) - return attachToExistingCluster(ctx, existingClusterName, clusterL, t.Cluster, opt, r.cr) + c, err := attachToExistingCluster(ctx, existingClusterName, clusterL, t.Cluster, opt, r.cr) + return c, nil, err } lopt.l.PrintfCtx(ctx, "Creating new cluster for test %s: %s", t.Name, t.Cluster) @@ -384,7 +395,7 @@ type clusterAllocatorFn func( alloc *quotapool.IntAlloc, artifactsDir string, wStatus *workerStatus, -) (*clusterImpl, error) +) (*clusterImpl, *vm.CreateOpts, error) // runWorker runs tests in a loop until work is exhausted. // @@ -406,11 +417,17 @@ type clusterAllocatorFn func( // Args: // name: The worker's name, to be used as a prefix for log messages. // artifactsRootDir: The artifacts dir. Each test's logs are going to be under a -// run_ dir. If empty, test log files will not be created. +// +// run_ dir. If empty, test log files will not be created. +// // literalArtifactsDir: The literal on-agent path where artifacts are stored. -// Only used for teamcity[publishArtifacts] messages. +// +// Only used for teamcity[publishArtifacts] messages. +// // stdout: The Writer to use for messages that need to go to stdout (e.g. the -// "=== RUN" and "--- FAIL" lines). +// +// "=== RUN" and "--- FAIL" lines). +// // teeOpt: The teeing option for future test loggers. // l: The logger to use for more verbose messages. func (r *testRunner) runWorker( @@ -532,35 +549,38 @@ func (r *testRunner) runWorker( } } var clusterCreateErr error + var vmCreateOpts *vm.CreateOpts if !testToRun.canReuseCluster { // Create a new cluster if can't reuse or reuse attempt failed. // N.B. non-reusable cluster would have been destroyed above. wStatus.SetTest(nil /* test */, testToRun) wStatus.SetStatus("creating cluster") - c, clusterCreateErr = allocateCluster(ctx, testToRun.spec, testToRun.alloc, artifactsRootDir, wStatus) + c, vmCreateOpts, clusterCreateErr = allocateCluster(ctx, testToRun.spec, testToRun.alloc, artifactsRootDir, wStatus) if clusterCreateErr != nil { + clusterCreateErr = errors.Mark(clusterCreateErr, errClusterProvisioningFailed) atomic.AddInt32(&r.numClusterErrs, 1) shout(ctx, l, stdout, "Unable to create (or reuse) cluster for test %s due to: %s.", testToRun.spec.Name, clusterCreateErr) } } - // Prepare the test's logger. - logPath := "" - var artifactsDir string - var artifactsSpec string - if artifactsRootDir != "" { - escapedTestName := teamCityNameEscape(testToRun.spec.Name) - runSuffix := "run_" + strconv.Itoa(testToRun.runNum) - - artifactsDir = filepath.Join(filepath.Join(artifactsRootDir, escapedTestName), runSuffix) - logPath = filepath.Join(artifactsDir, "test.log") - - // Map artifacts/TestFoo/run_?/** => TestFoo/run_?/**, i.e. collect the artifacts - // for this test exactly as they are laid out on disk (when the time - // comes). - artifactsSpec = fmt.Sprintf("%s/%s/** => %s/%s", filepath.Join(literalArtifactsDir, escapedTestName), runSuffix, escapedTestName, runSuffix) + // Prepare the test's logger. Always set this up with real files, using a + // temp dir if necessary. This simplifies testing. + if artifactsRootDir == "" { + artifactsRootDir, _ = os.MkdirTemp("", "roachtest-logger") } + + escapedTestName := teamCityNameEscape(testToRun.spec.Name) + runSuffix := "run_" + strconv.Itoa(testToRun.runNum) + + artifactsDir := filepath.Join(filepath.Join(artifactsRootDir, escapedTestName), runSuffix) + logPath := filepath.Join(artifactsDir, "test.log") + + // Map artifacts/TestFoo/run_?/** => TestFoo/run_?/**, i.e. collect the artifacts + // for this test exactly as they are laid out on disk (when the time + // comes). + artifactsSpec := fmt.Sprintf("%s/%s/** => %s/%s", filepath.Join(literalArtifactsDir, escapedTestName), runSuffix, escapedTestName, runSuffix) + testL, err := logger.RootLogger(logPath, teeOpt) if err != nil { return err @@ -578,24 +598,19 @@ func (r *testRunner) runWorker( // Now run the test. l.PrintfCtx(ctx, "starting test: %s:%d", testToRun.spec.Name, testToRun.runNum) + github := newGithubIssues(r.config.disableIssue, c, vmCreateOpts, l) + if clusterCreateErr != nil { // N.B. cluster creation must have failed... // We don't want to prematurely abort the test suite since it's likely a transient issue. // Instead, let's report an infrastructure issue, mark the test as failed and continue with the next test. - // Note, we fake the test name so that all cluster creation errors are posted to the same github issue. - oldName := t.spec.Name - oldOwner := t.spec.Owner + // Generate failure reason and mark the test failed to preclude fetching (cluster) artifacts. - t.printAndFail(0, clusterCreateErr) - issueOutput := "test %s was skipped due to %s" - issueOutput = fmt.Sprintf(issueOutput, oldName, t.FailureMsg()) + t.Error(clusterCreateErr) // N.B. issue title is of the form "roachtest: ${t.spec.Name} failed" (see UnitTestFormatter). - t.spec.Name = "cluster_creation" - t.spec.Owner = registry.OwnerDevInf - r.maybePostGithubIssue(ctx, l, t, stdout, issueOutput) - // Restore test name and owner. - t.spec.Name = oldName - t.spec.Owner = oldOwner + if err := github.MaybePost(t, t.failureMsg()); err != nil { + shout(ctx, l, stdout, "failed to post issue: %s", err) + } } else { // Tell the cluster that, from now on, it will be run "on behalf of this // test". @@ -618,14 +633,14 @@ func (r *testRunner) runWorker( wStatus.SetTest(t, testToRun) wStatus.SetStatus("running test") - err = r.runTest(ctx, t, testToRun.runNum, testToRun.runCount, c, stdout, testL) + err = r.runTest(ctx, t, testToRun.runNum, testToRun.runCount, c, stdout, testL, github) } if err != nil { shout(ctx, l, stdout, "test returned error: %s: %s", t.Name(), err) // Mark the test as failed if it isn't already. if !t.Failed() { - t.printAndFail(0 /* skip */, err) + t.Error(err) } } else { msg := "test passed: %s (run %d)" @@ -641,7 +656,7 @@ func (r *testRunner) runWorker( if err != nil { failureMsg += fmt.Sprintf("%+v", err) } else { - failureMsg += t.FailureMsg() + failureMsg += t.failureMsg() } if c != nil { if debug { @@ -721,7 +736,8 @@ func allStacks() []byte { // // Args: // c: The cluster on which the test will run. runTest() does not wipe or destroy -// the cluster. +// +// the cluster. func (r *testRunner) runTest( ctx context.Context, t *testImpl, @@ -730,6 +746,7 @@ func (r *testRunner) runTest( c *clusterImpl, stdout io.Writer, l *logger.Logger, + github *githubIssues, ) error { if t.Spec().(*registry.TestSpec).Skip != "" { return fmt.Errorf("can't run skipped test: %s: %s", t.Name(), t.Spec().(*registry.TestSpec).Skip) @@ -762,10 +779,7 @@ func (r *testRunner) runTest( // goroutine accidentally ends up calling t.Fatal; the test runs in a // different goroutine. if err := recover(); err != nil && err != errTestFatal { - t.mu.Lock() - t.mu.failed = true - t.mu.output = append(t.mu.output, t.decorate(0 /* skip */, fmt.Sprint(err))...) - t.mu.Unlock() + t.Error(err) } t.mu.Lock() @@ -774,9 +788,7 @@ func (r *testRunner) runTest( durationStr := fmt.Sprintf("%.2fs", t.duration().Seconds()) if t.Failed() { - t.mu.Lock() - output := fmt.Sprintf("test artifacts and logs in: %s\n", t.ArtifactsDir()) + string(t.mu.output) - t.mu.Unlock() + output := fmt.Sprintf("test artifacts and logs in: %s\n%s", t.ArtifactsDir(), t.failureMsg()) if teamCity { shout(ctx, l, stdout, "##teamcity[testFailed name='%s' details='%s' flowId='%s']", @@ -785,7 +797,9 @@ func (r *testRunner) runTest( shout(ctx, l, stdout, "--- FAIL: %s (%s)\n%s", runID, durationStr, output) - r.maybePostGithubIssue(ctx, l, t, stdout, output) + if err := github.MaybePost(t, output); err != nil { + shout(ctx, l, stdout, "failed to post issue: %s", err) + } } else { shout(ctx, l, stdout, "--- PASS: %s (%s)", runID, durationStr) // If `##teamcity[testFailed ...]` is not present before `##teamCity[testFinished ...]`, @@ -820,7 +834,7 @@ func (r *testRunner) runTest( start: t.start, end: t.end, pass: !t.Failed(), - failure: t.FailureMsg(), + failure: t.failureMsg(), }) r.status.Lock() delete(r.status.running, t) @@ -875,6 +889,8 @@ func (r *testRunner) runTest( // produced by t.Fatal*(). if r := recover(); r != nil && r != errTestFatal { // NB: we're careful to avoid t.Fatalf here, which re-panics. + // Note that the error will be logged to a file, and the stack will + // contain the source of the panic. t.Errorf("test panicked: %v", r) } }() @@ -987,7 +1003,7 @@ func (r *testRunner) teardownTest( c.FailOnReplicaDivergence(ctx, t) if timedOut || t.Failed() { - r.collectClusterArtifacts(ctx, c, t) + r.collectClusterArtifacts(ctx, c, t.L()) } }) @@ -1016,87 +1032,9 @@ func (r *testRunner) teardownTest( return nil } -func (r *testRunner) shouldPostGithubIssue(t test.Test) bool { - opts := issues.DefaultOptionsFromEnv() - return !r.config.disableIssue && - opts.CanPost() && - opts.IsReleaseBranch() && - t.Spec().(*registry.TestSpec).Run != nil && - // NB: check NodeCount > 0 to avoid posting issues from this pkg's unit tests. - t.Spec().(*registry.TestSpec).Cluster.NodeCount > 0 -} - -func (r *testRunner) maybePostGithubIssue( - ctx context.Context, l *logger.Logger, t test.Test, stdout io.Writer, output string, +func (r *testRunner) collectClusterArtifacts( + ctx context.Context, c *clusterImpl, l *logger.Logger, ) { - if !r.shouldPostGithubIssue(t) { - return - } - - // Issues posted from roachtest are identifiable as such and - // they are also release blockers (this label may be removed - // by a human upon closer investigation). - labels := []string{"O-roachtest"} - if !t.Spec().(*registry.TestSpec).NonReleaseBlocker { - labels = append(labels, "release-blocker") - } - - teams, err := team.DefaultLoadTeams() - if err != nil { - t.Fatalf("could not load teams: %v", err) - } - - var mention []string - var projColID int - if sl, ok := teams.GetAliasesForPurpose(ownerToAlias(t.Spec().(*registry.TestSpec).Owner), team.PurposeRoachtest); ok { - for _, alias := range sl { - mention = append(mention, "@"+string(alias)) - if label := teams[alias].Label; label != "" { - labels = append(labels, label) - } - } - projColID = teams[sl[0]].TriageColumnID - } - - branch := os.Getenv("TC_BUILD_BRANCH") - if branch == "" { - branch = "" - } - - msg := fmt.Sprintf("The test failed on branch=%s, cloud=%s:\n%s", - branch, t.Spec().(*registry.TestSpec).Cluster.Cloud, output) - artifacts := fmt.Sprintf("/%s", t.Name()) - - req := issues.PostRequest{ - MentionOnCreate: mention, - ProjectColumnID: projColID, - PackageName: "roachtest", - TestName: t.Name(), - Message: msg, - Artifacts: artifacts, - ExtraLabels: labels, - HelpCommand: func(renderer *issues.Renderer) { - issues.HelpCommandAsLink( - "roachtest README", - "https://github.com/cockroachdb/cockroach/blob/master/pkg/cmd/roachtest/README.md", - )(renderer) - issues.HelpCommandAsLink( - "How To Investigate (internal)", - "https://cockroachlabs.atlassian.net/l/c/SSSBr8c7", - )(renderer) - }, - } - if err := issues.Post( - context.Background(), - issues.UnitTestFormatter, - req, - ); err != nil { - shout(ctx, l, stdout, "failed to post issue: %s", err) - } -} - -// TODO(tbg): nothing in this method should have the `t`; they should have a `Logger` only. -func (r *testRunner) collectClusterArtifacts(ctx context.Context, c *clusterImpl, t test.Test) { // NB: fetch the logs even when we have a debug zip because // debug zip can't ever get the logs for down nodes. // We only save artifacts for failed tests in CI, so this @@ -1105,32 +1043,32 @@ func (r *testRunner) collectClusterArtifacts(ctx context.Context, c *clusterImpl // below has problems. For example, `debug zip` is known to // hang sometimes at the time of writing, see: // https://github.com/cockroachdb/cockroach/issues/39620 - t.L().PrintfCtx(ctx, "collecting cluster logs") + l.PrintfCtx(ctx, "collecting cluster logs") // Do this before collecting logs to make sure the file gets // downloaded below. if err := saveDiskUsageToLogsDir(ctx, c); err != nil { - t.L().Printf("failed to fetch disk uage summary: %s", err) + l.Printf("failed to fetch disk uage summary: %s", err) } - if err := c.FetchLogs(ctx, t); err != nil { - t.L().Printf("failed to download logs: %s", err) + if err := c.FetchLogs(ctx, l); err != nil { + l.Printf("failed to download logs: %s", err) } - if err := c.FetchDmesg(ctx, t); err != nil { - t.L().Printf("failed to fetch dmesg: %s", err) + if err := c.FetchDmesg(ctx, l); err != nil { + l.Printf("failed to fetch dmesg: %s", err) } - if err := c.FetchJournalctl(ctx, t); err != nil { - t.L().Printf("failed to fetch journalctl: %s", err) + if err := c.FetchJournalctl(ctx, l); err != nil { + l.Printf("failed to fetch journalctl: %s", err) } - if err := c.FetchCores(ctx, t); err != nil { - t.L().Printf("failed to fetch cores: %s", err) + if err := c.FetchCores(ctx, l); err != nil { + l.Printf("failed to fetch cores: %s", err) } if err := c.CopyRoachprodState(ctx); err != nil { - t.L().Printf("failed to copy roachprod state: %s", err) + l.Printf("failed to copy roachprod state: %s", err) } - if err := c.FetchTimeseriesData(ctx, t); err != nil { - t.L().Printf("failed to fetch timeseries data: %s", err) + if err := c.FetchTimeseriesData(ctx, l); err != nil { + l.Printf("failed to fetch timeseries data: %s", err) } - if err := c.FetchDebugZip(ctx, t); err != nil { - t.L().Printf("failed to collect zip: %s", err) + if err := c.FetchDebugZip(ctx, l); err != nil { + l.Printf("failed to collect zip: %s", err) } } @@ -1173,7 +1111,6 @@ type getWorkCallbacks struct { // getWork takes in a cluster; if not nil, tests that can reuse it are // preferred. If a test that can reuse it is not found (or if there's no more // work), the cluster is destroyed (and so its resources are released). -// func (r *testRunner) getWork( ctx context.Context, work *workPool, @@ -1225,13 +1162,15 @@ func (r *testRunner) removeWorker(ctx context.Context, name string) { // runHTTPServer starts a server running in the background. // // httpPort: The port on which to serve the web interface. Pass 0 for allocating -// a port automatically (which will be printed to stdout). -func (r *testRunner) runHTTPServer(httpPort int, stdout io.Writer) error { +// bindTo: The host/ip on which to bind. Leave empty to bind on all local ips +// +// a port automatically (which will be printed to stdout). +func (r *testRunner) runHTTPServer(httpPort int, stdout io.Writer, bindTo string) error { http.HandleFunc("/", r.serveHTTP) // Run an http server in the background. // We handle the case where httpPort is 0, which means we automatically // allocate a port. - listener, err := net.Listen("tcp", fmt.Sprintf(":%d", httpPort)) + listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", bindTo, httpPort)) if err != nil { return err } @@ -1241,7 +1180,11 @@ func (r *testRunner) runHTTPServer(httpPort int, stdout io.Writer) error { panic(err) } }() - fmt.Fprintf(stdout, "HTTP server listening on all network interfaces, port %d.\n", httpPort) + bindToDesc := "all network interfaces" + if bindTo != "" { + bindToDesc = bindTo + } + fmt.Fprintf(stdout, "HTTP server listening on %s, port %d.\n", bindToDesc, httpPort) return nil } diff --git a/pkg/cmd/roachtest/test_test.go b/pkg/cmd/roachtest/test_test.go index 91c9771dee96..7925dff30914 100644 --- a/pkg/cmd/roachtest/test_test.go +++ b/pkg/cmd/roachtest/test_test.go @@ -15,6 +15,7 @@ import ( "context" "io/ioutil" "math/rand" + "path/filepath" "regexp" "sort" "strings" @@ -27,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/roachprod/logger" + "github.com/cockroachdb/cockroach/pkg/roachprod/vm" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/quotapool" "github.com/cockroachdb/cockroach/pkg/util/stop" @@ -98,8 +100,8 @@ func alwaysFailingClusterAllocator( alloc *quotapool.IntAlloc, artifactsDir string, wStatus *workerStatus, -) (*clusterImpl, error) { - return nil, errors.New("cluster creation failed") +) (*clusterImpl, *vm.CreateOpts, error) { + return nil, nil, errors.New("cluster creation failed") } func TestRunnerRun(t *testing.T) { @@ -183,6 +185,7 @@ func TestRunnerRun(t *testing.T) { if exp := c.expOut; exp != "" && !strings.Contains(out, exp) { t.Fatalf("'%s' not found in output:\n%s", exp, out) } + t.Log(out) }) } } @@ -252,7 +255,13 @@ func setupRunnerTest(t *testing.T, r testRegistryImpl, testFilters []string) *ru var stdout syncedBuffer var stderr syncedBuffer lopt := loggingOpt{ - l: nilLogger(), + l: func() *logger.Logger { + l, err := logger.RootLogger(filepath.Join(t.TempDir(), "test.log"), logger.NoTee) + if err != nil { + panic(err) + } + return l + }(), tee: logger.NoTee, stdout: &stdout, stderr: &stderr, diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel index 2b64300956dd..1abc66e55c55 100644 --- a/pkg/cmd/roachtest/tests/BUILD.bazel +++ b/pkg/cmd/roachtest/tests/BUILD.bazel @@ -168,6 +168,7 @@ go_library( "//pkg/kv", "//pkg/roachpb", "//pkg/roachprod", + "//pkg/roachprod/errors", "//pkg/roachprod/install", "//pkg/roachprod/logger", "//pkg/security", diff --git a/pkg/cmd/roachtest/tests/activerecord.go b/pkg/cmd/roachtest/tests/activerecord.go index d18f8d52eb4b..c07ce366ca1b 100644 --- a/pkg/cmd/roachtest/tests/activerecord.go +++ b/pkg/cmd/roachtest/tests/activerecord.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" ) @@ -184,16 +185,10 @@ func registerActiveRecord(r registry.Registry) { `sudo RUBYOPT="-W0" TESTOPTS="-v" bundle exec rake test`, ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } // Result error contains stdout, stderr, and any error content returned by exec package. diff --git a/pkg/cmd/roachtest/tests/django.go b/pkg/cmd/roachtest/tests/django.go index d646543272f5..b3ca71159b08 100644 --- a/pkg/cmd/roachtest/tests/django.go +++ b/pkg/cmd/roachtest/tests/django.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" ) @@ -206,16 +207,10 @@ func registerDjango(r registry.Registry) { t.Status("Running django test app ", testName) result, err := c.RunWithDetailsSingleNode(ctx, t.L(), node, fmt.Sprintf(djangoRunTestCmd, testName)) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } rawResults := []byte(result.Stdout + result.Stderr) diff --git a/pkg/cmd/roachtest/tests/gopg.go b/pkg/cmd/roachtest/tests/gopg.go index f3075f43153d..d07a8bd18253 100644 --- a/pkg/cmd/roachtest/tests/gopg.go +++ b/pkg/cmd/roachtest/tests/gopg.go @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" ) @@ -115,16 +116,10 @@ func registerGopg(r registry.Registry) { destPath, removeColorCodes, resultsFilePath), ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } rawResults := []byte(result.Stdout + result.Stderr) @@ -152,16 +147,10 @@ func registerGopg(r registry.Registry) { destPath, goPath, resultsFilePath, goPath), ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } xmlResults := []byte(result.Stdout + result.Stderr) diff --git a/pkg/cmd/roachtest/tests/jepsen.go b/pkg/cmd/roachtest/tests/jepsen.go index 9d533256d98a..cc4e61d13b16 100644 --- a/pkg/cmd/roachtest/tests/jepsen.go +++ b/pkg/cmd/roachtest/tests/jepsen.go @@ -108,7 +108,7 @@ func initJepsen(ctx context.Context, t test.Test, c cluster.Cluster) { ctx, t.L(), controller, "sh", "-c", `"sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install openjdk-8-jre openjdk-8-jre-headless libjna-java gnuplot > /dev/null 2>&1"`, ); err != nil { - if result.RemoteExitStatus == "100" { + if result.RemoteExitStatus == 100 { t.Skip("apt-get failure (#31944)", result.Stdout+result.Stderr) } t.Fatal(err) diff --git a/pkg/cmd/roachtest/tests/nodejs_postgres.go b/pkg/cmd/roachtest/tests/nodejs_postgres.go index 223e5b1d536c..9065511956aa 100644 --- a/pkg/cmd/roachtest/tests/nodejs_postgres.go +++ b/pkg/cmd/roachtest/tests/nodejs_postgres.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" @@ -141,16 +142,10 @@ PGSSLCERT=$HOME/certs/client.%s.crt PGSSLKEY=$HOME/certs/client.%s.key PGSSLROOT ), ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } rawResultsStr := result.Stdout + result.Stderr diff --git a/pkg/cmd/roachtest/tests/pgx.go b/pkg/cmd/roachtest/tests/pgx.go index 9f81b2ac2c11..eacaeba3a35d 100644 --- a/pkg/cmd/roachtest/tests/pgx.go +++ b/pkg/cmd/roachtest/tests/pgx.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" ) @@ -121,16 +122,10 @@ func registerPgx(r registry.Registry) { "`go env GOPATH`/bin/go-junit-report", ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } // Result error contains stdout, stderr, and any error content returned by exec package. diff --git a/pkg/cmd/roachtest/tests/psycopg.go b/pkg/cmd/roachtest/tests/psycopg.go index b556720c02f6..b9de3e7ca68f 100644 --- a/pkg/cmd/roachtest/tests/psycopg.go +++ b/pkg/cmd/roachtest/tests/psycopg.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" ) @@ -124,16 +125,10 @@ func registerPsycopg(r registry.Registry) { make check PYTHON_VERSION=3`, ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } // Result error contains stdout, stderr, and any error content returned by exec package. diff --git a/pkg/cmd/roachtest/tests/ruby_pg.go b/pkg/cmd/roachtest/tests/ruby_pg.go index 8b061b92c80f..a02282c9cb42 100644 --- a/pkg/cmd/roachtest/tests/ruby_pg.go +++ b/pkg/cmd/roachtest/tests/ruby_pg.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" @@ -156,16 +157,10 @@ func registerRubyPG(r registry.Registry) { `cd /mnt/data1/ruby-pg/ && sudo rake`, ) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } rawResults := []byte(result.Stdout + result.Stderr) diff --git a/pkg/cmd/roachtest/tests/sqlalchemy.go b/pkg/cmd/roachtest/tests/sqlalchemy.go index 2ae36044364c..e78ab7d6b38e 100644 --- a/pkg/cmd/roachtest/tests/sqlalchemy.go +++ b/pkg/cmd/roachtest/tests/sqlalchemy.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/cockroach/pkg/roachprod/install" "github.com/cockroachdb/errors" ) @@ -170,16 +171,10 @@ func runSQLAlchemy(ctx context.Context, t test.Test, c cluster.Cluster) { test/test_suite_sqlalchemy.py `) - // Expected to fail but we should still scan the error to check if - // there's an SSH/roachprod error. - if err != nil { - // install.NonZeroExitCode includes unrelated to SSH errors ("255") - // or roachprod errors, so we call t.Fatal if the error is not an - // install.NonZeroExitCode error - commandError := (*install.NonZeroExitCode)(nil) - if !errors.As(err, &commandError) { - t.Fatal(err) - } + // Fatal for a roachprod or SSH error. A roachprod error is when result.Err==nil. + // Proceed for any other (command) errors + if err != nil && (result.Err == nil || errors.Is(err, rperrors.ErrSSH255)) { + t.Fatal(err) } rawResults := []byte(result.Stdout + result.Stderr) diff --git a/pkg/roachprod/errors/errors.go b/pkg/roachprod/errors/errors.go index 544e460bbcdb..a6f5823200ea 100644 --- a/pkg/roachprod/errors/errors.go +++ b/pkg/roachprod/errors/errors.go @@ -33,6 +33,10 @@ const ( unclassifiedExitCode = 1 ) +// ErrSSH255 is a reference error used to mark an SSH error with an exit +// code of 255. This could be indicative of an SSH flake. +var ErrSSH255 = errors.New("SSH error occurred with exit code 255") + // Cmd wraps errors that result from a command run against the cluster. type Cmd struct { Err error @@ -114,9 +118,9 @@ func ClassifyCmdError(err error) Error { return nil } - if exitErr, ok := asExitError(err); ok { - if exitErr.ExitCode() == 255 { - return SSH{err} + if exitCode, ok := GetExitCode(err); ok { + if exitCode == 255 { + return SSH{errors.Mark(err, ErrSSH255)} } return Cmd{err} } @@ -124,6 +128,16 @@ func ClassifyCmdError(err error) Error { return Unclassified{err} } +// GetExitCode returns an exit code, true if the error is an instance +// of an ExitError, or -1, false otherwise +func GetExitCode(err error) (int, bool) { + if exitErr, ok := asExitError(err); ok { + return exitErr.ExitCode(), true + } + + return -1, false +} + // Extract the ExitError from err's error tree or (nil, false) if none exists. func asExitError(err error) (*exec.ExitError, bool) { var exitErr *exec.ExitError @@ -141,40 +155,3 @@ func AsError(err error) (Error, bool) { } return nil, false } - -// SelectPriorityError selects an error from the list in this priority order: -// -// - the Error with the highest exit code -// - one of the `error`s -// - nil -func SelectPriorityError(errors []error) error { - var result Error - for _, err := range errors { - if err == nil { - continue - } - - rpErr, _ := AsError(err) - if result == nil { - result = rpErr - continue - } - - if rpErr.ExitCode() > result.ExitCode() { - result = rpErr - } - } - - if result != nil { - return result - } - - for _, err := range errors { - if err != nil { - return err - } - } - return nil -} - -var _ = SelectPriorityError diff --git a/pkg/roachprod/install/BUILD.bazel b/pkg/roachprod/install/BUILD.bazel index 1a1becdce9b6..31839aaa6794 100644 --- a/pkg/roachprod/install/BUILD.bazel +++ b/pkg/roachprod/install/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "//pkg/util", "//pkg/util/httputil", "//pkg/util/log", + "//pkg/util/retry", "//pkg/util/syncutil", "//pkg/util/timeutil", "//pkg/util/version", @@ -51,8 +52,11 @@ go_test( data = glob(["testdata/**"]), embed = [":install"], deps = [ + "//pkg/roachprod/logger", "//pkg/testutils", + "//pkg/util/retry", "@com_github_cockroachdb_datadriven//:datadriven", + "@com_github_cockroachdb_errors//:errors", "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/roachprod/install/cluster_synced.go b/pkg/roachprod/install/cluster_synced.go index 1f8a0ce73d14..129e4cfa7adc 100644 --- a/pkg/roachprod/install/cluster_synced.go +++ b/pkg/roachprod/install/cluster_synced.go @@ -23,7 +23,6 @@ import ( "os/exec" "os/signal" "path/filepath" - "reflect" "sort" "strings" "sync" @@ -41,6 +40,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachprod/vm/aws" "github.com/cockroachdb/cockroach/pkg/roachprod/vm/local" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/retry" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" @@ -100,6 +100,82 @@ func NewSyncedCluster( return c, nil } +// ErrAfterRetry marks an error that has occurred/persisted after retries +var ErrAfterRetry = errors.New("error occurred after retries") + +// The first retry is after 5s, the second and final is after 25s +var defaultRunRetryOpt = retry.Options{ + InitialBackoff: 5 * time.Second, + Multiplier: 5, + MaxBackoff: 1 * time.Minute, + // This will run a total of 3 times `runWithMaybeRetry` + MaxRetries: 2, +} + +// runWithMaybeRetry will run the specified function `f` at least once. +// Any returned error from `f` is passed to the `shouldRetryFn` which, +// if it returns true, will result in `f` being retried using the `retryOpts` +// If the `shouldRetryFn` is not specified (nil), then no retries will be +// performed. +// +// We operate on a pointer to RunResultDetails as it has already have been +// captured in a *RunResultDetails[] in Run, but here we may enrich with attempt +// number and a wrapper error. +func runWithMaybeRetry( + l *logger.Logger, + retryOpts retry.Options, + shouldRetryFn func(*RunResultDetails) bool, + f func() (*RunResultDetails, error), +) (*RunResultDetails, error) { + var err error + var res *RunResultDetails + + var cmdErr error + + for r := retry.Start(retryOpts); r.Next(); { + res, err = f() + res.Attempt = r.CurrentAttempt() + 1 + // nil err (denoting a roachprod error) indicates a potentially retryable res.Err + if err == nil && res.Err != nil { + cmdErr = errors.CombineErrors(cmdErr, res.Err) + if shouldRetryFn != nil && shouldRetryFn(res) { + l.Printf("Encountered [%v] on attempt %v of %v", res.Err, r.CurrentAttempt()+1, retryOpts.MaxRetries+1) + continue + } + } + break + } + + if res.Err != nil && res.Attempt > 1 { + // An error cannot be marked with more than one reference error. Since res.Err may already be marked, we create + // a new error here and mark it. + res.Err = errors.Mark(errors.Wrapf(cmdErr, "error persisted after %v attempts", res.Attempt), ErrAfterRetry) + } + return res, err +} + +// runWithDefaultSSHRetry will retry an SSH command which returns an error with exit code 255 +func runWithDefaultSSHRetry( + l *logger.Logger, f func() (*RunResultDetails, error), +) (*RunResultDetails, error) { + return runWithMaybeRetry( + l, + defaultRunRetryOpt, + func(res *RunResultDetails) bool { return errors.Is(res.Err, rperrors.ErrSSH255) }, + f, + ) +} + +// scpWithDefaultRetry assumes that any error returned from an scp attempt is retryable +func scpWithDefaultRetry(l *logger.Logger, src, dest string) (*RunResultDetails, error) { + return runWithMaybeRetry( + l, + defaultRunRetryOpt, + func(*RunResultDetails) bool { return true }, + func() (*RunResultDetails, error) { return scp(src, dest) }, + ) +} + // Host returns the public IP of a node. func (c *SyncedCluster) Host(n Node) string { return c.VMs[n-1].PublicIP @@ -125,10 +201,10 @@ func (c *SyncedCluster) localVMDir(n Node) string { // TargetNodes is the fully expanded, ordered list of nodes that any given // roachprod command is intending to target. // -// $ roachprod create local -n 4 -// $ roachprod start local # [1, 2, 3, 4] -// $ roachprod start local:2-4 # [2, 3, 4] -// $ roachprod start local:2,1,4 # [1, 2, 4] +// $ roachprod create local -n 4 +// $ roachprod start local # [1, 2, 3, 4] +// $ roachprod start local:2-4 # [2, 3, 4] +// $ roachprod start local:2,1,4 # [1, 2, 4] func (c *SyncedCluster) TargetNodes() Nodes { return append(Nodes{}, c.Nodes...) } @@ -169,25 +245,25 @@ func (c *SyncedCluster) GetInternalIP(ctx context.Context, n Node) (string, erro // correct process, when monitoring or stopping. // // Normally, the value is of the form: -// [/][/tag] // -// Examples: +// [/][/tag] // -// - non-local cluster without tags: -// ROACHPROD=1 +// Examples: // -// - non-local cluster with tag foo: -// ROACHPROD=1/foo +// - non-local cluster without tags: +// ROACHPROD=1 // -// - non-local cluster with hierarchical tag foo/bar: -// ROACHPROD=1/foo/bar +// - non-local cluster with tag foo: +// ROACHPROD=1/foo // -// - local cluster: -// ROACHPROD=local-foo/1 +// - non-local cluster with hierarchical tag foo/bar: +// ROACHPROD=1/foo/bar // -// - local cluster with tag bar: -// ROACHPROD=local-foo/1/bar +// - local cluster: +// ROACHPROD=local-foo/1 // +// - local cluster with tag bar: +// ROACHPROD=local-foo/1/bar func (c *SyncedCluster) roachprodEnvValue(node Node) string { var parts []string if c.IsLocal() { @@ -232,10 +308,11 @@ func (c *SyncedCluster) Stop(ctx context.Context, l *logger.Logger, sig int, wai if wait { display += " and waiting" } - return c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(c.Nodes[i]) + return c.Parallel(l, display, len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -252,7 +329,7 @@ func (c *SyncedCluster) Stop(ctx context.Context, l *logger.Logger, sig int, wai done echo "${pid}: dead" >> %[1]s/roachprod.log done`, - c.LogDir(c.Nodes[i]), // [1] + c.LogDir(node), // [1] ) } @@ -269,12 +346,15 @@ if [ -n "${pids}" ]; then kill -%[3]d ${pids} %[4]s fi`, - c.LogDir(c.Nodes[i]), // [1] - c.roachprodEnvRegex(c.Nodes[i]), // [2] - sig, // [3] - waitCmd, // [4] + c.LogDir(node), // [1] + c.roachprodEnvRegex(node), // [2] + sig, // [3] + waitCmd, // [4] ) - return sess.CombinedOutput(ctx, cmd) + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + return res, res.Err }) } @@ -284,10 +364,11 @@ func (c *SyncedCluster) Wipe(ctx context.Context, l *logger.Logger, preserveCert if err := c.Stop(ctx, l, 9, true /* wait */); err != nil { return err } - return c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(c.Nodes[i]) + return c.Parallel(l, display, len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -312,7 +393,10 @@ sudo rm -fr logs && cmd += "sudo rm -fr tenant-certs* ;\n" } } - return sess.CombinedOutput(ctx, cmd) + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + return res, res.Err }) } @@ -320,19 +404,19 @@ sudo rm -fr logs && func (c *SyncedCluster) Status(ctx context.Context, l *logger.Logger) error { display := fmt.Sprintf("%s: status", c.Name) results := make([]string, len(c.Nodes)) - if err := c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] sess, err := c.newSession(c.Nodes[i]) if err != nil { - results[i] = err.Error() - return nil, nil + return newRunResultDetails(node, err), err } defer sess.Close() - binary := cockroachNodeBinary(c, c.Nodes[i]) + binary := cockroachNodeBinary(c, node) cmd := fmt.Sprintf(`out=$(ps axeww -o pid -o ucomm -o command | \ sed 's/export ROACHPROD=//g' | \ awk '/%s/ {print $2, $1}'`, - c.roachprodEnvRegex(c.Nodes[i])) + c.roachprodEnvRegex(node)) cmd += ` | sort | uniq); vers=$(` + binary + ` version 2>/dev/null | awk '/Build Tag:/ {print $NF}') if [ -n "${out}" -a -n "${vers}" ]; then @@ -341,17 +425,20 @@ else echo ${out} fi ` - out, err := sess.CombinedOutput(ctx, cmd) - var msg string - if err != nil { - return nil, errors.Wrapf(err, "~ %s\n%s", cmd, out) + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + return res, errors.Wrapf(res.Err, "~ %s\n%s", cmd, res.CombinedOut) } - msg = strings.TrimSpace(string(out)) + + msg := strings.TrimSpace(string(res.CombinedOut)) if msg == "" { msg = "not running" } results[i] = msg - return nil, nil + return res, nil }); err != nil { return err } @@ -551,47 +638,45 @@ type RunResultDetails struct { Node Node Stdout string Stderr string + CombinedOut []byte Err error - RemoteExitStatus string + RemoteExitStatus int + Attempt int } -func processStdout(stdout string) (string, string) { - retStdout := stdout - exitStatusPattern := "LAST EXIT STATUS: " - exitStatusIndex := strings.LastIndex(retStdout, exitStatusPattern) - remoteExitStatus := "-1" - // If exitStatusIndex is -1 then "echo LAST EXIT STATUS: $?" didn't run - // mostly due to an ssh error but avoid speculation and temporarily - // use "-1" for unknown error before checking if it's SSH related later. - if exitStatusIndex != -1 { - retStdout = stdout[:exitStatusIndex] - remoteExitStatus = strings.TrimSpace(stdout[exitStatusIndex+len(exitStatusPattern):]) - } - return retStdout, remoteExitStatus +func newRunResultDetails(node Node, err error) *RunResultDetails { + res := RunResultDetails{ + Node: node, + Err: err, + } + if exitCode, success := rperrors.GetExitCode(err); success { + res.RemoteExitStatus = exitCode + } + + return &res } -func runCmdOnSingleNode( - ctx context.Context, l *logger.Logger, c *SyncedCluster, node Node, cmd string, -) (RunResultDetails, error) { - result := RunResultDetails{Node: node} +func (c *SyncedCluster) runCmdOnSingleNode( + ctx context.Context, + l *logger.Logger, + node Node, + cmd string, + combined bool, + stdout, stderr io.Writer, +) (*RunResultDetails, error) { sess, err := c.newSession(node) if err != nil { - return result, err + return newRunResultDetails(node, err), err } defer sess.Close() - sess.SetWithExitStatus(true) - var stdoutBuffer, stderrBuffer bytes.Buffer - sess.SetStdout(&stdoutBuffer) - sess.SetStderr(&stderrBuffer) - // Argument template expansion is node specific (e.g. for {store-dir}). e := expander{ node: node, } expandedCmd, err := e.expand(ctx, l, c, cmd) if err != nil { - return result, err + return newRunResultDetails(node, err), err } // Be careful about changing these command strings. In particular, we need @@ -608,31 +693,29 @@ func runCmdOnSingleNode( nodeCmd = fmt.Sprintf("cd %s; %s", c.localVMDir(node), nodeCmd) } - err = sess.Run(ctx, nodeCmd) - result.Stderr = stderrBuffer.String() - result.Stdout, result.RemoteExitStatus = processStdout(stdoutBuffer.String()) + var res *RunResultDetails + if combined { + out, cmdErr := sess.CombinedOutput(ctx, nodeCmd) + res = newRunResultDetails(node, cmdErr) + res.CombinedOut = out + } else { + // We stream the output if running on a single node. + var stdoutBuffer, stderrBuffer bytes.Buffer + multStdout := io.MultiWriter(&stdoutBuffer, stdout) + multStderr := io.MultiWriter(&stderrBuffer, stderr) + sess.SetStdout(multStdout) + sess.SetStderr(multStderr) - if err != nil { - detailMsg := fmt.Sprintf("Node %d. Command with error:\n```\n%s\n```\n", node, cmd) - err = errors.WithDetail(err, detailMsg) - err = rperrors.ClassifyCmdError(err) - if reflect.TypeOf(err) == reflect.TypeOf(rperrors.SSH{}) { - result.RemoteExitStatus = "255" - } - result.Err = err - } else if result.RemoteExitStatus != "0" { - result.Err = &NonZeroExitCode{fmt.Sprintf("Non-zero exit code: %s", result.RemoteExitStatus)} + res = newRunResultDetails(node, sess.Run(ctx, nodeCmd)) + res.Stderr = stderrBuffer.String() + res.Stdout = stdoutBuffer.String() } - return result, nil -} - -// NonZeroExitCode is returned when a command executed by Run() exits with a non-zero status. -type NonZeroExitCode struct { - message string -} -func (e *NonZeroExitCode) Error() string { - return e.message + if res.Err != nil { + detailMsg := fmt.Sprintf("Node %d. Command with error:\n```\n%s\n```\n", node, cmd) + res.Err = errors.WithDetail(res.Err, detailMsg) + } + return res, nil } // Run a command on >= 1 node in the cluster. @@ -656,74 +739,47 @@ func (c *SyncedCluster) Run( display = fmt.Sprintf("%s: %s", c.Name, title) } - errs := make([]error, len(nodes)) - results := make([]string, len(nodes)) - if err := c.Parallel(l, display, len(nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(nodes[i]) - if err != nil { - errs[i] = err - results[i] = err.Error() - return nil, nil - } - defer sess.Close() + results := make([]*RunResultDetails, len(nodes)) - // Argument template expansion is node specific (e.g. for {store-dir}). - e := expander{ - node: nodes[i], - } - expandedCmd, err := e.expand(ctx, l, c, cmd) - if err != nil { - return nil, err + // A result is the output of running a command (could be interpreted as an error) + if _, err := c.ParallelE(l, display, len(nodes), 0, func(i int) (*RunResultDetails, error) { + // An err returned here is an unexpected state within roachprod (non-command error). + // For errors that occur as part of running a command over ssh, the `result` will contain + // the actual error on a specific node. + result, err := c.runCmdOnSingleNode(ctx, l, nodes[i], cmd, !stream, stdout, stderr) + results[i] = result + return result, err + }); err != nil { + return err + } + + return processResults(results, stream, stdout) +} + +// processResults returns the error from the RunResultDetails with the highest RemoteExitStatus +func processResults(results []*RunResultDetails, stream bool, stdout io.Writer) error { + var resultWithError *RunResultDetails + for i, r := range results { + if !stream { + fmt.Fprintf(stdout, " %2d: %s\n%v\n", i+1, strings.TrimSpace(string(r.CombinedOut)), r.Err) } - // Be careful about changing these command strings. In particular, we need - // to support running commands in the background on both local and remote - // nodes. For example: - // - // roachprod run cluster -- "sleep 60 &> /dev/null < /dev/null &" - // - // That command should return immediately. And a "roachprod status" should - // reveal that the sleep command is running on the cluster. - nodeCmd := fmt.Sprintf(`export ROACHPROD=%s GOTRACEBACK=crash && bash -c %s`, - c.roachprodEnvValue(nodes[i]), ssh.Escape1(expandedCmd)) - if c.IsLocal() { - nodeCmd = fmt.Sprintf("cd %s; %s", c.localVMDir(nodes[i]), nodeCmd) - } - - if stream { - sess.SetStdout(stdout) - sess.SetStderr(stderr) - errs[i] = sess.Run(ctx, nodeCmd) - if errs[i] != nil { - detailMsg := fmt.Sprintf("Node %d. Command with error:\n```\n%s\n```\n", nodes[i], cmd) - err = errors.WithDetail(errs[i], detailMsg) - err = rperrors.ClassifyCmdError(err) - errs[i] = err + if r.Err != nil { + if resultWithError == nil { + resultWithError = r + continue } - return nil, nil - } - out, err := sess.CombinedOutput(ctx, nodeCmd) - msg := strings.TrimSpace(string(out)) - if err != nil { - detailMsg := fmt.Sprintf("Node %d. Command with error:\n```\n%s\n```\n", nodes[i], cmd) - err = errors.WithDetail(err, detailMsg) - err = rperrors.ClassifyCmdError(err) - errs[i] = err - msg += fmt.Sprintf("\n%v", err) + if r.RemoteExitStatus > resultWithError.RemoteExitStatus { + resultWithError = r + } } - results[i] = msg - return nil, nil - }); err != nil { - return err } - if !stream { - for i, r := range results { - fmt.Fprintf(stdout, " %2d: %s\n", nodes[i], r) - } + if resultWithError != nil { + return resultWithError.Err } - return rperrors.SelectPriorityError(errs) + return nil } // RunWithDetails runs a command on the specified nodes and returns results details and an error. @@ -731,19 +787,24 @@ func (c *SyncedCluster) RunWithDetails( ctx context.Context, l *logger.Logger, nodes Nodes, title, cmd string, ) ([]RunResultDetails, error) { display := fmt.Sprintf("%s: %s", c.Name, title) - results := make([]RunResultDetails, len(nodes)) - failed, err := c.ParallelE(l, display, len(nodes), 0, func(i int) ([]byte, error) { - result, err := runCmdOnSingleNode(ctx, l, c, nodes[i], cmd) - if err != nil { - return nil, err - } - results[i] = result - return nil, nil + // We use pointers here as we are capturing the state of a result even though it may + // be processed further by the caller. + resultPtrs := make([]*RunResultDetails, len(nodes)) + + // Both return values are explicitly ignored because, in this case, resultPtrs + // capture both error and result state for each node + _, _ = c.ParallelE(l, display, len(nodes), 0, func(i int) (*RunResultDetails, error) { //nolint:errcheck + result, err := c.runCmdOnSingleNode(ctx, l, nodes[i], cmd, false, l.Stdout, l.Stderr) + resultPtrs[i] = result + return result, err }) - if err != nil { - for _, node := range failed { - results[node.Index].Err = node.Err + + // Return values to preserve API + results := make([]RunResultDetails, len(nodes)) + for i, v := range resultPtrs { + if v != nil { + results[i] = *v } } return results, nil @@ -753,9 +814,11 @@ func (c *SyncedCluster) RunWithDetails( func (c *SyncedCluster) Wait(ctx context.Context, l *logger.Logger) error { display := fmt.Sprintf("%s: waiting for nodes to start", c.Name) errs := make([]error, len(c.Nodes)) - if err := c.Parallel(l, display, len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, display, len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + res := &RunResultDetails{Node: node} for j := 0; j < 600; j++ { - sess, err := c.newSession(c.Nodes[i]) + sess, err := c.newSession(node) if err != nil { time.Sleep(500 * time.Millisecond) continue @@ -767,10 +830,11 @@ func (c *SyncedCluster) Wait(ctx context.Context, l *logger.Logger) error { time.Sleep(500 * time.Millisecond) continue } - return nil, nil + return res, nil } errs[i] = errors.New("timed out after 5m") - return nil, nil + res.Err = errs[i] + return res, nil }); err != nil { return err } @@ -788,6 +852,16 @@ func (c *SyncedCluster) Wait(ctx context.Context, l *logger.Logger) error { return nil } +// setupSession is a helper which creates a new session and +// populates RunResultDetails with an error if one occurrs (unlikely +// given the code in `newSession`) +// RunResultDetails is used across all functions which +// create a session and holds error and stdout information +func (c *SyncedCluster) setupSession(node Node) (session, error) { + sess, err := c.newSession(node) + return sess, err +} + // SetupSSH configures the cluster for use with SSH. This is generally run after // the cloud.Cluster has been synced which resets the SSH credentials on the // machines and sets them up for the current user. This method enables the @@ -795,12 +869,12 @@ func (c *SyncedCluster) Wait(ctx context.Context, l *logger.Logger) error { // added to the hosts via the c.AuthorizedKeys field. It does so in the following // steps: // -// 1. Creates an ssh key pair on the first host to be used on all hosts if -// none exists. -// 2. Distributes the public key, private key, and authorized_keys file from -// the first host to the others. -// 3. Merges the data in c.AuthorizedKeys with the existing authorized_keys -// files on all hosts. +// 1. Creates an ssh key pair on the first host to be used on all hosts if +// none exists. +// 2. Distributes the public key, private key, and authorized_keys file from +// the first host to the others. +// 3. Merges the data in c.AuthorizedKeys with the existing authorized_keys +// files on all hosts. // // This call strives to be idempotent. func (c *SyncedCluster) SetupSSH(ctx context.Context, l *logger.Logger) error { @@ -816,10 +890,10 @@ func (c *SyncedCluster) SetupSSH(ctx context.Context, l *logger.Logger) error { // Generate an ssh key that we'll distribute to all of the nodes in the // cluster in order to allow inter-node ssh. var sshTar []byte - if err := c.Parallel(l, "generating ssh key", 1, 0, func(i int) ([]byte, error) { - sess, err := c.newSession(1) + if err := c.Parallel(l, "generating ssh key", 1, 0, func(i int) (*RunResultDetails, error) { + sess, err := c.setupSession(1) if err != nil { - return nil, err + return newRunResultDetails(1, err), err } defer sess.Close() @@ -838,30 +912,40 @@ tar cf - .ssh/id_rsa .ssh/id_rsa.pub .ssh/authorized_keys sess.SetStdout(&stdout) sess.SetStderr(&stderr) - if err := sess.Run(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "%s: stderr:\n%s", cmd, stderr.String()) + res := newRunResultDetails(1, sess.Run(ctx, cmd)) + + res.Stdout = stdout.String() + res.Stderr = stderr.String() + if res.Err != nil { + return res, errors.Wrapf(res.Err, "%s: stderr:\n%s", cmd, res.Stderr) } - sshTar = stdout.Bytes() - return nil, nil + sshTar = []byte(res.Stdout) + return res, nil }); err != nil { return err } // Skip the first node which is where we generated the key. nodes := c.Nodes[1:] - if err := c.Parallel(l, "distributing ssh key", len(nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(nodes[i]) + if err := c.Parallel(l, "distributing ssh key", len(nodes), 0, func(i int) (*RunResultDetails, error) { + node := nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() sess.SetStdin(bytes.NewReader(sshTar)) cmd := `tar xf -` - if out, err := sess.CombinedOutput(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "%s: output:\n%s", cmd, out) + + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + return res, errors.Wrapf(res.Err, "%s: output:\n%s", cmd, res.CombinedOut) } - return nil, nil + return res, nil }); err != nil { return err } @@ -871,19 +955,23 @@ tar cf - .ssh/id_rsa .ssh/id_rsa.pub .ssh/authorized_keys // known hosts file in unhashed format, working around a limitation of jsch // (which is used in jepsen tests). ips := make([]string, len(c.Nodes), len(c.Nodes)*2) - if err := c.Parallel(l, "retrieving hosts", len(c.Nodes), 0, func(i int) ([]byte, error) { + if err := c.Parallel(l, "retrieving hosts", len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + res := &RunResultDetails{Node: node} for j := 0; j < 20 && ips[i] == ""; j++ { var err error - ips[i], err = c.GetInternalIP(ctx, c.Nodes[i]) + ips[i], err = c.GetInternalIP(ctx, node) if err != nil { - return nil, errors.Wrapf(err, "pgurls") + res.Err = errors.Wrapf(err, "pgurls") + return res, res.Err } time.Sleep(time.Second) } if ips[i] == "" { - return nil, fmt.Errorf("retrieved empty IP address") + res.Err = fmt.Errorf("retrieved empty IP address") + return res, res.Err } - return nil, nil + return res, nil }); err != nil { return err } @@ -892,10 +980,11 @@ tar cf - .ssh/id_rsa .ssh/id_rsa.pub .ssh/authorized_keys ips = append(ips, c.Host(i)) } var knownHostsData []byte - if err := c.Parallel(l, "scanning hosts", 1, 0, func(i int) ([]byte, error) { - sess, err := c.newSession(c.Nodes[i]) + if err := c.Parallel(l, "scanning hosts", 1, 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -925,19 +1014,25 @@ exit 1 var stderr bytes.Buffer sess.SetStdout(&stdout) sess.SetStderr(&stderr) - if err := sess.Run(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "%s: stderr:\n%s", cmd, stderr.String()) + + res := newRunResultDetails(node, sess.Run(ctx, cmd)) + + res.Stdout = stdout.String() + res.Stderr = stderr.String() + if res.Err != nil { + return res, errors.Wrapf(res.Err, "%s: stderr:\n%s", cmd, res.Stderr) } knownHostsData = stdout.Bytes() - return nil, nil + return res, nil }); err != nil { return err } - if err := c.Parallel(l, "distributing known_hosts", len(c.Nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(c.Nodes[i]) + if err := c.Parallel(l, "distributing known_hosts", len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -969,10 +1064,14 @@ if [[ "$(whoami)" != "` + config.SharedUser + `" ]]; then '"'"'{}'"'"' ~` + config.SharedUser + `/.ssh' \; fi ` - if out, err := sess.CombinedOutput(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "%s: output:\n%s", cmd, out) + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + return res, errors.Wrapf(res.Err, "%s: output:\n%s", cmd, res.CombinedOut) } - return nil, nil + return res, nil }); err != nil { return err } @@ -983,10 +1082,11 @@ fi // additional authorized_keys to both the current user (your username on // gce and the shared user on aws) as well as to the shared user on both // platforms. - if err := c.Parallel(l, "adding additional authorized keys", len(c.Nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(c.Nodes[i]) + if err := c.Parallel(l, "adding additional authorized keys", len(c.Nodes), 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -1013,10 +1113,14 @@ if [[ "$(whoami)" != "` + config.SharedUser + `" ]]; then "${tmp2}" ~` + config.SharedUser + `/.ssh/authorized_keys fi ` - if out, err := sess.CombinedOutput(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "~ %s\n%s", cmd, out) + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + return res, errors.Wrapf(res.Err, "~ %s\n%s", cmd, res.CombinedOut) } - return nil, nil + return res, nil }); err != nil { return err } @@ -1045,10 +1149,10 @@ func (c *SyncedCluster) DistributeCerts(ctx context.Context, l *logger.Logger) e // Generate the ca, client and node certificates on the first node. var msg string display := fmt.Sprintf("%s: initializing certs", c.Name) - if err := c.Parallel(l, display, 1, 0, func(i int) ([]byte, error) { - sess, err := c.newSession(1) + if err := c.Parallel(l, display, 1, 0, func(i int) (*RunResultDetails, error) { + sess, err := c.setupSession(1) if err != nil { - return nil, err + return newRunResultDetails(1, err), err } defer sess.Close() @@ -1065,10 +1169,15 @@ mkdir -p certs %[1]s cert create-node %[2]s --certs-dir=certs --ca-key=certs/ca.key tar cvf %[3]s certs `, cockroachNodeBinary(c, 1), strings.Join(nodeNames, " "), certsTarName) - if out, err := sess.CombinedOutput(ctx, cmd); err != nil { - msg = fmt.Sprintf("%s: %v", out, err) + + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(1, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + msg = fmt.Sprintf("%s: %v", res.CombinedOut, res.Err) } - return nil, nil + return res, nil }); err != nil { return err } @@ -1078,7 +1187,7 @@ tar cvf %[3]s certs exit.WithCode(exit.UnspecifiedError()) } - tarfile, cleanup, err := c.getFileFromFirstNode(certsTarName) + tarfile, cleanup, err := c.getFileFromFirstNode(l, certsTarName) if err != nil { return err } @@ -1111,7 +1220,7 @@ func (c *SyncedCluster) DistributeTenantCerts( return err } - tarfile, cleanup, err := hostCluster.getFileFromFirstNode(tenantCertsTarName) + tarfile, cleanup, err := hostCluster.getFileFromFirstNode(l, tenantCertsTarName) if err != nil { return err } @@ -1129,11 +1238,11 @@ func (c *SyncedCluster) createTenantCertBundle( ctx context.Context, l *logger.Logger, bundleName string, tenantID int, nodeNames []string, ) error { display := fmt.Sprintf("%s: initializing tenant certs", c.Name) - return c.Parallel(l, display, 1, 0, func(i int) ([]byte, error) { + return c.Parallel(l, display, 1, 0, func(i int) (*RunResultDetails, error) { node := c.Nodes[i] sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -1167,10 +1276,14 @@ tar cvf %[5]s $CERT_DIR bundleName, ) - if out, err := sess.CombinedOutput(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "certificate creation error: %s", out) + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + return res, errors.Wrapf(res.Err, "certificate creation error: %s", res.CombinedOut) } - return nil, nil + return res, nil }) } @@ -1195,7 +1308,9 @@ func (c *SyncedCluster) cockroachBinSupportsTenantScope(ctx context.Context, nod // getFile retrieves the given file from the first node in the cluster. The // filename is assumed to be relative from the home directory of the node's // user. -func (c *SyncedCluster) getFileFromFirstNode(name string) (string, func(), error) { +func (c *SyncedCluster) getFileFromFirstNode( + l *logger.Logger, name string, +) (string, func(), error) { var tmpfileName string cleanup := func() {} if c.IsLocal() { @@ -1211,9 +1326,9 @@ func (c *SyncedCluster) getFileFromFirstNode(name string) (string, func(), error } srcFileName := fmt.Sprintf("%s@%s:%s", c.user(1), c.Host(1), name) - if err := c.scp(srcFileName, tmpfile.Name()); err != nil { + if res, _ := scpWithDefaultRetry(l, srcFileName, tmpfile.Name()); res.Err != nil { cleanup() - return "", nil, err + return "", nil, res.Err } tmpfileName = tmpfile.Name() } @@ -1245,14 +1360,20 @@ func (c *SyncedCluster) fileExistsOnFirstNode( ) bool { var existsErr error display := fmt.Sprintf("%s: checking %s", c.Name, path) - if err := c.Parallel(l, display, 1, 0, func(i int) ([]byte, error) { - sess, err := c.newSession(c.Nodes[i]) + if err := c.Parallel(l, display, 1, 0, func(i int) (*RunResultDetails, error) { + node := c.Nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() - _, existsErr = sess.CombinedOutput(ctx, `test -e `+path) - return nil, nil + + out, cmdErr := sess.CombinedOutput(ctx, `test -e `+path) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + existsErr = res.Err + return res, nil }); err != nil { return false } @@ -1271,10 +1392,13 @@ func (c *SyncedCluster) createNodeCertArguments( nodes := allNodes(len(c.VMs)) if !c.IsLocal() { ips = make([]string, len(nodes)) - if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { - var err error - ips[i], err = c.GetInternalIP(ctx, nodes[i]) - return nil, errors.Wrapf(err, "IPs") + if err := c.Parallel(l, "", len(nodes), 0, func(i int) (*RunResultDetails, error) { + node := nodes[i] + res := &RunResultDetails{Node: node} + + res.Stdout, res.Err = c.GetInternalIP(ctx, node) + ips[i] = res.Stdout + return res, errors.Wrapf(res.Err, "IPs") }); err != nil { return nil, err } @@ -1315,27 +1439,33 @@ func (c *SyncedCluster) distributeLocalCertsTar( } display := c.Name + ": distributing certs" - return c.Parallel(l, display, len(nodes), 0, func(i int) ([]byte, error) { - sess, err := c.newSession(nodes[i]) + return c.Parallel(l, display, len(nodes), 0, func(i int) (*RunResultDetails, error) { + node := nodes[i] + sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() sess.SetStdin(bytes.NewReader(certsTar)) var cmd string if c.IsLocal() { - cmd = fmt.Sprintf("cd %s ; ", c.localVMDir(nodes[i])) + cmd = fmt.Sprintf("cd %s ; ", c.localVMDir(node)) } if stripComponents > 0 { cmd += fmt.Sprintf("tar --strip-components=%d -xf -", stripComponents) } else { cmd += "tar xf -" } - if out, err := sess.CombinedOutput(ctx, cmd); err != nil { - return nil, errors.Wrapf(err, "~ %s\n%s", cmd, out) + + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out + + if res.Err != nil { + return res, errors.Wrapf(res.Err, "~ %s\n%s", cmd, res.CombinedOut) } - return nil, nil + return res, nil }) } @@ -1501,10 +1631,10 @@ func (c *SyncedCluster) Put(ctx context.Context, l *logger.Logger, src, dest str return } - err = c.scp(from, to) - results <- result{i, err} + res, _ := scpWithDefaultRetry(l, from, to) + results <- result{i, res.Err} - if err != nil { + if res.Err != nil { // The copy failed. Re-add the original source. pushSource(srcIndex) } else { @@ -1602,9 +1732,9 @@ func (c *SyncedCluster) Put(ctx context.Context, l *logger.Logger, src, dest str // For example, if dest is "tpcc-test.logs" then the logs for each node will be // stored like: // -// tpcc-test.logs/1.logs/... -// tpcc-test.logs/2.logs/... -// ... +// tpcc-test.logs/1.logs/... +// tpcc-test.logs/2.logs/... +// ... // // Log file syncing uses rsync which attempts to be efficient when deciding // which files to update. The logs are merged by calling @@ -1856,8 +1986,8 @@ func (c *SyncedCluster) Get(l *logger.Logger, src, dest string) error { return } - err := c.scp(fmt.Sprintf("%s@%s:%s", c.user(c.Nodes[0]), c.Host(c.Nodes[i]), src), dest) - if err == nil { + res, _ := scpWithDefaultRetry(l, fmt.Sprintf("%s@%s:%s", c.user(c.Nodes[0]), c.Host(c.Nodes[i]), src), dest) + if res.Err == nil { // Make sure all created files and directories are world readable. // The CRDB process intentionally sets a 0007 umask (resulting in // non-world-readable files). This creates annoyances during CI @@ -1877,10 +2007,10 @@ func (c *SyncedCluster) Get(l *logger.Logger, src, dest string) error { } return nil } - err = filepath.Walk(dest, chmod) + res.Err = filepath.Walk(dest, chmod) } - results <- result{i, err} + results <- result{i, res.Err} }(i) } @@ -1937,7 +2067,7 @@ func (c *SyncedCluster) Get(l *logger.Logger, src, dest string) error { } } - if config.Quiet && l.File == nil { + if config.Quiet && l.File != nil { l.Printf("\n") linesMu.Lock() for i := range lines { @@ -1972,10 +2102,12 @@ func (c *SyncedCluster) pghosts( ctx context.Context, l *logger.Logger, nodes Nodes, ) (map[Node]string, error) { ips := make([]string, len(nodes)) - if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { - var err error - ips[i], err = c.GetInternalIP(ctx, nodes[i]) - return nil, errors.Wrapf(err, "pghosts") + if err := c.Parallel(l, "", len(nodes), 0, func(i int) (*RunResultDetails, error) { + node := nodes[i] + res := &RunResultDetails{Node: node} + res.Stdout, res.Err = c.GetInternalIP(ctx, node) + ips[i] = res.Stdout + return res, errors.Wrapf(res.Err, "pghosts") }); err != nil { return nil, err } @@ -2049,7 +2181,10 @@ func (c *SyncedCluster) SSH(ctx context.Context, l *logger.Logger, sshArgs, args return syscall.Exec(sshPath, allArgs, os.Environ()) } -func (c *SyncedCluster) scp(src, dest string) error { +// scp return type conforms to what runWithMaybeRetry expects. A nil error +// is always returned here since the only error that can happen is an scp error +// which we do want to be able to retry. +func scp(src, dest string) (*RunResultDetails, error) { args := []string{ "scp", "-r", "-C", "-o", "StrictHostKeyChecking=no", @@ -2059,9 +2194,12 @@ func (c *SyncedCluster) scp(src, dest string) error { cmd := exec.Command(args[0], args[1:]...) out, err := cmd.CombinedOutput() if err != nil { - return errors.Wrapf(err, "~ %s\n%s", strings.Join(args, " "), out) + err = errors.Wrapf(err, "~ %s\n%s", strings.Join(args, " "), out) } - return nil + + res := newRunResultDetails(-1, err) + res.CombinedOut = out + return res, nil } // ParallelResult captures the result of a user-defined function @@ -2078,7 +2216,10 @@ type ParallelResult struct { // // See ParallelE for more information. func (c *SyncedCluster) Parallel( - l *logger.Logger, display string, count, concurrency int, fn func(i int) ([]byte, error), + l *logger.Logger, + display string, + count, concurrency int, + fn func(i int) (*RunResultDetails, error), ) error { failed, err := c.ParallelE(l, display, count, concurrency, fn) if err != nil { @@ -2099,10 +2240,16 @@ func (c *SyncedCluster) Parallel( // `config.MaxConcurrency` if it is lower) in parallel. If `concurrency` is // 0, then it defaults to `count`. // +// The function returns a pointer to RunResultDetails as we may enrich +// the result with retry information (attempt number, wrapper error) +// // If err is non-nil, the slice of ParallelResults will contain the // results from any of the failed invocations. func (c *SyncedCluster) ParallelE( - l *logger.Logger, display string, count, concurrency int, fn func(i int) ([]byte, error), + l *logger.Logger, + display string, + count, concurrency int, + fn func(i int) (*RunResultDetails, error), ) ([]ParallelResult, error) { if concurrency == 0 || concurrency > count { concurrency = count @@ -2119,8 +2266,8 @@ func (c *SyncedCluster) ParallelE( startNext := func() { go func(i int) { defer wg.Done() - out, err := fn(i) - results <- ParallelResult{i, out, err} + res, err := runWithDefaultSSHRetry(l, func() (*RunResultDetails, error) { return fn(i) }) + results <- ParallelResult{i, res.CombinedOut, err} }(index) index++ } @@ -2199,36 +2346,30 @@ func (c *SyncedCluster) ParallelE( } if len(failed) > 0 { - return failed, errors.New("one or more parallel execution failure") + var err error + for _, res := range failed { + err = errors.CombineErrors(err, res.Err) + } + return failed, errors.Wrap(err, "parallel execution failure") } return nil, nil } // Init initializes the cluster. It does it through node 1 (as per TargetNodes) // to maintain parity with auto-init behavior of `roachprod start` (when -// --skip-init) is not specified. The implementation should be kept in -// sync with Start(). +// --skip-init) is not specified. func (c *SyncedCluster) Init(ctx context.Context, l *logger.Logger) error { - // See Start(). We reserve a few special operations for the first node, so we + // We reserve a few special operations for the first node, so we // strive to maintain the same here for interoperability. - const firstNodeIdx = 0 + const firstNodeIdx = 1 - l.Printf("%s: initializing cluster\n", c.Name) - initOut, err := c.initializeCluster(ctx, firstNodeIdx) - if err != nil { + if err := c.initializeCluster(ctx, l, firstNodeIdx); err != nil { return errors.WithDetail(err, "install.Init() failed: unable to initialize cluster.") } - if initOut != "" { - l.Printf(initOut) - } - l.Printf("%s: setting cluster settings", c.Name) - clusterSettingsOut, err := c.setClusterSettings(ctx, l, firstNodeIdx) - if err != nil { + if err := c.setClusterSettings(ctx, l, firstNodeIdx); err != nil { return errors.WithDetail(err, "install.Init() failed: unable to set cluster settings.") } - if clusterSettingsOut != "" { - l.Printf(clusterSettingsOut) - } + return nil } diff --git a/pkg/roachprod/install/cluster_synced_test.go b/pkg/roachprod/install/cluster_synced_test.go index fb25e09ea689..a5e58706a69b 100644 --- a/pkg/roachprod/install/cluster_synced_test.go +++ b/pkg/roachprod/install/cluster_synced_test.go @@ -12,7 +12,14 @@ package install import ( "fmt" + "io" "testing" + "time" + + "github.com/cockroachdb/cockroach/pkg/roachprod/logger" + "github.com/cockroachdb/cockroach/pkg/util/retry" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" ) // TestRoachprodEnv tests the roachprodEnvRegex and roachprodEnvValue methods. @@ -75,3 +82,92 @@ func TestRoachprodEnv(t *testing.T) { }) } } + +func TestRunWithMaybeRetry(t *testing.T) { + var testRetryOpts = retry.Options{ + InitialBackoff: 10 * time.Millisecond, + Multiplier: 2, + MaxBackoff: 1 * time.Second, + // This will run a total of 3 times `runWithMaybeRetry` + MaxRetries: 2, + } + + l := nilLogger() + + attempt := 0 + cases := []struct { + f func() (*RunResultDetails, error) + shouldRetryFn func(*RunResultDetails) bool + expectedAttempts int + shouldError bool + }{ + { // Happy path: no error, no retry required + f: func() (*RunResultDetails, error) { + return newResult(0), nil + }, + expectedAttempts: 1, + shouldError: false, + }, + { // Error, but not retry function specified + f: func() (*RunResultDetails, error) { + return newResult(1), nil + }, + expectedAttempts: 1, + shouldError: true, + }, + { // Error, with retries exhausted + f: func() (*RunResultDetails, error) { + return newResult(255), nil + }, + shouldRetryFn: func(d *RunResultDetails) bool { return d.RemoteExitStatus == 255 }, + expectedAttempts: 3, + shouldError: true, + }, + { // Eventual success after retries + f: func() (*RunResultDetails, error) { + attempt++ + if attempt == 3 { + return newResult(0), nil + } + return newResult(255), nil + }, + shouldRetryFn: func(d *RunResultDetails) bool { return d.RemoteExitStatus == 255 }, + expectedAttempts: 3, + shouldError: false, + }, + } + + for idx, tc := range cases { + attempt = 0 + t.Run(fmt.Sprintf("%d", idx+1), func(t *testing.T) { + res, _ := runWithMaybeRetry(l, testRetryOpts, tc.shouldRetryFn, tc.f) + + require.Equal(t, tc.shouldError, res.Err != nil) + require.Equal(t, tc.expectedAttempts, res.Attempt) + + if tc.shouldError && tc.expectedAttempts == 3 { + require.True(t, errors.Is(res.Err, ErrAfterRetry)) + } + }) + } +} + +func newResult(exitCode int) *RunResultDetails { + var err error + if exitCode != 0 { + err = errors.Newf("Error with exit code %v", exitCode) + } + return &RunResultDetails{RemoteExitStatus: exitCode, Err: err} +} + +func nilLogger() *logger.Logger { + lcfg := logger.Config{ + Stdout: io.Discard, + Stderr: io.Discard, + } + l, err := lcfg.NewLogger("" /* path */) + if err != nil { + panic(err) + } + return l +} diff --git a/pkg/roachprod/install/cockroach.go b/pkg/roachprod/install/cockroach.go index c6c03d485ea4..8ea9251a48a8 100644 --- a/pkg/roachprod/install/cockroach.go +++ b/pkg/roachprod/install/cockroach.go @@ -172,28 +172,30 @@ func (c *SyncedCluster) Start(ctx context.Context, l *logger.Logger, startOpts S } l.Printf("%s: starting nodes", c.Name) - return c.Parallel(l, "", len(nodes), parallelism, func(nodeIdx int) ([]byte, error) { + return c.Parallel(l, "", len(nodes), parallelism, func(nodeIdx int) (*RunResultDetails, error) { node := nodes[nodeIdx] + res := &RunResultDetails{Node: node} vers, err := getCockroachVersion(ctx, c, node) if err != nil { - return nil, err + res.Err = err + return res, err } - // NB: if cockroach started successfully, we ignore the output as it is // some harmless start messaging. if _, err := c.startNode(ctx, l, node, startOpts, vers); err != nil { - return nil, err + res.Err = err + return res, err } // Code that follows applies only for regular nodes. if startOpts.Target != StartDefault { - return nil, nil + return res, nil } // We reserve a few special operations (bootstrapping, and setting // cluster settings) for node 1. if node != 1 { - return nil, nil + return res, nil } // NB: The code blocks below are not parallelized, so it's safe for us @@ -203,34 +205,22 @@ func (c *SyncedCluster) Start(ctx context.Context, l *logger.Logger, startOpts S // 2. We don't init when invoking with `start-single-node`. if startOpts.SkipInit { - return nil, nil + return res, nil } shouldInit := !c.useStartSingleNode() if shouldInit { - l.Printf("%s: initializing cluster", c.Name) - initOut, err := c.initializeCluster(ctx, node) - if err != nil { - return nil, errors.WithDetail(err, "unable to initialize cluster") - } - - if initOut != "" { - l.Printf(initOut) + if err := c.initializeCluster(ctx, l, node); err != nil { + res.Err = err + return res, errors.Wrap(err, "failed to initialize cluster") } } - // We're sure to set cluster settings after having initialized the - // cluster. - - l.Printf("%s: setting cluster settings", c.Name) - clusterSettingsOut, err := c.setClusterSettings(ctx, l, node) - if err != nil { - return nil, errors.Wrap(err, "unable to set cluster settings") - } - if clusterSettingsOut != "" { - l.Printf(clusterSettingsOut) + if err := c.setClusterSettings(ctx, l, node); err != nil { + res.Err = err + return res, errors.Wrap(err, "failed to set cluster settings") } - return nil, nil + return res, nil }) } @@ -326,11 +316,11 @@ func (c *SyncedCluster) RunSQL(ctx context.Context, l *logger.Logger, args []str resultChan := make(chan result, len(c.Nodes)) display := fmt.Sprintf("%s: executing sql", c.Name) - if err := c.Parallel(l, display, len(c.Nodes), 0, func(nodeIdx int) ([]byte, error) { + if err := c.Parallel(l, display, len(c.Nodes), 0, func(nodeIdx int) (*RunResultDetails, error) { node := c.Nodes[nodeIdx] sess, err := c.newSession(node) if err != nil { - return nil, err + return newRunResultDetails(node, err), err } defer sess.Close() @@ -342,13 +332,15 @@ func (c *SyncedCluster) RunSQL(ctx context.Context, l *logger.Logger, args []str c.NodeURL("localhost", c.NodePort(node)) + " " + ssh.Escape(args) - out, err := sess.CombinedOutput(ctx, cmd) - if err != nil { - return nil, errors.Wrapf(err, "~ %s\n%s", cmd, out) - } + out, cmdErr := sess.CombinedOutput(ctx, cmd) + res := newRunResultDetails(node, cmdErr) + res.CombinedOut = out - resultChan <- result{node: node, output: string(out)} - return nil, nil + if res.Err != nil { + return res, errors.Wrapf(res.Err, "~ %s\n%s", cmd, res.CombinedOut) + } + resultChan <- result{node: node, output: string(res.CombinedOut)} + return res, nil }); err != nil { return err } @@ -637,38 +629,45 @@ func (c *SyncedCluster) maybeScaleMem(val int) int { return val } -func (c *SyncedCluster) initializeCluster(ctx context.Context, node Node) (string, error) { +func (c *SyncedCluster) initializeCluster(ctx context.Context, l *logger.Logger, node Node) error { + l.Printf("%s: initializing cluster\n", c.Name) initCmd := c.generateInitCmd(node) sess, err := c.newSession(node) if err != nil { - return "", err + return err } defer sess.Close() out, err := sess.CombinedOutput(ctx, initCmd) if err != nil { - return "", errors.Wrapf(err, "~ %s\n%s", initCmd, out) + return errors.Wrapf(err, "~ %s\n%s", initCmd, out) } - return strings.TrimSpace(string(out)), nil + + if out := strings.TrimSpace(string(out)); out != "" { + l.Printf(out) + } + return nil } -func (c *SyncedCluster) setClusterSettings( - ctx context.Context, l *logger.Logger, node Node, -) (string, error) { +func (c *SyncedCluster) setClusterSettings(ctx context.Context, l *logger.Logger, node Node) error { + l.Printf("%s: setting cluster settings", c.Name) clusterSettingCmd := c.generateClusterSettingCmd(l, node) sess, err := c.newSession(node) if err != nil { - return "", err + return err } defer sess.Close() out, err := sess.CombinedOutput(ctx, clusterSettingCmd) if err != nil { - return "", errors.Wrapf(err, "~ %s\n%s", clusterSettingCmd, out) + return errors.Wrapf(err, "~ %s\n%s", clusterSettingCmd, out) } - return strings.TrimSpace(string(out)), nil + if out := strings.TrimSpace(string(out)); out != "" { + l.Printf(out) + } + return nil } func (c *SyncedCluster) generateClusterSettingCmd(l *logger.Logger, node Node) string { diff --git a/pkg/roachprod/install/session.go b/pkg/roachprod/install/session.go index 345998df3d8f..ea2fd8fe9b17 100644 --- a/pkg/roachprod/install/session.go +++ b/pkg/roachprod/install/session.go @@ -16,17 +16,16 @@ import ( "os" "os/exec" "path/filepath" - "strings" "sync" "github.com/cockroachdb/cockroach/pkg/roachprod/config" + rperrors "github.com/cockroachdb/cockroach/pkg/roachprod/errors" "github.com/cockroachdb/errors" ) type session interface { CombinedOutput(ctx context.Context, cmd string) ([]byte, error) Run(ctx context.Context, cmd string) error - SetWithExitStatus(value bool) SetStdin(r io.Reader) SetStdout(w io.Writer) SetStderr(w io.Writer) @@ -41,9 +40,8 @@ type session interface { type remoteSession struct { *exec.Cmd - cancel func() - logfile string // captures ssh -vvv - withExitStatus bool + cancel func() + logfile string // captures ssh -vvv } func newRemoteSession(user, host string, logdir string) (*remoteSession, error) { @@ -56,7 +54,6 @@ func newRemoteSession(user, host string, logdir string) (*remoteSession, error) // fmt.Sprintf("ssh_%s_%s", host, timeutil.Now().Format(time.RFC3339)), // ) const logfile = "" - withExitStatus := false args := []string{ user + "@" + host, @@ -81,7 +78,7 @@ func newRemoteSession(user, host string, logdir string) (*remoteSession, error) args = append(args, sshAuthArgs()...) ctx, cancel := context.WithCancel(context.Background()) cmd := exec.CommandContext(ctx, "ssh", args...) - return &remoteSession{cmd, cancel, logfile, withExitStatus}, nil + return &remoteSession{cmd, cancel, logfile}, nil } func (s *remoteSession) errWithDebug(err error) error { @@ -93,13 +90,6 @@ func (s *remoteSession) errWithDebug(err error) error { } func (s *remoteSession) CombinedOutput(ctx context.Context, cmd string) ([]byte, error) { - if s.withExitStatus { - cmd = strings.TrimSpace(cmd) - if !strings.HasSuffix(cmd, ";") { - cmd += ";" - } - cmd += "echo -n 'LAST EXIT STATUS: '$?;" - } s.Cmd.Args = append(s.Cmd.Args, cmd) var b []byte @@ -115,22 +105,13 @@ func (s *remoteSession) CombinedOutput(ctx context.Context, cmd string) ([]byte, select { case <-ctx.Done(): s.Close() - err = ctx.Err() - break + return nil, ctx.Err() case <-commandFinished: - break + return b, rperrors.ClassifyCmdError(err) } - return b, err } func (s *remoteSession) Run(ctx context.Context, cmd string) error { - if s.withExitStatus { - cmd = strings.TrimSpace(cmd) - if !strings.HasSuffix(cmd, ";") { - cmd += ";" - } - cmd += "echo -n 'LAST EXIT STATUS: '$?;" - } s.Cmd.Args = append(s.Cmd.Args, cmd) var err error @@ -143,28 +124,15 @@ func (s *remoteSession) Run(ctx context.Context, cmd string) error { select { case <-ctx.Done(): s.Close() - err = ctx.Err() - break + return ctx.Err() case <-commandFinished: - break + return rperrors.ClassifyCmdError(err) } - return err } func (s *remoteSession) Start(cmd string) error { - if s.withExitStatus { - cmd = strings.TrimSpace(cmd) - if !strings.HasSuffix(cmd, ";") { - cmd += ";" - } - cmd += "echo -n 'LAST EXIT STATUS: '$?;" - } s.Cmd.Args = append(s.Cmd.Args, cmd) - return s.Cmd.Start() -} - -func (s *remoteSession) SetWithExitStatus(value bool) { - s.withExitStatus = value + return rperrors.ClassifyCmdError(s.Cmd.Start()) } func (s *remoteSession) SetStdin(r io.Reader) { @@ -211,25 +179,16 @@ func (s *remoteSession) Close() { type localSession struct { *exec.Cmd - cancel func() - withExitStatus bool + cancel func() } func newLocalSession() *localSession { ctx, cancel := context.WithCancel(context.Background()) - withExitStatus := false cmd := exec.CommandContext(ctx, "/bin/bash", "-c") - return &localSession{cmd, cancel, withExitStatus} + return &localSession{cmd, cancel} } func (s *localSession) CombinedOutput(ctx context.Context, cmd string) ([]byte, error) { - if s.withExitStatus { - cmd = strings.TrimSpace(cmd) - if !strings.HasSuffix(cmd, ";") { - cmd += ";" - } - cmd += "echo -n 'LAST EXIT STATUS: '$?;" - } s.Cmd.Args = append(s.Cmd.Args, cmd) var b []byte @@ -244,23 +203,13 @@ func (s *localSession) CombinedOutput(ctx context.Context, cmd string) ([]byte, select { case <-ctx.Done(): s.Close() - err = ctx.Err() - break + return nil, ctx.Err() case <-commandFinished: - break + return b, rperrors.ClassifyCmdError(err) } - - return b, err } func (s *localSession) Run(ctx context.Context, cmd string) error { - if s.withExitStatus { - cmd = strings.TrimSpace(cmd) - if !strings.HasSuffix(cmd, ";") { - cmd += ";" - } - cmd += "echo -n 'LAST EXIT STATUS: '$?;" - } s.Cmd.Args = append(s.Cmd.Args, cmd) var err error @@ -273,28 +222,15 @@ func (s *localSession) Run(ctx context.Context, cmd string) error { select { case <-ctx.Done(): s.Close() - err = ctx.Err() - break + return ctx.Err() case <-commandFinished: - break + return rperrors.ClassifyCmdError(err) } - return err } func (s *localSession) Start(cmd string) error { - if s.withExitStatus { - cmd = strings.TrimSpace(cmd) - if !strings.HasSuffix(cmd, ";") { - cmd += ";" - } - cmd += "echo -n 'LAST EXIT STATUS: '$?;" - } s.Cmd.Args = append(s.Cmd.Args, cmd) - return s.Cmd.Start() -} - -func (s *localSession) SetWithExitStatus(value bool) { - s.withExitStatus = value + return rperrors.ClassifyCmdError(s.Cmd.Start()) } func (s *localSession) SetStdin(r io.Reader) { diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index b5def775bba4..fbe18e057253 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -449,9 +449,12 @@ func IP( } } else { var err error - if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { - ips[i], err = c.GetInternalIP(ctx, nodes[i]) - return nil, err + if err := c.Parallel(l, "", len(nodes), 0, func(i int) (*install.RunResultDetails, error) { + node := nodes[i] + res := &install.RunResultDetails{Node: node} + res.Stdout, res.Err = c.GetInternalIP(ctx, node) + ips[i] = res.Stdout + return res, err }); err != nil { return nil, err } @@ -852,9 +855,12 @@ func PgURL( } } else { var err error - if err := c.Parallel(l, "", len(nodes), 0, func(i int) ([]byte, error) { - ips[i], err = c.GetInternalIP(ctx, nodes[i]) - return nil, err + if err := c.Parallel(l, "", len(nodes), 0, func(i int) (*install.RunResultDetails, error) { + node := nodes[i] + res := &install.RunResultDetails{Node: node} + res.Stdout, res.Err = c.GetInternalIP(ctx, node) + ips[i] = res.Stdout + return res, err }); err != nil { return nil, err } @@ -961,8 +967,9 @@ func Pprof(l *logger.Logger, clusterName string, opts PprofOpts) error { httpClient := httputil.NewClientWithTimeout(timeout) startTime := timeutil.Now().Unix() nodes := c.TargetNodes() - failed, err := c.ParallelE(l, description, len(nodes), 0, func(i int) ([]byte, error) { + failed, err := c.ParallelE(l, description, len(nodes), 0, func(i int) (*install.RunResultDetails, error) { node := nodes[i] + res := &install.RunResultDetails{Node: node} host := c.Host(node) port := c.NodeUIPort(node) scheme := "http" @@ -973,7 +980,8 @@ func Pprof(l *logger.Logger, clusterName string, opts PprofOpts) error { outputDir := filepath.Dir(outputFile) file, err := ioutil.TempFile(outputDir, ".pprof") if err != nil { - return nil, errors.Wrap(err, "create tmpfile for pprof download") + res.Err = errors.Wrap(err, "create tmpfile for pprof download") + return res, res.Err } defer func() { @@ -990,31 +998,37 @@ func Pprof(l *logger.Logger, clusterName string, opts PprofOpts) error { pprofURL := fmt.Sprintf("%s://%s:%d/%s", scheme, host, port, pprofPath) resp, err := httpClient.Get(context.Background(), pprofURL) if err != nil { - return nil, err + res.Err = err + return res, res.Err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, errors.Newf("unexpected status from pprof endpoint: %s", resp.Status) + res.Err = errors.Newf("unexpected status from pprof endpoint: %s", resp.Status) + return res, res.Err } if _, err := io.Copy(file, resp.Body); err != nil { - return nil, err + res.Err = err + return res, res.Err } if err := file.Sync(); err != nil { - return nil, err + res.Err = err + return res, res.Err } if err := file.Close(); err != nil { - return nil, err + res.Err = err + return res, res.Err } if err := os.Rename(file.Name(), outputFile); err != nil { - return nil, err + res.Err = err + return res, res.Err } mu.Lock() outputFiles = append(outputFiles, outputFile) mu.Unlock() - return nil, nil + return res, nil }) for _, s := range outputFiles { diff --git a/pkg/testutils/lint/passes/fmtsafe/fmtsafe.go b/pkg/testutils/lint/passes/fmtsafe/fmtsafe.go index f55046ac48ee..b73d041a5928 100644 --- a/pkg/testutils/lint/passes/fmtsafe/fmtsafe.go +++ b/pkg/testutils/lint/passes/fmtsafe/fmtsafe.go @@ -245,13 +245,13 @@ func checkCallExpr(pass *analysis.Pass, enclosingFnName string, call *ast.CallEx return } - pass.Reportf(call.Lparen, escNl("%s(): %s argument is not a constant expression"+Tip), - enclosingFnName, argType) + pass.Reportf(call.Lparen, escNl("%s(): %s argument is not a constant expression"+Tip), enclosingFnName, argType) } // Tip is exported for use in tests. var Tip = ` -Tip: use YourFuncf("descriptive prefix %%s", ...) or list new formatting wrappers in pkg/testutils/lint/passes/fmtsafe/functions.go.` +Tip: use YourFuncf("descriptive prefix %%s", ...) or list new formatting wrappers in pkg/testutils/lint/passes/fmtsafe/functions.go. +N.B. additional entry is required functions in the main package. See functions.go#requireConstFmt` func hasNoLintComment(pass *analysis.Pass, call *ast.CallExpr, idx int) bool { fPos, f := findContainingFile(pass, call) diff --git a/pkg/testutils/lint/passes/fmtsafe/functions.go b/pkg/testutils/lint/passes/fmtsafe/functions.go index 6e205334bcbf..55fd1dfbf718 100644 --- a/pkg/testutils/lint/passes/fmtsafe/functions.go +++ b/pkg/testutils/lint/passes/fmtsafe/functions.go @@ -30,8 +30,14 @@ var requireConstMsg = map[string]bool{ "(*github.com/cockroachdb/cockroach/pkg/sql.optPlanningCtx).log": true, } -// requireConstFmt records functions for which the string arg -// before the final ellipsis must be a constant string. +/* +requireConstFmt records functions for which the string arg +before the final ellipsis must be a constant string. + +Definitions surrounded in parentheses are functions attached to a struct. +For functions defined in the main package, a *second* entry is required +in the form (main.yourStruct).yourFuncF +*/ var requireConstFmt = map[string]bool{ // Logging things. "log.Printf": true, @@ -83,6 +89,17 @@ var requireConstFmt = map[string]bool{ "(*github.com/cockroachdb/cockroach/pkg/util/grpcutil.grpcLogger).Errorf": true, "(*github.com/cockroachdb/cockroach/pkg/util/grpcutil.grpcLogger).Fatalf": true, + // Both of these signatures need to be included for the linter to not flag + // roachtest testImpl.addFailure since it is in the main package + "(*github.com/cockroachdb/cockroach/pkg/cmd/roachtest.testImpl).addFailure": true, + "(*main.testImpl).addFailure": true, + + "(*main.testImpl).Fatalf": true, + "(*github.com/cockroachdb/cockroach/pkg/cmd/roachtest.testImpl).Fatalf": true, + + "(*main.testImpl).Errorf": true, + "(*github.com/cockroachdb/cockroach/pkg/cmd/roachtest.testImpl).Errorf": true, + "(*github.com/cockroachdb/cockroach/pkg/kv/kvserver.raftLogger).Debugf": true, "(*github.com/cockroachdb/cockroach/pkg/kv/kvserver.raftLogger).Infof": true, "(*github.com/cockroachdb/cockroach/pkg/kv/kvserver.raftLogger).Warningf": true, diff --git a/pkg/util/retry/retry.go b/pkg/util/retry/retry.go index 7e635a42b112..55912b6a4c0e 100644 --- a/pkg/util/retry/retry.go +++ b/pkg/util/retry/retry.go @@ -157,6 +157,11 @@ func (r *Retry) NextCh() <-chan time.Time { return time.After(r.retryIn()) } +// CurrentAttempt returns the current attempt +func (r *Retry) CurrentAttempt() int { + return r.currentAttempt +} + // Do invokes the closure according to the retry options until it returns // success or no more retries are possible. Always returns an error unless the // return is prompted by a successful invocation of `fn`.