Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: cockroachdb/cockroach
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: e887bf08da5afe0da9261d8a88bf0efc8efe57c8
Choose a base ref
..
head repository: cockroachdb/cockroach
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 408e227b2d23fd72869b8190a71b12674a0fdf23
Choose a head ref
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -343,6 +343,7 @@
/pkg/cmd/returncheck/ @cockroachdb/dev-inf
/pkg/cmd/roachprod/ @cockroachdb/test-eng
/pkg/cmd/roachprod/vm/azure/auth.go @cockroachdb/test-eng @cockroachdb/prodsec
/pkg/cmd/roachprod-microbench/ @cockroachdb/test-eng
/pkg/cmd/roachprod-stress/ @cockroachdb/test-eng
/pkg/cmd/roachtest/ @cockroachdb/test-eng
/pkg/cmd/label-merged-pr/ @cockroachdb/dev-inf
9 changes: 9 additions & 0 deletions pkg/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -134,6 +134,7 @@ ALL_TESTS = [
"//pkg/cmd/reduce/reduce/reducesql:reducesql_test",
"//pkg/cmd/reduce/reduce:reduce_test",
"//pkg/cmd/release:release_test",
"//pkg/cmd/roachprod-microbench:roachprod-microbench_test",
"//pkg/cmd/roachtest/clusterstats:clusterstats_test",
"//pkg/cmd/roachtest/roachtestutil/mixedversion:mixedversion_test",
"//pkg/cmd/roachtest/tests:tests_test",
@@ -1006,6 +1007,11 @@ GO_TARGETS = [
"//pkg/cmd/release:release_test",
"//pkg/cmd/returncheck:returncheck",
"//pkg/cmd/returncheck:returncheck_lib",
"//pkg/cmd/roachprod-microbench/cluster:cluster",
"//pkg/cmd/roachprod-microbench/google:google",
"//pkg/cmd/roachprod-microbench:roachprod-microbench",
"//pkg/cmd/roachprod-microbench:roachprod-microbench_lib",
"//pkg/cmd/roachprod-microbench:roachprod-microbench_test",
"//pkg/cmd/roachprod-stress:roachprod-stress",
"//pkg/cmd/roachprod-stress:roachprod-stress_lib",
"//pkg/cmd/roachprod:roachprod",
@@ -2496,6 +2502,9 @@ GET_X_DATA_TARGETS = [
"//pkg/cmd/release:get_x_data",
"//pkg/cmd/returncheck:get_x_data",
"//pkg/cmd/roachprod:get_x_data",
"//pkg/cmd/roachprod-microbench:get_x_data",
"//pkg/cmd/roachprod-microbench/cluster:get_x_data",
"//pkg/cmd/roachprod-microbench/google:get_x_data",
"//pkg/cmd/roachprod-stress:get_x_data",
"//pkg/cmd/roachtest:get_x_data",
"//pkg/cmd/roachtest/cluster:get_x_data",
71 changes: 36 additions & 35 deletions pkg/cmd/dev/build.go
Original file line number Diff line number Diff line change
@@ -73,41 +73,42 @@ func makeBuildCmd(runE func(cmd *cobra.Command, args []string) error) *cobra.Com

// buildTargetMapping maintains shorthands that map 1:1 with bazel targets.
var buildTargetMapping = map[string]string{
"bazel-remote": bazelRemoteTarget,
"buildifier": "@com_github_bazelbuild_buildtools//buildifier:buildifier",
"buildozer": "@com_github_bazelbuild_buildtools//buildozer:buildozer",
"cockroach": "//pkg/cmd/cockroach:cockroach",
"cockroach-sql": "//pkg/cmd/cockroach-sql:cockroach-sql",
"cockroach-oss": "//pkg/cmd/cockroach-oss:cockroach-oss",
"cockroach-short": "//pkg/cmd/cockroach-short:cockroach-short",
"crlfmt": "@com_github_cockroachdb_crlfmt//:crlfmt",
"dev": devTarget,
"docgen": "//pkg/cmd/docgen:docgen",
"docs-issue-gen": "//pkg/cmd/docs-issue-generation:docs-issue-generation",
"execgen": "//pkg/sql/colexec/execgen/cmd/execgen:execgen",
"gofmt": "@com_github_cockroachdb_gostdlib//cmd/gofmt:gofmt",
"goimports": "@com_github_cockroachdb_gostdlib//x/tools/cmd/goimports:goimports",
"label-merged-pr": "//pkg/cmd/label-merged-pr:label-merged-pr",
"geos": geosTarget,
"langgen": "//pkg/sql/opt/optgen/cmd/langgen:langgen",
"libgeos": geosTarget,
"obsservice": "//pkg/obsservice/cmd/obsservice:obsservice",
"optgen": "//pkg/sql/opt/optgen/cmd/optgen:optgen",
"optfmt": "//pkg/sql/opt/optgen/cmd/optfmt:optfmt",
"oss": "//pkg/cmd/cockroach-oss:cockroach-oss",
"reduce": "//pkg/cmd/reduce:reduce",
"roachprod": "//pkg/cmd/roachprod:roachprod",
"roachprod-stress": "//pkg/cmd/roachprod-stress:roachprod-stress",
"roachtest": "//pkg/cmd/roachtest:roachtest",
"short": "//pkg/cmd/cockroach-short:cockroach-short",
"smith": "//pkg/cmd/smith:smith",
"smithcmp": "//pkg/cmd/smithcmp:smithcmp",
"smithtest": "//pkg/cmd/smithtest:smithtest",
"staticcheck": "@co_honnef_go_tools//cmd/staticcheck:staticcheck",
"stress": stressTarget,
"swagger": "@com_github_go_swagger_go_swagger//cmd/swagger:swagger",
"tests": "//pkg:all_tests",
"workload": "//pkg/cmd/workload:workload",
"bazel-remote": bazelRemoteTarget,
"buildifier": "@com_github_bazelbuild_buildtools//buildifier:buildifier",
"buildozer": "@com_github_bazelbuild_buildtools//buildozer:buildozer",
"cockroach": "//pkg/cmd/cockroach:cockroach",
"cockroach-sql": "//pkg/cmd/cockroach-sql:cockroach-sql",
"cockroach-oss": "//pkg/cmd/cockroach-oss:cockroach-oss",
"cockroach-short": "//pkg/cmd/cockroach-short:cockroach-short",
"crlfmt": "@com_github_cockroachdb_crlfmt//:crlfmt",
"dev": devTarget,
"docgen": "//pkg/cmd/docgen:docgen",
"docs-issue-gen": "//pkg/cmd/docs-issue-generation:docs-issue-generation",
"execgen": "//pkg/sql/colexec/execgen/cmd/execgen:execgen",
"gofmt": "@com_github_cockroachdb_gostdlib//cmd/gofmt:gofmt",
"goimports": "@com_github_cockroachdb_gostdlib//x/tools/cmd/goimports:goimports",
"label-merged-pr": "//pkg/cmd/label-merged-pr:label-merged-pr",
"geos": geosTarget,
"langgen": "//pkg/sql/opt/optgen/cmd/langgen:langgen",
"libgeos": geosTarget,
"obsservice": "//pkg/obsservice/cmd/obsservice:obsservice",
"optgen": "//pkg/sql/opt/optgen/cmd/optgen:optgen",
"optfmt": "//pkg/sql/opt/optgen/cmd/optfmt:optfmt",
"oss": "//pkg/cmd/cockroach-oss:cockroach-oss",
"reduce": "//pkg/cmd/reduce:reduce",
"roachprod": "//pkg/cmd/roachprod:roachprod",
"roachprod-stress": "//pkg/cmd/roachprod-stress:roachprod-stress",
"roachprod-microbench": "//pkg/cmd/roachprod-microbench:roachprod-microbench",
"roachtest": "//pkg/cmd/roachtest:roachtest",
"short": "//pkg/cmd/cockroach-short:cockroach-short",
"smith": "//pkg/cmd/smith:smith",
"smithcmp": "//pkg/cmd/smithcmp:smithcmp",
"smithtest": "//pkg/cmd/smithtest:smithtest",
"staticcheck": "@co_honnef_go_tools//cmd/staticcheck:staticcheck",
"stress": stressTarget,
"swagger": "@com_github_go_swagger_go_swagger//cmd/swagger:swagger",
"tests": "//pkg:all_tests",
"workload": "//pkg/cmd/workload:workload",
}

// allBuildTargets is a sorted list of all the available build targets.
148 changes: 142 additions & 6 deletions pkg/cmd/docs-issue-generation/docs_issue_generation.go
Original file line number Diff line number Diff line change
@@ -150,7 +150,42 @@ type cockroachCommit struct {
MessageBody string `json:"messageBody"`
}

type epicIssueRefInfo struct {
epicRefs map[string]int
epicNone bool
issueCloseRefs map[string]int
issueInformRefs map[string]int
isBugFix bool
}

// Regex components for finding and validating issue and epic references in a string
var (
ghIssuePart = `(#\d+)` // e.g., #12345
ghIssueRepoPart = `([\w.-]+[/][\w.-]+#\d+)` // e.g., cockroachdb/cockroach#12345
ghURLPart = `(https://github.com/[-a-z0-9]+/[-._a-z0-9/]+/issues/\d+)` // e.g., https://github.com/cockroachdb/cockroach/issues/12345
jiraIssuePart = `([[:alpha:]]+-\d+)` // e.g., DOC-3456
jiraURLPart = "https://cockroachlabs.atlassian.net/browse/" + jiraIssuePart // e.g., https://cockroachlabs.atlassian.net/browse/DOC-3456
issueRefPart = ghIssuePart + "|" + ghIssueRepoPart + "|" + ghURLPart + "|" + jiraIssuePart + "|" + jiraURLPart
afterRefPart = `[,.;]?(?:[ \t\n\r]+|$)`
)

// RegExes of each issue part
var (
ghIssuePartRE = regexp.MustCompile(ghIssuePart)
ghIssueRepoPartRE = regexp.MustCompile(ghIssueRepoPart)
ghURLPartRE = regexp.MustCompile(ghURLPart)
jiraIssuePartRE = regexp.MustCompile(jiraIssuePart)
jiraURLPartRE = regexp.MustCompile(jiraURLPart)
)

// Fully composed regexs used to match strings.
var (
fixIssueRefRE = regexp.MustCompile(`(?im)(?i:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:(?:` + issueRefPart + `)` + afterRefPart + ")+")
informIssueRefRE = regexp.MustCompile(`(?im)(?:part of|see also|informs):?\s+(?:(?:` + issueRefPart + `)` + afterRefPart + ")+")
epicRefRE = regexp.MustCompile(`(?im)epic:?\s+(?:(?:` + jiraIssuePart + "|" + jiraURLPart + `)` + afterRefPart + ")+")
epicNoneRE = regexp.MustCompile(`(?im)epic:?\s+(?:(none)` + afterRefPart + ")+")
githubJiraIssueRefRE = regexp.MustCompile(issueRefPart)
jiraIssueRefRE = regexp.MustCompile(jiraIssuePart + "|" + jiraURLPart)
releaseNoteNoneRE = regexp.MustCompile(`(?i)release note:? [nN]one`)
allRNRE = regexp.MustCompile(`(?i)release note:? \(.*`)
nonBugFixRNRE = regexp.MustCompile(`(?i)release note:? \(([^b]|b[^u]|bu[^g]|bug\S|bug [^f]|bug f[^i]|bug fi[^x]).*`)
@@ -576,13 +611,109 @@ func searchCockroachPRCommitsSingle(
return pageInfo.HasNextPage, pageInfo.EndCursor, result, nil
}

// extractStringsFromMessage takes in a commit message or PR body as well as two regular expressions. The first
// regular expression checks for a valid formatted epic or issue reference. If one is found, it searches that exact
// string for the individual issue references. The output is a map where the key is each epic or issue ref and the
// value is the count of references of that ref.
func extractStringsFromMessage(
message string, firstMatch, secondMatch *regexp.Regexp,
) map[string]int {
ids := map[string]int{}
if allMatches := firstMatch.FindAllString(message, -1); len(allMatches) > 0 {
for _, x := range allMatches {
matches := secondMatch.FindAllString(x, -1)
for _, match := range matches {
ids[match]++
}
}
}
return ids
}

func extractFixIssueIDs(message string) map[string]int {
return extractStringsFromMessage(message, fixIssueRefRE, githubJiraIssueRefRE)
}

func extractInformIssueIDs(message string) map[string]int {
return extractStringsFromMessage(message, informIssueRefRE, githubJiraIssueRefRE)
}

func extractEpicIDs(message string) map[string]int {
return extractStringsFromMessage(message, epicRefRE, jiraIssueRefRE)
}

func containsEpicNone(message string) bool {
if allMatches := epicNoneRE.FindAllString(message, -1); len(allMatches) > 0 {
return true
}
return false
}

func containsBugFix(message string) bool {
if allMatches := bugFixRNRE.FindAllString(message, -1); len(allMatches) > 0 {
return true
}
return false
}

func getUrlFromRef(ref string) string {
if ghURLPartRE.MatchString(ref) || jiraURLPartRE.MatchString(ref) {
return ref
} else if ghIssueRepoPartRE.MatchString(ref) {
split := strings.Split(ref, "#")
return "https://github.com/" + split[0] + "/issues/" + split[1]
} else if ghIssuePartRE.MatchString(ref) {
return "https://github.com/cockroachdb/cockroach/issues/" + strings.Replace(ref, "#", "", 1)
} else if jiraIssuePartRE.MatchString(ref) {
return "https://cockroachlabs.atlassian.net/browse/" + ref
} else {
return "Malformed epic/issue ref (" + ref + ")"
}
}

func extractIssueEpicRefs(prBody, commitBody string) string {
refInfo := epicIssueRefInfo{
epicRefs: extractEpicIDs(prBody + "\n" + commitBody),
epicNone: containsEpicNone(prBody + "\n" + commitBody),
issueCloseRefs: extractFixIssueIDs(prBody + "\n" + commitBody),
issueInformRefs: extractInformIssueIDs(prBody + "\n" + commitBody),
isBugFix: containsBugFix(prBody + "\n" + commitBody),
}
var builder strings.Builder
if len(refInfo.epicRefs) > 0 {
builder.WriteString("Epic:")
for x := range refInfo.epicRefs {
builder.WriteString(" " + getUrlFromRef(x))
}
builder.WriteString("\n")
}
if len(refInfo.issueCloseRefs) > 0 {
builder.WriteString("Fixes:")
for x := range refInfo.issueCloseRefs {
builder.WriteString(" " + getUrlFromRef(x))
}
builder.WriteString("\n")
}
if len(refInfo.issueInformRefs) > 0 {
builder.WriteString("Informs:")
for x := range refInfo.issueInformRefs {
builder.WriteString(" " + getUrlFromRef(x))
}
builder.WriteString("\n")
}
if refInfo.epicNone && builder.Len() == 0 {
builder.WriteString("Epic: none\n")
}
return builder.String()
}

// getIssues takes a list of commits from GitHub as well as the PR number associated with those commits and outputs a
// formatted list of docs issues with valid release notes
func constructDocsIssues(prs []cockroachPR) []docsIssue {
var result []docsIssue
for _, pr := range prs {
for _, commit := range pr.Commits {
rns := formatReleaseNotes(commit.MessageBody, pr.Number, commit.Sha)
rns := formatReleaseNotes(commit.MessageBody, pr.Number, pr.Body, commit.Sha)
for i, rn := range rns {
x := docsIssue{
sourceCommitSha: commit.Sha,
@@ -610,12 +741,15 @@ func formatTitle(title string, prNumber int, index int, totalLength int) string
}

// formatReleaseNotes generates a list of docsIssue bodies for the docs repo based on a given CRDB sha
func formatReleaseNotes(message string, prNumber int, crdbSha string) []string {
func formatReleaseNotes(
commitMessage string, prNumber int, prBody string, crdbSha string,
) []string {
rnBodySlice := []string{}
if releaseNoteNoneRE.MatchString(message) {
if releaseNoteNoneRE.MatchString(commitMessage) {
return rnBodySlice
}
splitString := strings.Split(message, "\n")
epicIssueRefs := extractIssueEpicRefs(prBody, commitMessage)
splitString := strings.Split(commitMessage, "\n")
releaseNoteLines := []string{}
var rnBody string
for _, x := range splitString {
@@ -626,9 +760,10 @@ func formatReleaseNotes(message string, prNumber int, crdbSha string) []string {
rnBody = fmt.Sprintf(
"Related PR: https://github.com/cockroachdb/cockroach/pull/%s\n"+
"Commit: https://github.com/cockroachdb/cockroach/commit/%s\n"+
"\n---\n\n%s",
"%s\n---\n\n%s",
strconv.Itoa(prNumber),
crdbSha,
epicIssueRefs,
strings.Join(releaseNoteLines, "\n"),
)
rnBodySlice = append(rnBodySlice, strings.TrimSuffix(rnBody, "\n"))
@@ -643,9 +778,10 @@ func formatReleaseNotes(message string, prNumber int, crdbSha string) []string {
rnBody = fmt.Sprintf(
"Related PR: https://github.com/cockroachdb/cockroach/pull/%s\n"+
"Commit: https://github.com/cockroachdb/cockroach/commit/%s\n"+
"\n---\n\n%s",
"%s\n---\n\n%s",
strconv.Itoa(prNumber),
crdbSha,
epicIssueRefs,
strings.Join(releaseNoteLines, "\n"),
)
rnBodySlice = append(rnBodySlice, strings.TrimSuffix(rnBody, "\n"))
Loading