diff --git a/.chloggen/add-githubgen.yaml b/.chloggen/add-githubgen.yaml new file mode 100644 index 00000000..27eabfb2 --- /dev/null +++ b/.chloggen/add-githubgen.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'new_component' + +# The name of the component, or a single word describing the area of concern, (e.g. crosslink) +component: githubgen + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Moved githubgen tool here from open-telemetry/opentelemetry-collector-contrib + +# One or more tracking issues related to the change +issues: [639] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 55eda4d1..cc6d3b14 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,7 +20,7 @@ updates: interval: weekly day: sunday - package-ecosystem: gomod - directory: /checkdoc + directory: /checkfile labels: - dependencies - go @@ -29,7 +29,7 @@ updates: interval: weekly day: sunday - package-ecosystem: gomod - directory: /checkfile + directory: /chloggen labels: - dependencies - go @@ -38,7 +38,7 @@ updates: interval: weekly day: sunday - package-ecosystem: gomod - directory: /chloggen + directory: /crosslink labels: - dependencies - go @@ -47,7 +47,7 @@ updates: interval: weekly day: sunday - package-ecosystem: gomod - directory: /crosslink + directory: /dbotconf labels: - dependencies - go @@ -56,7 +56,7 @@ updates: interval: weekly day: sunday - package-ecosystem: gomod - directory: /dbotconf + directory: /githubgen labels: - dependencies - go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c876152e..46b7bec7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,10 @@ jobs: with: go-version: ${{ env.DEFAULT_GO_VERSION }} check-latest: true + - name: Checkout Repo uses: actions/checkout@v4 + - name: Module cache uses: actions/cache@v4 env: @@ -30,17 +32,34 @@ jobs: with: path: ~/go/pkg/mod key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Tools cache + id: cache-tools uses: actions/cache@v4 env: cache-name: go-tools-cache with: - path: ~/.tools + path: .tools key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('./internal/tools/**') }} + + - name: Install tools + if: steps.cache-tools.outputs.cache-hit != 'true' + run: make tools + + - name: Add .exe to tools on Windows + if: matrix.os == 'windows-latest' + shell: bash + run: | + for file in ./.tools/*; do + cp "$file" "${file}.exe" + done + - name: Run linters run: make multimod-verify dependabot-check license-check lint + - name: Build run: make build + - name: Check clean repository run: make check-clean-work-tree @@ -55,12 +74,15 @@ jobs: with: go-version: ${{ env.DEFAULT_GO_VERSION }} check-latest: true + - name: Checkout Repo uses: actions/checkout@v4 + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Module cache uses: actions/cache@v4 env: @@ -68,6 +90,7 @@ jobs: with: path: ~/go/pkg/mod key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Run tests with race detector run: make test-race @@ -79,12 +102,15 @@ jobs: with: go-version: ${{ env.DEFAULT_GO_VERSION }} check-latest: true + - name: Checkout Repo uses: actions/checkout@v4 + - name: Setup Environment run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Module cache uses: actions/cache@v4 env: @@ -92,6 +118,7 @@ jobs: with: path: ~/go/pkg/mod key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }} + - name: Run coverage tests run: | make test-coverage @@ -99,6 +126,7 @@ jobs: cp coverage.out $TEST_RESULTS cp coverage.txt $TEST_RESULTS cp coverage.html $TEST_RESULTS + - name: Upload coverage report uses: codecov/codecov-action@v5.1.1 with: @@ -106,6 +134,7 @@ jobs: fail_ci_if_error: true verbose: true token: ${{ secrets.CODECOV_TOKEN }} + - name: Store coverage test output uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index 1d9545f4..54eb6df6 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +SHELL_CASE_EXP = case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) echo "true";; esac; +UNIX_SHELL_ON_WINDOWS := $(shell $(SHELL_CASE_EXP)) + +ifeq ($(UNIX_SHELL_ON_WINDOWS),true) + # The "sed" transformation below is needed on Windows, since commands like `go list -f '{{ .Dir }}'` + # return Windows paths and such paths are incompatible with other *nix tools, like `find`, + # used by the Makefile shell. + # The backslash needs to be doubled so its passed correctly to the shell. + NORMALIZE_DIRS = sed -e 's/^/\\//' -e 's/://' -e 's/\\\\/\\//g' | sort + PATH_SEPARATOR=; +else + NORMALIZE_DIRS = sort + PATH_SEPARATOR=: +endif + TOOLS_MOD_DIR := ./internal/tools # All source code and documents. Used in spell check. @@ -20,7 +35,7 @@ ALL_DOCS := $(shell find . -name '*.md' -type f | sort) ALL_GO_MOD_DIRS := $(filter-out $(TOOLS_MOD_DIR), $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort)) ALL_COVERAGE_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | egrep -v '^$(TOOLS_MOD_DIR)' | sort) -GO = go +GO ?= go TIMEOUT = 60 .DEFAULT_GOAL := precommit @@ -60,17 +75,23 @@ $(TOOLS)/chloggen: PACKAGE=go.opentelemetry.io/build-tools/chloggen GOVULNCHECK = $(TOOLS)/govulncheck $(TOOLS)/govulncheck: PACKAGE=golang.org/x/vuln/cmd/govulncheck +MOQ = $(TOOLS)/moq + $(TOOLS)/moq: PACKAGE=github.com/matryer/moq + .PHONY: tools -tools: $(DBOTCONF) $(GOLANGCI_LINT) $(MISSPELL) $(MULTIMOD) $(CROSSLINK) $(CHLOGGEN) $(GOVULNCHECK) +tools: $(DBOTCONF) $(GOLANGCI_LINT) $(MISSPELL) $(MULTIMOD) $(CROSSLINK) $(CHLOGGEN) $(GOVULNCHECK) $(MOQ) # Build +UPDATED_PATH := $(shell echo $(TOOLS) | $(NORMALIZE_DIRS)) +NEW_PATH := $(UPDATED_PATH)$(PATH_SEPARATOR)$(PATH) + .PHONY: generate build generate: set -e; for dir in $(ALL_GO_MOD_DIRS); do \ echo "$(GO) generate $${dir}/..."; \ (cd "$${dir}" && \ - PATH="$(TOOLS):$${PATH}" $(GO) generate ./...); \ + PATH="$(UPDATED_PATH)$(PATH_SEPARATOR)$${PATH}" $(GO) generate ./...); \ done build: generate diff --git a/githubgen/README.md b/githubgen/README.md new file mode 100644 index 00000000..581985ed --- /dev/null +++ b/githubgen/README.md @@ -0,0 +1,37 @@ +# githubgen + +This executable is used to generate `.github/CODEOWNERS` and +`.github/ALLOWLIST` files. + +It reads status metadata from `metadata.yaml` files located throughout the +repository. + +It checks that codeowners are known members of the OpenTelemetry organization. + +## Usage + +```shell +$> ./githubgen +``` + +The equivalent of: + +```shell +$> GITHUB_TOKEN= githubgen --folder . [--allowlist cmd/githubgen/allowlist.txt] +``` + +## Checking codeowners against OpenTelemetry membership via GitHub API + +To authenticate, set the environment variable `GITHUB_TOKEN` to a PAT token. +If a PAT is not available you can use the `--skipgithub` flag to avoid checking +for membership in the GitHub organization. + +For each codeowner, the script will check if the user is registered as a member +of the OpenTelemetry organization. + +If any codeowner is missing, it will stop and print names of missing codeowners. + +These can be added to allowlist.txt as a workaround. + +If a codeowner is present in allowlist.txt and also a member of the +OpenTelemetry organization, the script will error out. diff --git a/githubgen/codeowners.go b/githubgen/codeowners.go new file mode 100644 index 00000000..e23eb77e --- /dev/null +++ b/githubgen/codeowners.go @@ -0,0 +1,233 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/google/go-github/v66/github" + + "go.opentelemetry.io/build-tools/githubgen/datatype" +) + +const allowlistHeader = `# Code generated by githubgen. DO NOT EDIT. +##################################################### +# +# List of components in OpenTelemetry Collector Contrib +# waiting on owners to be assigned +# +##################################################### +# +# Learn about membership in OpenTelemetry community: +# https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md +# +# +# Learn about CODEOWNERS file format: +# https://help.github.com/en/articles/about-code-owners +# + +## +# NOTE: New components MUST have one or more codeowners. Add codeowners to the component metadata.yaml and run make gengithub +## + +## COMMON & SHARED components +internal/common + +` + +const unmaintainedHeader = ` + +## UNMAINTAINED components + +` + +const codeownersHeader = `# Code generated by githubgen. DO NOT EDIT. +##################################################### +# +# List of codeowners for OpenTelemetry Collector Contrib +# +##################################################### +# +# Learn about membership in OpenTelemetry community: +# https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md +# +# +# Learn about CODEOWNERS file format: +# https://help.github.com/en/articles/about-code-owners +# + +* @open-telemetry/collector-contrib-approvers +` + +const distributionCodeownersHeader = ` +##################################################### +# +# List of distribution maintainers for OpenTelemetry Collector Contrib +# +##################################################### +` + +type codeownersGenerator struct { + skipGithub bool +} + +func (cg *codeownersGenerator) Generate(data datatype.GithubData) error { + allowlistData, err := os.ReadFile(data.AllowlistFilePath) + if err != nil { + return err + } + allowlistLines := strings.Split(string(allowlistData), "\n") + + allowlist := make(map[string]struct{}, len(allowlistLines)) + unusedAllowlist := make(map[string]struct{}, len(allowlistLines)) + for _, line := range allowlistLines { + if line == "" { + continue + } + allowlist[line] = struct{}{} + unusedAllowlist[line] = struct{}{} + } + var missingCodeowners []string + var duplicateCodeowners []string + members, err := cg.getGithubMembers() + if err != nil { + return err + } + for _, codeowner := range data.Codeowners { + _, present := members[codeowner] + + if !present { + _, allowed := allowlist[codeowner] + delete(unusedAllowlist, codeowner) + allowed = allowed || strings.HasPrefix(codeowner, "open-telemetry/") + if !allowed { + missingCodeowners = append(missingCodeowners, codeowner) + } + } else if _, ok := allowlist[codeowner]; ok { + duplicateCodeowners = append(duplicateCodeowners, codeowner) + } + } + if len(missingCodeowners) > 0 && !cg.skipGithub { + sort.Strings(missingCodeowners) + return fmt.Errorf("codeowners are not members: %s", strings.Join(missingCodeowners, ", ")) + } + if len(duplicateCodeowners) > 0 { + sort.Strings(duplicateCodeowners) + return fmt.Errorf("codeowners members duplicate in allowlist: %s", strings.Join(duplicateCodeowners, ", ")) + } + if len(unusedAllowlist) > 0 { + var unused []string + for k := range unusedAllowlist { + unused = append(unused, k) + } + sort.Strings(unused) + return fmt.Errorf("unused members in allowlist: %s", strings.Join(unused, ", ")) + } + + codeowners := codeownersHeader + deprecatedList := "## DEPRECATED components\n" + unmaintainedList := "\n## UNMAINTAINED components\n" + + unmaintainedCodeowners := unmaintainedHeader + currentFirstSegment := "" +LOOP: + for _, key := range data.Folders { + m := data.Components[key] + for stability := range m.Status.Stability { + if stability == unmaintainedStatus { + unmaintainedList += key + "/\n" + unmaintainedCodeowners += fmt.Sprintf("%s/%s @open-telemetry/collector-contrib-approvers \n", key, strings.Repeat(" ", data.MaxLength-len(key))) + continue LOOP + } + if stability == "deprecated" && (m.Status.Codeowners == nil || len(m.Status.Codeowners.Active) == 0) { + deprecatedList += key + "/\n" + } + } + + if m.Status.Codeowners != nil { + parts := strings.Split(key, string(os.PathSeparator)) + firstSegment := parts[0] + if firstSegment != currentFirstSegment { + currentFirstSegment = firstSegment + codeowners += "\n" + } + owners := "" + for _, owner := range m.Status.Codeowners.Active { + owners += " " + owners += "@" + owner + } + codeowners += fmt.Sprintf("%s/%s @open-telemetry/collector-contrib-approvers%s\n", key, strings.Repeat(" ", data.MaxLength-len(key)), owners) + } + } + + codeowners += distributionCodeownersHeader + longestName := 0 + for _, dist := range data.Distributions { + if longestName < len(dist.Name) { + longestName = len(dist.Name) + } + } + + for _, dist := range data.Distributions { + var maintainers []string + for _, m := range dist.Maintainers { + maintainers = append(maintainers, fmt.Sprintf("@%s", m)) + } + codeowners += fmt.Sprintf("reports/distributions/%s.yaml%s @open-telemetry/collector-contrib-approvers %s\n", dist.Name, strings.Repeat(" ", longestName-len(dist.Name)), strings.Join(maintainers, " ")) + } + + err = os.WriteFile(filepath.Join(".github", "CODEOWNERS"), []byte(codeowners+unmaintainedCodeowners), 0o600) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(".github", "ALLOWLIST"), []byte(allowlistHeader+deprecatedList+unmaintainedList), 0o600) + if err != nil { + return err + } + return nil +} + +func (cg *codeownersGenerator) getGithubMembers() (map[string]struct{}, error) { + if cg.skipGithub { + // don't try to get organization members if no token is expected + return map[string]struct{}{}, nil + } + githubToken := os.Getenv("GITHUB_TOKEN") + if githubToken == "" { + return nil, fmt.Errorf("Set the environment variable `GITHUB_TOKEN` to a PAT token to authenticate") + } + client := github.NewTokenClient(context.Background(), githubToken) + var allUsers []*github.User + pageIndex := 0 + for { + users, resp, err := client.Organizations.ListMembers(context.Background(), "open-telemetry", + &github.ListMembersOptions{ + PublicOnly: false, + ListOptions: github.ListOptions{ + PerPage: 50, + Page: pageIndex, + }, + }, + ) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if len(users) == 0 { + break + } + allUsers = append(allUsers, users...) + pageIndex++ + } + + usernames := make(map[string]struct{}, len(allUsers)) + for _, u := range allUsers { + usernames[*u.Login] = struct{}{} + } + return usernames, nil +} diff --git a/githubgen/datatype/data.go b/githubgen/datatype/data.go new file mode 100644 index 00000000..9e84415f --- /dev/null +++ b/githubgen/datatype/data.go @@ -0,0 +1,47 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package datatype + +//go:generate moq -pkg fake -skip-ensure -out ./fake/mock_generator.go . Generator:MockGenerator +type Generator interface { + Generate(data GithubData) error +} + +type GithubData struct { + Folders []string + Codeowners []string + AllowlistFilePath string + MaxLength int + Components map[string]Metadata + Distributions []DistributionData +} + +type Codeowners struct { + // Active codeowners + Active []string `mapstructure:"active"` + // Emeritus codeowners + Emeritus []string `mapstructure:"emeritus"` +} + +type Status struct { + Stability map[string][]string `mapstructure:"stability"` + Distributions []string `mapstructure:"distributions"` + Class string `mapstructure:"class"` + Warnings []string `mapstructure:"warnings"` + Codeowners *Codeowners `mapstructure:"codeowners"` +} +type Metadata struct { + // Type of the component. + Type string `mapstructure:"type"` + // Type of the parent component (applicable to subcomponents). + Parent string `mapstructure:"parent"` + // Status information for the component. + Status *Status `mapstructure:"status"` +} + +type DistributionData struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Maintainers []string `yaml:"maintainers,omitempty"` +} diff --git a/githubgen/datatype/fake/mock_generator.go b/githubgen/datatype/fake/mock_generator.go new file mode 100644 index 00000000..4c3afd3a --- /dev/null +++ b/githubgen/datatype/fake/mock_generator.go @@ -0,0 +1,71 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package fake + +import ( + "go.opentelemetry.io/build-tools/githubgen/datatype" + "sync" +) + +// MockGenerator is a mock implementation of datatype.Generator. +// +// func TestSomethingThatUsesGenerator(t *testing.T) { +// +// // make and configure a mocked datatype.Generator +// mockedGenerator := &MockGenerator{ +// GenerateFunc: func(data datatype.GithubData) error { +// panic("mock out the Generate method") +// }, +// } +// +// // use mockedGenerator in code that requires datatype.Generator +// // and then make assertions. +// +// } +type MockGenerator struct { + // GenerateFunc mocks the Generate method. + GenerateFunc func(data datatype.GithubData) error + + // calls tracks calls to the methods. + calls struct { + // Generate holds details about calls to the Generate method. + Generate []struct { + // Data is the data argument value. + Data datatype.GithubData + } + } + lockGenerate sync.RWMutex +} + +// Generate calls GenerateFunc. +func (mock *MockGenerator) Generate(data datatype.GithubData) error { + if mock.GenerateFunc == nil { + panic("MockGenerator.GenerateFunc: method is nil but Generator.Generate was just called") + } + callInfo := struct { + Data datatype.GithubData + }{ + Data: data, + } + mock.lockGenerate.Lock() + mock.calls.Generate = append(mock.calls.Generate, callInfo) + mock.lockGenerate.Unlock() + return mock.GenerateFunc(data) +} + +// GenerateCalls gets all the calls that were made to Generate. +// Check the length with: +// +// len(mockedGenerator.GenerateCalls()) +func (mock *MockGenerator) GenerateCalls() []struct { + Data datatype.GithubData +} { + var calls []struct { + Data datatype.GithubData + } + mock.lockGenerate.RLock() + calls = mock.calls.Generate + mock.lockGenerate.RUnlock() + return calls +} diff --git a/githubgen/distributions.go b/githubgen/distributions.go new file mode 100644 index 00000000..c554db2b --- /dev/null +++ b/githubgen/distributions.go @@ -0,0 +1,64 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "gopkg.in/yaml.v3" + + "go.opentelemetry.io/build-tools/githubgen/datatype" +) + +type distributionsGenerator struct{} + +type distOutput struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Maintainers []string `yaml:"maintainers"` + Components map[string][]string `yaml:"components"` +} + +func (cg *distributionsGenerator) Generate(data datatype.GithubData) error { + for _, dist := range data.Distributions { + components := map[string][]string{} + for _, c := range data.Components { + inDistro := false + for _, componentDistro := range c.Status.Distributions { + if dist.Name == componentDistro { + inDistro = true + break + } + } + if inDistro { + array, ok := components[c.Status.Class] + if !ok { + array = []string{} + } + components[c.Status.Class] = append(array, c.Type) + } + } + for _, comps := range components { + sort.Strings(comps) + } + output := distOutput{ + Name: dist.Name, + URL: dist.URL, + Maintainers: dist.Maintainers, + Components: components, + } + b, err := yaml.Marshal(output) + if err != nil { + return nil + } + err = os.WriteFile(filepath.Join("reports", "distributions", fmt.Sprintf("%s.yaml", dist.Name)), b, 0o600) + if err != nil { + return nil + } + } + return nil +} diff --git a/githubgen/go.mod b/githubgen/go.mod new file mode 100644 index 00000000..8e2448fc --- /dev/null +++ b/githubgen/go.mod @@ -0,0 +1,25 @@ +module go.opentelemetry.io/build-tools/githubgen + +go 1.22.0 + +require ( + github.com/google/go-github/v66 v66.0.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/confmap v1.20.0 + go.opentelemetry.io/collector/confmap/provider/fileprovider v1.20.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.2 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect +) diff --git a/githubgen/go.sum b/githubgen/go.sum new file mode 100644 index 00000000..eaaa57a2 --- /dev/null +++ b/githubgen/go.sum @@ -0,0 +1,47 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd/D/9+M= +github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ= +github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/collector/confmap v1.20.0 h1:ARfOwmkKxFOud1njl03yAHQ30+uenlzqCO6LBYamDTE= +go.opentelemetry.io/collector/confmap v1.20.0/go.mod h1:DMpd9Ay/ffls3JoQBQ73vWeRsz1rNuLbwjo6WtjSQus= +go.opentelemetry.io/collector/confmap/provider/fileprovider v1.20.0 h1:wWxvQ7wj+1O9yDGM5m1HPEz8FJewAHAUWadAAi0KVbM= +go.opentelemetry.io/collector/confmap/provider/fileprovider v1.20.0/go.mod h1:/5HWIPjGYk8IUurs1CZUSjGaSsaQyJsfR8+Zs5rGJ6Y= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/githubgen/issuetemplates.go b/githubgen/issuetemplates.go new file mode 100644 index 00000000..369dad30 --- /dev/null +++ b/githubgen/issuetemplates.go @@ -0,0 +1,70 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "bytes" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "go.opentelemetry.io/build-tools/githubgen/datatype" +) + +const ( + startComponentList = `# Start Collector components list` + endComponentList = `# End Collector components list` +) + +func folderToShortName(folder string) string { + if folder == "internal/coreinternal" { + return "internal/core" + } + path := strings.Split(folder, "/") + switch path[0] { + case "receiver", "exporter", "extension", "processor", "connector": + path[1] = strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(path[1], "internal"), "extension"), "exporter"), "connector"), "processor"), "receiver") + path[len(path)-1] = strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(path[len(path)-1], "internal"), "extension"), "exporter"), "connector"), "processor"), "receiver") + default: + } + + return strings.Join(path, "/") +} + +type issueTemplatesGenerator struct{} + +func (itg *issueTemplatesGenerator) Generate(data datatype.GithubData) error { + keys := map[string]struct{}{} + for _, f := range data.Folders { + keys[folderToShortName(f)] = struct{}{} + } + shortNames := make([]string, 0, len(keys)) + for k := range keys { + shortNames = append(shortNames, k) + } + sort.Strings(shortNames) + replacement := []byte(startComponentList + "\n - " + strings.Join(shortNames, "\n - ") + "\n " + endComponentList) + issuesFolder := filepath.Join(".github", "ISSUE_TEMPLATE") + entries, err := os.ReadDir(issuesFolder) + if err != nil { + return err + } + for _, e := range entries { + templateContents, err := os.ReadFile(filepath.Join(issuesFolder, e.Name())) // nolint: gosec + if err != nil { + return err + } + matchOldContent := regexp.MustCompile("(?s)" + startComponentList + ".*" + endComponentList) + oldContent := matchOldContent.FindSubmatch(templateContents) + if len(oldContent) > 0 { + templateContents = bytes.ReplaceAll(templateContents, oldContent[0], replacement) + err = os.WriteFile(filepath.Join(issuesFolder, e.Name()), templateContents, 0o600) + if err != nil { + return err + } + } + } + return nil +} diff --git a/githubgen/main.go b/githubgen/main.go new file mode 100644 index 00000000..454ec902 --- /dev/null +++ b/githubgen/main.go @@ -0,0 +1,153 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "flag" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "slices" + + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/confmap/provider/fileprovider" + "gopkg.in/yaml.v3" + + "go.opentelemetry.io/build-tools/githubgen/datatype" +) + +const unmaintainedStatus = "unmaintained" + +// Generates files specific to GitHub according to status datatype.Metadata: +// .github/CODEOWNERS +// .github/ALLOWLIST +// .github/ISSUE_TEMPLATES/*.yaml (list of components) +// reports/distributions/* +func main() { + folder := flag.String("folder", ".", "folder investigated for codeowners") + allowlistFilePath := flag.String("allowlist", "cmd/githubgen/allowlist.txt", "path to a file containing an allowlist of members outside the OpenTelemetry organization") + skipGithubCheck := flag.Bool("skipgithub", false, "skip checking GitHub membership check for CODEOWNERS datatype.Generator") + flag.Parse() + var generators []datatype.Generator + for _, arg := range flag.Args() { + switch arg { + case "issue-templates": + generators = append(generators, &issueTemplatesGenerator{}) + case "codeowners": + generators = append(generators, &codeownersGenerator{skipGithub: *skipGithubCheck}) + case "distributions": + generators = append(generators, &distributionsGenerator{}) + default: + panic(fmt.Sprintf("Unknown datatype.Generator: %s", arg)) + } + } + if len(generators) == 0 { + generators = []datatype.Generator{&issueTemplatesGenerator{}, &codeownersGenerator{skipGithub: *skipGithubCheck}} + } + + distributions, err := getDistributions(*folder) + if err != nil { + log.Fatal(err) + } + + if err = run(*folder, *allowlistFilePath, generators, distributions); err != nil { + log.Fatal(err) + } +} + +func loadMetadata(filePath string) (datatype.Metadata, error) { + cp, err := fileprovider.NewFactory().Create(confmap.ProviderSettings{}).Retrieve(context.Background(), "file:"+filePath, nil) + if err != nil { + return datatype.Metadata{}, err + } + + conf, err := cp.AsConf() + if err != nil { + return datatype.Metadata{}, err + } + + md := datatype.Metadata{} + if err := conf.Unmarshal(&md, confmap.WithIgnoreUnused()); err != nil { + return md, err + } + + return md, nil +} + +func run(folder string, allowlistFilePath string, generators []datatype.Generator, distros []datatype.DistributionData) error { + components := map[string]datatype.Metadata{} + var foldersList []string + maxLength := 0 + var allCodeowners []string + err := filepath.Walk(folder, func(path string, info fs.FileInfo, _ error) error { + if info.Name() == "metadata.yaml" { + m, err := loadMetadata(path) + if err != nil { + return err + } + if m.Status == nil { + return nil + } + currentFolder := filepath.Dir(path) + + components[currentFolder] = m + foldersList = append(foldersList, currentFolder) + + for stability := range m.Status.Stability { + if stability == unmaintainedStatus { + // do not account for unmaintained status to change the max length of the component line. + return nil + } + } + if m.Status.Codeowners == nil { + return fmt.Errorf("component %q has no codeowners section", currentFolder) + } + + allCodeowners = append(allCodeowners, m.Status.Codeowners.Active...) + if len(currentFolder) > maxLength { + maxLength = len(currentFolder) + } + } + return nil + }) + if err != nil { + return err + } + + slices.Sort(foldersList) + slices.Sort(allCodeowners) + allCodeowners = slices.Compact(allCodeowners) + + data := datatype.GithubData{ + Folders: foldersList, + Codeowners: allCodeowners, + AllowlistFilePath: allowlistFilePath, + MaxLength: maxLength, + Components: components, + Distributions: distros, + } + + for _, g := range generators { + if err = g.Generate(data); err != nil { + return err + } + } + return nil +} + +func getDistributions(folder string) ([]datatype.DistributionData, error) { + var distributions []datatype.DistributionData + dd, err := os.ReadFile(filepath.Join(folder, "distributions.yaml")) // nolint: gosec + if err != nil { + return nil, err + } + err = yaml.Unmarshal(dd, &distributions) + if err != nil { + return nil, err + } + return distributions, nil +} diff --git a/githubgen/main_test.go b/githubgen/main_test.go new file mode 100644 index 00000000..94a201b9 --- /dev/null +++ b/githubgen/main_test.go @@ -0,0 +1,59 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/build-tools/githubgen/datatype" + "go.opentelemetry.io/build-tools/githubgen/datatype/fake" +) + +func Test_run(t *testing.T) { + + type args struct { + folder string + allowlistFilePath string + generators fake.MockGenerator + distributions []datatype.DistributionData + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "codeowners", + args: args{ + folder: ".", + allowlistFilePath: "cmd/githubgen/allowlist.txt", + generators: fake.MockGenerator{ + GenerateFunc: func(_ datatype.GithubData) error { + return nil + }, + }, + distributions: []datatype.DistributionData{ + { + Name: "my-distro", + URL: "some-url", + Maintainers: nil, + }, + }, + }, + wantErr: false, + }, + } + + // nolint:govet + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := run(tt.args.folder, tt.args.allowlistFilePath, []datatype.Generator{&tt.args.generators}, tt.args.distributions); (err != nil) != tt.wantErr { + t.Errorf("run() error = %v, wantErr %v", err, tt.wantErr) + } + require.Equal(t, len(tt.args.generators.GenerateCalls()), 1) + }) + } +} diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 7c7d9ee3..eab810bc 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -1,11 +1,12 @@ module go.opentelemetry.io/build-tools/internal/tools -go 1.22.1 +go 1.22.5 require ( github.com/client9/misspell v0.3.4 github.com/gogo/protobuf v1.3.2 github.com/golangci/golangci-lint v1.62.2 + github.com/matryer/moq v0.4.0 go.opentelemetry.io/build-tools/chloggen v0.7.0 go.opentelemetry.io/build-tools/crosslink v0.7.0 go.opentelemetry.io/build-tools/dbotconf v0.7.0 diff --git a/internal/tools/go.sum b/internal/tools/go.sum index cc91f220..62930244 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -402,6 +402,8 @@ github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 h1:gWg6ZQ4JhDfJPqlo2 github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/matryer/moq v0.4.0 h1:HsZIdEsj8+9nE940WW7FFxMgrgSxGfMkNXhVTHUhfMU= +github.com/matryer/moq v0.4.0/go.mod h1:kUfalaLk7TcyXhrhonBYQ2Ewun63+/xGbZ7/MzzzC4Y= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= diff --git a/internal/tools/tools.go b/internal/tools/tools.go index 55eacb37..a84edd00 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -26,4 +26,5 @@ import ( _ "go.opentelemetry.io/build-tools/dbotconf" _ "go.opentelemetry.io/build-tools/multimod" _ "golang.org/x/vuln/cmd/govulncheck" + _ "github.com/matryer/moq" ) diff --git a/versions.yaml b/versions.yaml index 9304efad..974c1988 100644 --- a/versions.yaml +++ b/versions.yaml @@ -21,6 +21,7 @@ module-sets: - go.opentelemetry.io/build-tools/chloggen - go.opentelemetry.io/build-tools/crosslink - go.opentelemetry.io/build-tools/dbotconf + - go.opentelemetry.io/build-tools/githubgen - go.opentelemetry.io/build-tools/gotmpl - go.opentelemetry.io/build-tools/issuegenerator - go.opentelemetry.io/build-tools/multimod