From 9e4a906d78aab000516d2eb58a8a14fdab7569d1 Mon Sep 17 00:00:00 2001 From: Adriel Perkins Date: Wed, 5 Jun 2024 05:27:56 -0400 Subject: [PATCH] [receiver/gitproviderreceiver] add branch time, commit, and line metrics (#32812) **Description:** Adds the following branch based metrics. * git.repository.branch.time * git.repository.branch.commit.aheadby.count * git.repository.branch.commit.behindby.count * git.repository.branch.line.deletion.count * git.repository.branch.line.addition.count Additionally brings this receiver very close to the source version in the Liatrio Distro. Brings the testing up to par as well. After this pull request is merged, I'd like to move this component to alpha status so it can be built within the contrib binary and used by others. Once in alpha, I'd like to bring over the newest metric (repository.cve.count), bring over the GitLab scraper, and potentially propose a new component I developed to work in-tandem with this receiver, the GitHub App Authentication extension. **Link to tracking Issue:** #22028 **Testing:** Brings testing inline with source and include scrape testing. **Documentation:** Updated the documentation to reflect the additional metrics. --------- Co-authored-by: Curtis Robert --- .chloggen/branch-metrics.yaml | 34 ++ receiver/gitproviderreceiver/README.md | 61 ++- receiver/gitproviderreceiver/documentation.md | 89 +++- receiver/gitproviderreceiver/go.mod | 31 +- receiver/gitproviderreceiver/go.sum | 41 +- .../internal/metadata/generated_config.go | 20 + .../metadata/generated_config_test.go | 10 + .../internal/metadata/generated_metrics.go | 322 +++++++++++++- .../metadata/generated_metrics_test.go | 124 +++++- .../internal/metadata/testdata/config.yaml | 20 + .../internal/scraper/githubscraper/README.md | 109 +++++ .../githubscraper/generated_graphql.go | 420 ++++++++++++++++++ .../scraper/githubscraper/genqlient.graphql | 35 ++ .../scraper/githubscraper/github_scraper.go | 50 ++- .../githubscraper/github_scraper_test.go | 198 +++++++++ .../internal/scraper/githubscraper/helpers.go | 127 +++++- .../scraper/githubscraper/helpers_test.go | 265 ++++++++++- .../testdata/scraper/expected_happy_path.yaml | 165 +++++++ .../testdata/scraper/expected_no_repos.yaml | 23 + receiver/gitproviderreceiver/metadata.yaml | 55 ++- 20 files changed, 2088 insertions(+), 111 deletions(-) create mode 100644 .chloggen/branch-metrics.yaml create mode 100644 receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md create mode 100644 receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_happy_path.yaml create mode 100644 receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_no_repos.yaml diff --git a/.chloggen/branch-metrics.yaml b/.chloggen/branch-metrics.yaml new file mode 100644 index 000000000000..763962791f63 --- /dev/null +++ b/.chloggen/branch-metrics.yaml @@ -0,0 +1,34 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: gitproviderreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Adds branch commit and line based metrics + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [22028] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Adds the following branch based metrics. + * git.repository.branch.time + * git.repository.branch.commit.aheadby.count + * git.repository.branch.commit.behindby.count + * git.repository.branch.line.deletion.count + * git.repository.branch.line.addition.count + + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/gitproviderreceiver/README.md b/receiver/gitproviderreceiver/README.md index 5b551de58634..0933a27a8914 100644 --- a/receiver/gitproviderreceiver/README.md +++ b/receiver/gitproviderreceiver/README.md @@ -19,35 +19,17 @@ The current default set of metrics common across all vendors can be found in [do These default metrics can be used as leading indicators to the DORA metrics; helping provide insight into modern-day engineering practices. -## GitHub Metrics - -The current metrics available via scraping from GitHub are: - -- [x] Repository count -- [ ] Repository branch time -- [x] Repository branch count -- [x] Repository contributor count -- [x] Repository pull request open time -- [x] Repository pull request time to merge -- [ ] Repository pull request deployment time -- [x] Repository pull request time to approval -- [x] Repository pull request count | stores an attribute of `pull_request.state` equal to `open` or `merged` - -> Note: Some metrics may be disabled by default and have to be explicitly enabled. -> For example, the repository contributor count metric is one such metric. This is -> because this metric relies on the REST API which is subject to lower rate limits. - ## Getting Started The collection interval is common to all scrapers and is set to 30 seconds by default. > Note: Generally speaking, if the vendor allows for anonymous API calls, then you > won't have to configure any authentication, but you may only see public repositories -> and organizations. +> and organizations. You may run into significantly more rate limiting. ```yaml gitprovider: - collection_interval: #default = 30s + collection_interval: #default = 30s recommended 300s scrapers: : : @@ -71,7 +53,7 @@ receivers: git.repository.contributor.count: enabled: true github_org: myfancyorg - search_query: "org:myfancyorg topic:o11yalltheway" #optional query override, defaults to "{org,user}:" + search_query: "org:myfancyorg topic:o11yalltheway" #Recommended optional query override, defaults to "{org,user}:" endpoint: "https://selfmanagedenterpriseserver.com" auth: authenticator: bearertokenauth/github @@ -87,7 +69,44 @@ service: This receiver is developed upstream in the [liatrio-otel-collector distribution](https://github.com/liatrio/liatrio-otel-collector) where a quick start exists with an [example config](https://github.com/liatrio/liatrio-otel-collector/blob/main/config/config.yaml) +A Grafana Dashboard exists on the marketplace for this receiver and can be +found [here](https://grafana.com/grafana/dashboards/20976-engineering-effectiveness-metrics/). + The available scrapers are: | Scraper | Description | |----------|-------------------------| | [github] | Git Metrics from [GitHub](https://github.com/) | + +## GitHub Scraper + +> Important: +> * The GitHub scraper does not emit metrics for branches that have not had +> changes since creation from the default branch (trunk). +> * Due to GitHub API limitations, it is possible for the branch time metric to +> change when rebases occur, recreating the commits with new timestamps. + + +For additional context on GitHub scraper limitations and inner workings please +see the [GitHub Scraper README][ghsread]. + +[ghsread]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md#github-limitations + + +The current metrics available via scraping from GitHub are: + +- [x] Repository count +- [x] Repository contributor count +- [x] Repository branch count +- [x] Repository branch time +- [x] Repository branch commit aheadby count +- [x] Repository branch commit behindby count +- [x] Repository branch line addition count +- [x] Repository branch line deletion count +- [x] Repository pull request open time +- [x] Repository pull request time to merge +- [x] Repository pull request time to approval +- [x] Repository pull request count | stores an attribute of `pull_request.state` equal to `open` or `merged` + +> Note: Some metrics may be disabled by default and have to be explicitly enabled. +> For example, the repository contributor count metric is one such metric. This is +> because this metric relies on the REST API which is subject to lower rate limits. diff --git a/receiver/gitproviderreceiver/documentation.md b/receiver/gitproviderreceiver/documentation.md index ffdb07c23b62..5b40b97f79b6 100644 --- a/receiver/gitproviderreceiver/documentation.md +++ b/receiver/gitproviderreceiver/documentation.md @@ -12,9 +12,39 @@ metrics: enabled: false ``` +### git.repository.branch.commit.aheadby.count + +The number of commits a branch is ahead of the default branch (trunk). + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {commit} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| repository.name | The name of a Git repository | Any Str | +| branch.name | The name of a Git branch | Any Str | + +### git.repository.branch.commit.behindby.count + +The number of commits a branch is behind the default branch (trunk). + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {commit} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| repository.name | The name of a Git repository | Any Str | +| branch.name | The name of a Git branch | Any Str | + ### git.repository.branch.count -Number of branches in a repository +The number of branches in a repository. | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | @@ -26,9 +56,54 @@ Number of branches in a repository | ---- | ----------- | ------ | | repository.name | The name of a Git repository | Any Str | +### git.repository.branch.line.addition.count + +The number of lines added in a branch relative to the default branch (trunk). + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {line} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| repository.name | The name of a Git repository | Any Str | +| branch.name | The name of a Git branch | Any Str | + +### git.repository.branch.line.deletion.count + +The number of lines deleted in a branch relative to the default branch (trunk). + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {line} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| repository.name | The name of a Git repository | Any Str | +| branch.name | The name of a Git branch | Any Str | + +### git.repository.branch.time + +Time a branch created from the default branch (trunk) has existed. + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| s | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| repository.name | The name of a Git repository | Any Str | +| branch.name | The name of a Git branch | Any Str | + ### git.repository.count -Number of repositories in an organization +The number of repositories in an organization. | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | @@ -36,7 +111,7 @@ Number of repositories in an organization ### git.repository.pull_request.count -The number of pull requests in a repository, categorized by their state (either open or merged) +The number of pull requests in a repository, categorized by their state (either open or merged). | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | @@ -51,7 +126,7 @@ The number of pull requests in a repository, categorized by their state (either ### git.repository.pull_request.time_open -The amount of time a pull request has been open +The amount of time a pull request has been open. | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | @@ -66,7 +141,7 @@ The amount of time a pull request has been open ### git.repository.pull_request.time_to_approval -The amount of time it took a pull request to go from open to approved +The amount of time it took a pull request to go from open to approved. | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | @@ -81,7 +156,7 @@ The amount of time it took a pull request to go from open to approved ### git.repository.pull_request.time_to_merge -The amount of time it took a pull request to go from open to merged +The amount of time it took a pull request to go from open to merged. | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | @@ -106,7 +181,7 @@ metrics: ### git.repository.contributor.count -Total number of unique contributors to a repository +The number of unique contributors to a repository. | Unit | Metric Type | Value Type | | ---- | ----------- | ---------- | diff --git a/receiver/gitproviderreceiver/go.mod b/receiver/gitproviderreceiver/go.mod index 82bd0a652d9d..ae5e864d0271 100644 --- a/receiver/gitproviderreceiver/go.mod +++ b/receiver/gitproviderreceiver/go.mod @@ -6,6 +6,8 @@ require ( github.com/Khan/genqlient v0.7.0 github.com/google/go-cmp v0.6.0 github.com/google/go-github/v62 v62.0.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.102.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.102.0 go.opentelemetry.io/collector/config/confighttp v0.102.0 @@ -29,10 +31,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect @@ -46,25 +48,26 @@ require ( github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.1 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.102.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.15.0 // indirect - github.com/rs/cors v1.10.1 // indirect + github.com/rs/cors v1.11.0 // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/vektah/gqlparser/v2 v2.5.11 // indirect + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/tklauser/numcpus v0.7.0 // indirect + github.com/vektah/gqlparser/v2 v2.5.12 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector v0.102.0 // indirect @@ -107,12 +110,18 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden diff --git a/receiver/gitproviderreceiver/go.sum b/receiver/gitproviderreceiver/go.sum index 67b606633b04..634d1f301a46 100644 --- a/receiver/gitproviderreceiver/go.sum +++ b/receiver/gitproviderreceiver/go.sum @@ -26,14 +26,15 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= -github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-viper/mapstructure/v2 v2.0.0 h1:dhn8MZ1gZ0mzeodTG3jt5Vj/o87xZKuNAprG2mQfMfc= +github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -93,8 +94,9 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 h1:1KuuSOy4ZNgW0KA2oYIngXVFhQcXxhLqCVK7cBcldkk= +github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -119,8 +121,8 @@ github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= -github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= +github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= @@ -145,12 +147,14 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8= -github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= +github.com/vektah/gqlparser/v2 v2.5.12 h1:COMhVVnql6RoaF7+aTBWiTADdpLGyZWU3K/NwW0ph98= +github.com/vektah/gqlparser/v2 v2.5.12/go.mod h1:WQQjFc+I1YIzoPvZBhUQX7waZgg3pMLi0r8KymvAE2w= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= @@ -301,15 +305,16 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -331,8 +336,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 h1:Q2RxlXqh1cgzzUgV261vBO2jI5R/3DD1J2pM0nI4NhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/receiver/gitproviderreceiver/internal/metadata/generated_config.go b/receiver/gitproviderreceiver/internal/metadata/generated_config.go index cbd5d30d1920..0a9d445c6c93 100644 --- a/receiver/gitproviderreceiver/internal/metadata/generated_config.go +++ b/receiver/gitproviderreceiver/internal/metadata/generated_config.go @@ -28,7 +28,12 @@ func (ms *MetricConfig) Unmarshal(parser *confmap.Conf) error { // MetricsConfig provides config for gitprovider metrics. type MetricsConfig struct { + GitRepositoryBranchCommitAheadbyCount MetricConfig `mapstructure:"git.repository.branch.commit.aheadby.count"` + GitRepositoryBranchCommitBehindbyCount MetricConfig `mapstructure:"git.repository.branch.commit.behindby.count"` GitRepositoryBranchCount MetricConfig `mapstructure:"git.repository.branch.count"` + GitRepositoryBranchLineAdditionCount MetricConfig `mapstructure:"git.repository.branch.line.addition.count"` + GitRepositoryBranchLineDeletionCount MetricConfig `mapstructure:"git.repository.branch.line.deletion.count"` + GitRepositoryBranchTime MetricConfig `mapstructure:"git.repository.branch.time"` GitRepositoryContributorCount MetricConfig `mapstructure:"git.repository.contributor.count"` GitRepositoryCount MetricConfig `mapstructure:"git.repository.count"` GitRepositoryPullRequestCount MetricConfig `mapstructure:"git.repository.pull_request.count"` @@ -39,9 +44,24 @@ type MetricsConfig struct { func DefaultMetricsConfig() MetricsConfig { return MetricsConfig{ + GitRepositoryBranchCommitAheadbyCount: MetricConfig{ + Enabled: true, + }, + GitRepositoryBranchCommitBehindbyCount: MetricConfig{ + Enabled: true, + }, GitRepositoryBranchCount: MetricConfig{ Enabled: true, }, + GitRepositoryBranchLineAdditionCount: MetricConfig{ + Enabled: true, + }, + GitRepositoryBranchLineDeletionCount: MetricConfig{ + Enabled: true, + }, + GitRepositoryBranchTime: MetricConfig{ + Enabled: true, + }, GitRepositoryContributorCount: MetricConfig{ Enabled: false, }, diff --git a/receiver/gitproviderreceiver/internal/metadata/generated_config_test.go b/receiver/gitproviderreceiver/internal/metadata/generated_config_test.go index d65d550e1452..54a93e4abf8b 100644 --- a/receiver/gitproviderreceiver/internal/metadata/generated_config_test.go +++ b/receiver/gitproviderreceiver/internal/metadata/generated_config_test.go @@ -25,7 +25,12 @@ func TestMetricsBuilderConfig(t *testing.T) { name: "all_set", want: MetricsBuilderConfig{ Metrics: MetricsConfig{ + GitRepositoryBranchCommitAheadbyCount: MetricConfig{Enabled: true}, + GitRepositoryBranchCommitBehindbyCount: MetricConfig{Enabled: true}, GitRepositoryBranchCount: MetricConfig{Enabled: true}, + GitRepositoryBranchLineAdditionCount: MetricConfig{Enabled: true}, + GitRepositoryBranchLineDeletionCount: MetricConfig{Enabled: true}, + GitRepositoryBranchTime: MetricConfig{Enabled: true}, GitRepositoryContributorCount: MetricConfig{Enabled: true}, GitRepositoryCount: MetricConfig{Enabled: true}, GitRepositoryPullRequestCount: MetricConfig{Enabled: true}, @@ -43,7 +48,12 @@ func TestMetricsBuilderConfig(t *testing.T) { name: "none_set", want: MetricsBuilderConfig{ Metrics: MetricsConfig{ + GitRepositoryBranchCommitAheadbyCount: MetricConfig{Enabled: false}, + GitRepositoryBranchCommitBehindbyCount: MetricConfig{Enabled: false}, GitRepositoryBranchCount: MetricConfig{Enabled: false}, + GitRepositoryBranchLineAdditionCount: MetricConfig{Enabled: false}, + GitRepositoryBranchLineDeletionCount: MetricConfig{Enabled: false}, + GitRepositoryBranchTime: MetricConfig{Enabled: false}, GitRepositoryContributorCount: MetricConfig{Enabled: false}, GitRepositoryCount: MetricConfig{Enabled: false}, GitRepositoryPullRequestCount: MetricConfig{Enabled: false}, diff --git a/receiver/gitproviderreceiver/internal/metadata/generated_metrics.go b/receiver/gitproviderreceiver/internal/metadata/generated_metrics.go index 214278c46f7a..4c6050098731 100644 --- a/receiver/gitproviderreceiver/internal/metadata/generated_metrics.go +++ b/receiver/gitproviderreceiver/internal/metadata/generated_metrics.go @@ -39,6 +39,110 @@ var MapAttributePullRequestState = map[string]AttributePullRequestState{ "merged": AttributePullRequestStateMerged, } +type metricGitRepositoryBranchCommitAheadbyCount struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills git.repository.branch.commit.aheadby.count metric with initial data. +func (m *metricGitRepositoryBranchCommitAheadbyCount) init() { + m.data.SetName("git.repository.branch.commit.aheadby.count") + m.data.SetDescription("The number of commits a branch is ahead of the default branch (trunk).") + m.data.SetUnit("{commit}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricGitRepositoryBranchCommitAheadbyCount) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("repository.name", repositoryNameAttributeValue) + dp.Attributes().PutStr("branch.name", branchNameAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricGitRepositoryBranchCommitAheadbyCount) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricGitRepositoryBranchCommitAheadbyCount) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricGitRepositoryBranchCommitAheadbyCount(cfg MetricConfig) metricGitRepositoryBranchCommitAheadbyCount { + m := metricGitRepositoryBranchCommitAheadbyCount{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricGitRepositoryBranchCommitBehindbyCount struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills git.repository.branch.commit.behindby.count metric with initial data. +func (m *metricGitRepositoryBranchCommitBehindbyCount) init() { + m.data.SetName("git.repository.branch.commit.behindby.count") + m.data.SetDescription("The number of commits a branch is behind the default branch (trunk).") + m.data.SetUnit("{commit}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricGitRepositoryBranchCommitBehindbyCount) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("repository.name", repositoryNameAttributeValue) + dp.Attributes().PutStr("branch.name", branchNameAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricGitRepositoryBranchCommitBehindbyCount) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricGitRepositoryBranchCommitBehindbyCount) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricGitRepositoryBranchCommitBehindbyCount(cfg MetricConfig) metricGitRepositoryBranchCommitBehindbyCount { + m := metricGitRepositoryBranchCommitBehindbyCount{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricGitRepositoryBranchCount struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -48,7 +152,7 @@ type metricGitRepositoryBranchCount struct { // init fills git.repository.branch.count metric with initial data. func (m *metricGitRepositoryBranchCount) init() { m.data.SetName("git.repository.branch.count") - m.data.SetDescription("Number of branches in a repository") + m.data.SetDescription("The number of branches in a repository.") m.data.SetUnit("{branch}") m.data.SetEmptyGauge() m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) @@ -90,6 +194,162 @@ func newMetricGitRepositoryBranchCount(cfg MetricConfig) metricGitRepositoryBran return m } +type metricGitRepositoryBranchLineAdditionCount struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills git.repository.branch.line.addition.count metric with initial data. +func (m *metricGitRepositoryBranchLineAdditionCount) init() { + m.data.SetName("git.repository.branch.line.addition.count") + m.data.SetDescription("The number of lines added in a branch relative to the default branch (trunk).") + m.data.SetUnit("{line}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricGitRepositoryBranchLineAdditionCount) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("repository.name", repositoryNameAttributeValue) + dp.Attributes().PutStr("branch.name", branchNameAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricGitRepositoryBranchLineAdditionCount) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricGitRepositoryBranchLineAdditionCount) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricGitRepositoryBranchLineAdditionCount(cfg MetricConfig) metricGitRepositoryBranchLineAdditionCount { + m := metricGitRepositoryBranchLineAdditionCount{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricGitRepositoryBranchLineDeletionCount struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills git.repository.branch.line.deletion.count metric with initial data. +func (m *metricGitRepositoryBranchLineDeletionCount) init() { + m.data.SetName("git.repository.branch.line.deletion.count") + m.data.SetDescription("The number of lines deleted in a branch relative to the default branch (trunk).") + m.data.SetUnit("{line}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricGitRepositoryBranchLineDeletionCount) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("repository.name", repositoryNameAttributeValue) + dp.Attributes().PutStr("branch.name", branchNameAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricGitRepositoryBranchLineDeletionCount) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricGitRepositoryBranchLineDeletionCount) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricGitRepositoryBranchLineDeletionCount(cfg MetricConfig) metricGitRepositoryBranchLineDeletionCount { + m := metricGitRepositoryBranchLineDeletionCount{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricGitRepositoryBranchTime struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills git.repository.branch.time metric with initial data. +func (m *metricGitRepositoryBranchTime) init() { + m.data.SetName("git.repository.branch.time") + m.data.SetDescription("Time a branch created from the default branch (trunk) has existed.") + m.data.SetUnit("s") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricGitRepositoryBranchTime) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("repository.name", repositoryNameAttributeValue) + dp.Attributes().PutStr("branch.name", branchNameAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricGitRepositoryBranchTime) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricGitRepositoryBranchTime) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricGitRepositoryBranchTime(cfg MetricConfig) metricGitRepositoryBranchTime { + m := metricGitRepositoryBranchTime{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricGitRepositoryContributorCount struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -99,7 +359,7 @@ type metricGitRepositoryContributorCount struct { // init fills git.repository.contributor.count metric with initial data. func (m *metricGitRepositoryContributorCount) init() { m.data.SetName("git.repository.contributor.count") - m.data.SetDescription("Total number of unique contributors to a repository") + m.data.SetDescription("The number of unique contributors to a repository.") m.data.SetUnit("{contributor}") m.data.SetEmptyGauge() m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) @@ -150,7 +410,7 @@ type metricGitRepositoryCount struct { // init fills git.repository.count metric with initial data. func (m *metricGitRepositoryCount) init() { m.data.SetName("git.repository.count") - m.data.SetDescription("Number of repositories in an organization") + m.data.SetDescription("The number of repositories in an organization.") m.data.SetUnit("{repository}") m.data.SetEmptyGauge() } @@ -199,7 +459,7 @@ type metricGitRepositoryPullRequestCount struct { // init fills git.repository.pull_request.count metric with initial data. func (m *metricGitRepositoryPullRequestCount) init() { m.data.SetName("git.repository.pull_request.count") - m.data.SetDescription("The number of pull requests in a repository, categorized by their state (either open or merged)") + m.data.SetDescription("The number of pull requests in a repository, categorized by their state (either open or merged).") m.data.SetUnit("{pull_request}") m.data.SetEmptyGauge() m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) @@ -251,7 +511,7 @@ type metricGitRepositoryPullRequestTimeOpen struct { // init fills git.repository.pull_request.time_open metric with initial data. func (m *metricGitRepositoryPullRequestTimeOpen) init() { m.data.SetName("git.repository.pull_request.time_open") - m.data.SetDescription("The amount of time a pull request has been open") + m.data.SetDescription("The amount of time a pull request has been open.") m.data.SetUnit("s") m.data.SetEmptyGauge() m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) @@ -303,7 +563,7 @@ type metricGitRepositoryPullRequestTimeToApproval struct { // init fills git.repository.pull_request.time_to_approval metric with initial data. func (m *metricGitRepositoryPullRequestTimeToApproval) init() { m.data.SetName("git.repository.pull_request.time_to_approval") - m.data.SetDescription("The amount of time it took a pull request to go from open to approved") + m.data.SetDescription("The amount of time it took a pull request to go from open to approved.") m.data.SetUnit("s") m.data.SetEmptyGauge() m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) @@ -355,7 +615,7 @@ type metricGitRepositoryPullRequestTimeToMerge struct { // init fills git.repository.pull_request.time_to_merge metric with initial data. func (m *metricGitRepositoryPullRequestTimeToMerge) init() { m.data.SetName("git.repository.pull_request.time_to_merge") - m.data.SetDescription("The amount of time it took a pull request to go from open to merged") + m.data.SetDescription("The amount of time it took a pull request to go from open to merged.") m.data.SetUnit("s") m.data.SetEmptyGauge() m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) @@ -408,7 +668,12 @@ type MetricsBuilder struct { buildInfo component.BuildInfo // contains version information. resourceAttributeIncludeFilter map[string]filter.Filter resourceAttributeExcludeFilter map[string]filter.Filter + metricGitRepositoryBranchCommitAheadbyCount metricGitRepositoryBranchCommitAheadbyCount + metricGitRepositoryBranchCommitBehindbyCount metricGitRepositoryBranchCommitBehindbyCount metricGitRepositoryBranchCount metricGitRepositoryBranchCount + metricGitRepositoryBranchLineAdditionCount metricGitRepositoryBranchLineAdditionCount + metricGitRepositoryBranchLineDeletionCount metricGitRepositoryBranchLineDeletionCount + metricGitRepositoryBranchTime metricGitRepositoryBranchTime metricGitRepositoryContributorCount metricGitRepositoryContributorCount metricGitRepositoryCount metricGitRepositoryCount metricGitRepositoryPullRequestCount metricGitRepositoryPullRequestCount @@ -429,11 +694,16 @@ func WithStartTime(startTime pcommon.Timestamp) metricBuilderOption { func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.CreateSettings, options ...metricBuilderOption) *MetricsBuilder { mb := &MetricsBuilder{ - config: mbc, - startTime: pcommon.NewTimestampFromTime(time.Now()), - metricsBuffer: pmetric.NewMetrics(), - buildInfo: settings.BuildInfo, + config: mbc, + startTime: pcommon.NewTimestampFromTime(time.Now()), + metricsBuffer: pmetric.NewMetrics(), + buildInfo: settings.BuildInfo, + metricGitRepositoryBranchCommitAheadbyCount: newMetricGitRepositoryBranchCommitAheadbyCount(mbc.Metrics.GitRepositoryBranchCommitAheadbyCount), + metricGitRepositoryBranchCommitBehindbyCount: newMetricGitRepositoryBranchCommitBehindbyCount(mbc.Metrics.GitRepositoryBranchCommitBehindbyCount), metricGitRepositoryBranchCount: newMetricGitRepositoryBranchCount(mbc.Metrics.GitRepositoryBranchCount), + metricGitRepositoryBranchLineAdditionCount: newMetricGitRepositoryBranchLineAdditionCount(mbc.Metrics.GitRepositoryBranchLineAdditionCount), + metricGitRepositoryBranchLineDeletionCount: newMetricGitRepositoryBranchLineDeletionCount(mbc.Metrics.GitRepositoryBranchLineDeletionCount), + metricGitRepositoryBranchTime: newMetricGitRepositoryBranchTime(mbc.Metrics.GitRepositoryBranchTime), metricGitRepositoryContributorCount: newMetricGitRepositoryContributorCount(mbc.Metrics.GitRepositoryContributorCount), metricGitRepositoryCount: newMetricGitRepositoryCount(mbc.Metrics.GitRepositoryCount), metricGitRepositoryPullRequestCount: newMetricGitRepositoryPullRequestCount(mbc.Metrics.GitRepositoryPullRequestCount), @@ -517,7 +787,12 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { ils.Scope().SetName("otelcol/gitproviderreceiver") ils.Scope().SetVersion(mb.buildInfo.Version) ils.Metrics().EnsureCapacity(mb.metricsCapacity) + mb.metricGitRepositoryBranchCommitAheadbyCount.emit(ils.Metrics()) + mb.metricGitRepositoryBranchCommitBehindbyCount.emit(ils.Metrics()) mb.metricGitRepositoryBranchCount.emit(ils.Metrics()) + mb.metricGitRepositoryBranchLineAdditionCount.emit(ils.Metrics()) + mb.metricGitRepositoryBranchLineDeletionCount.emit(ils.Metrics()) + mb.metricGitRepositoryBranchTime.emit(ils.Metrics()) mb.metricGitRepositoryContributorCount.emit(ils.Metrics()) mb.metricGitRepositoryCount.emit(ils.Metrics()) mb.metricGitRepositoryPullRequestCount.emit(ils.Metrics()) @@ -555,11 +830,36 @@ func (mb *MetricsBuilder) Emit(rmo ...ResourceMetricsOption) pmetric.Metrics { return metrics } +// RecordGitRepositoryBranchCommitAheadbyCountDataPoint adds a data point to git.repository.branch.commit.aheadby.count metric. +func (mb *MetricsBuilder) RecordGitRepositoryBranchCommitAheadbyCountDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + mb.metricGitRepositoryBranchCommitAheadbyCount.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue, branchNameAttributeValue) +} + +// RecordGitRepositoryBranchCommitBehindbyCountDataPoint adds a data point to git.repository.branch.commit.behindby.count metric. +func (mb *MetricsBuilder) RecordGitRepositoryBranchCommitBehindbyCountDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + mb.metricGitRepositoryBranchCommitBehindbyCount.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue, branchNameAttributeValue) +} + // RecordGitRepositoryBranchCountDataPoint adds a data point to git.repository.branch.count metric. func (mb *MetricsBuilder) RecordGitRepositoryBranchCountDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string) { mb.metricGitRepositoryBranchCount.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue) } +// RecordGitRepositoryBranchLineAdditionCountDataPoint adds a data point to git.repository.branch.line.addition.count metric. +func (mb *MetricsBuilder) RecordGitRepositoryBranchLineAdditionCountDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + mb.metricGitRepositoryBranchLineAdditionCount.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue, branchNameAttributeValue) +} + +// RecordGitRepositoryBranchLineDeletionCountDataPoint adds a data point to git.repository.branch.line.deletion.count metric. +func (mb *MetricsBuilder) RecordGitRepositoryBranchLineDeletionCountDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + mb.metricGitRepositoryBranchLineDeletionCount.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue, branchNameAttributeValue) +} + +// RecordGitRepositoryBranchTimeDataPoint adds a data point to git.repository.branch.time metric. +func (mb *MetricsBuilder) RecordGitRepositoryBranchTimeDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string, branchNameAttributeValue string) { + mb.metricGitRepositoryBranchTime.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue, branchNameAttributeValue) +} + // RecordGitRepositoryContributorCountDataPoint adds a data point to git.repository.contributor.count metric. func (mb *MetricsBuilder) RecordGitRepositoryContributorCountDataPoint(ts pcommon.Timestamp, val int64, repositoryNameAttributeValue string) { mb.metricGitRepositoryContributorCount.recordDataPoint(mb.startTime, ts, val, repositoryNameAttributeValue) diff --git a/receiver/gitproviderreceiver/internal/metadata/generated_metrics_test.go b/receiver/gitproviderreceiver/internal/metadata/generated_metrics_test.go index d241678ec245..eb466ca782d9 100644 --- a/receiver/gitproviderreceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/gitproviderreceiver/internal/metadata/generated_metrics_test.go @@ -68,10 +68,30 @@ func TestMetricsBuilder(t *testing.T) { defaultMetricsCount := 0 allMetricsCount := 0 + defaultMetricsCount++ + allMetricsCount++ + mb.RecordGitRepositoryBranchCommitAheadbyCountDataPoint(ts, 1, "repository.name-val", "branch.name-val") + + defaultMetricsCount++ + allMetricsCount++ + mb.RecordGitRepositoryBranchCommitBehindbyCountDataPoint(ts, 1, "repository.name-val", "branch.name-val") + defaultMetricsCount++ allMetricsCount++ mb.RecordGitRepositoryBranchCountDataPoint(ts, 1, "repository.name-val") + defaultMetricsCount++ + allMetricsCount++ + mb.RecordGitRepositoryBranchLineAdditionCountDataPoint(ts, 1, "repository.name-val", "branch.name-val") + + defaultMetricsCount++ + allMetricsCount++ + mb.RecordGitRepositoryBranchLineDeletionCountDataPoint(ts, 1, "repository.name-val", "branch.name-val") + + defaultMetricsCount++ + allMetricsCount++ + mb.RecordGitRepositoryBranchTimeDataPoint(ts, 1, "repository.name-val", "branch.name-val") + allMetricsCount++ mb.RecordGitRepositoryContributorCountDataPoint(ts, 1, "repository.name-val") @@ -120,12 +140,48 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics := make(map[string]bool) for i := 0; i < ms.Len(); i++ { switch ms.At(i).Name() { + case "git.repository.branch.commit.aheadby.count": + assert.False(t, validatedMetrics["git.repository.branch.commit.aheadby.count"], "Found a duplicate in the metrics slice: git.repository.branch.commit.aheadby.count") + validatedMetrics["git.repository.branch.commit.aheadby.count"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "The number of commits a branch is ahead of the default branch (trunk).", ms.At(i).Description()) + assert.Equal(t, "{commit}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("repository.name") + assert.True(t, ok) + assert.EqualValues(t, "repository.name-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("branch.name") + assert.True(t, ok) + assert.EqualValues(t, "branch.name-val", attrVal.Str()) + case "git.repository.branch.commit.behindby.count": + assert.False(t, validatedMetrics["git.repository.branch.commit.behindby.count"], "Found a duplicate in the metrics slice: git.repository.branch.commit.behindby.count") + validatedMetrics["git.repository.branch.commit.behindby.count"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "The number of commits a branch is behind the default branch (trunk).", ms.At(i).Description()) + assert.Equal(t, "{commit}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("repository.name") + assert.True(t, ok) + assert.EqualValues(t, "repository.name-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("branch.name") + assert.True(t, ok) + assert.EqualValues(t, "branch.name-val", attrVal.Str()) case "git.repository.branch.count": assert.False(t, validatedMetrics["git.repository.branch.count"], "Found a duplicate in the metrics slice: git.repository.branch.count") validatedMetrics["git.repository.branch.count"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "Number of branches in a repository", ms.At(i).Description()) + assert.Equal(t, "The number of branches in a repository.", ms.At(i).Description()) assert.Equal(t, "{branch}", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) @@ -135,12 +191,66 @@ func TestMetricsBuilder(t *testing.T) { attrVal, ok := dp.Attributes().Get("repository.name") assert.True(t, ok) assert.EqualValues(t, "repository.name-val", attrVal.Str()) + case "git.repository.branch.line.addition.count": + assert.False(t, validatedMetrics["git.repository.branch.line.addition.count"], "Found a duplicate in the metrics slice: git.repository.branch.line.addition.count") + validatedMetrics["git.repository.branch.line.addition.count"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "The number of lines added in a branch relative to the default branch (trunk).", ms.At(i).Description()) + assert.Equal(t, "{line}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("repository.name") + assert.True(t, ok) + assert.EqualValues(t, "repository.name-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("branch.name") + assert.True(t, ok) + assert.EqualValues(t, "branch.name-val", attrVal.Str()) + case "git.repository.branch.line.deletion.count": + assert.False(t, validatedMetrics["git.repository.branch.line.deletion.count"], "Found a duplicate in the metrics slice: git.repository.branch.line.deletion.count") + validatedMetrics["git.repository.branch.line.deletion.count"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "The number of lines deleted in a branch relative to the default branch (trunk).", ms.At(i).Description()) + assert.Equal(t, "{line}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("repository.name") + assert.True(t, ok) + assert.EqualValues(t, "repository.name-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("branch.name") + assert.True(t, ok) + assert.EqualValues(t, "branch.name-val", attrVal.Str()) + case "git.repository.branch.time": + assert.False(t, validatedMetrics["git.repository.branch.time"], "Found a duplicate in the metrics slice: git.repository.branch.time") + validatedMetrics["git.repository.branch.time"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "Time a branch created from the default branch (trunk) has existed.", ms.At(i).Description()) + assert.Equal(t, "s", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("repository.name") + assert.True(t, ok) + assert.EqualValues(t, "repository.name-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("branch.name") + assert.True(t, ok) + assert.EqualValues(t, "branch.name-val", attrVal.Str()) case "git.repository.contributor.count": assert.False(t, validatedMetrics["git.repository.contributor.count"], "Found a duplicate in the metrics slice: git.repository.contributor.count") validatedMetrics["git.repository.contributor.count"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "Total number of unique contributors to a repository", ms.At(i).Description()) + assert.Equal(t, "The number of unique contributors to a repository.", ms.At(i).Description()) assert.Equal(t, "{contributor}", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) @@ -155,7 +265,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["git.repository.count"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "Number of repositories in an organization", ms.At(i).Description()) + assert.Equal(t, "The number of repositories in an organization.", ms.At(i).Description()) assert.Equal(t, "{repository}", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) @@ -167,7 +277,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["git.repository.pull_request.count"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "The number of pull requests in a repository, categorized by their state (either open or merged)", ms.At(i).Description()) + assert.Equal(t, "The number of pull requests in a repository, categorized by their state (either open or merged).", ms.At(i).Description()) assert.Equal(t, "{pull_request}", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) @@ -185,7 +295,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["git.repository.pull_request.time_open"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "The amount of time a pull request has been open", ms.At(i).Description()) + assert.Equal(t, "The amount of time a pull request has been open.", ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) @@ -203,7 +313,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["git.repository.pull_request.time_to_approval"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "The amount of time it took a pull request to go from open to approved", ms.At(i).Description()) + assert.Equal(t, "The amount of time it took a pull request to go from open to approved.", ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) @@ -221,7 +331,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["git.repository.pull_request.time_to_merge"] = true assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) - assert.Equal(t, "The amount of time it took a pull request to go from open to merged", ms.At(i).Description()) + assert.Equal(t, "The amount of time it took a pull request to go from open to merged.", ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) dp := ms.At(i).Gauge().DataPoints().At(0) assert.Equal(t, start, dp.StartTimestamp()) diff --git a/receiver/gitproviderreceiver/internal/metadata/testdata/config.yaml b/receiver/gitproviderreceiver/internal/metadata/testdata/config.yaml index 24cb3661783c..b4b0ee7347de 100644 --- a/receiver/gitproviderreceiver/internal/metadata/testdata/config.yaml +++ b/receiver/gitproviderreceiver/internal/metadata/testdata/config.yaml @@ -1,8 +1,18 @@ default: all_set: metrics: + git.repository.branch.commit.aheadby.count: + enabled: true + git.repository.branch.commit.behindby.count: + enabled: true git.repository.branch.count: enabled: true + git.repository.branch.line.addition.count: + enabled: true + git.repository.branch.line.deletion.count: + enabled: true + git.repository.branch.time: + enabled: true git.repository.contributor.count: enabled: true git.repository.count: @@ -22,8 +32,18 @@ all_set: enabled: true none_set: metrics: + git.repository.branch.commit.aheadby.count: + enabled: false + git.repository.branch.commit.behindby.count: + enabled: false git.repository.branch.count: enabled: false + git.repository.branch.line.addition.count: + enabled: false + git.repository.branch.line.deletion.count: + enabled: false + git.repository.branch.time: + enabled: false git.repository.contributor.count: enabled: false git.repository.count: diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md b/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md new file mode 100644 index 000000000000..7cc1d46bbbcc --- /dev/null +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md @@ -0,0 +1,109 @@ +# GitHub Limitations + +## API Limitations + +The GitHub scraper is reliant on limitations found within GitHub's REST and +GraphQL APIs. The following limitations are known: + +* The original creation date of a branch is not available via either of the + APIs. GitSCM (the tool) does provide Ref creation time however this is not + exposed. As such, we're forced to calculate the age by looking to see if any + changes have been made to the branch, using that commit as the time from + which we can grab the date. This means that age will reflect the time between + now and the first commit on a new branch. It also means that we don't have + ages for branches that have been created from trunk but have not had any + changes made to them. +* It's possible that some queries may run against a branch that has been + deleted. This is unlikely due to the speed of the requests, however, + possible. +* Both APIs have primary and secondary rate limits applied to them. The default + rate limit for GraphQL API is 5,000 points per hour when authenticated with a + GitHub Personal Access Token (PAT). If using the [GitHub App Auth + extension][ghext] then your rate limit increases to 10,000. The receiver on + average costs 4 points per repository (which can heavily fluctuate), allowing + it to scrape up to 1250 repositories per hour under normal conditions. You + may use the following equation to roughly calculate your ideal collection + interval. + +```math +\text{collection\_interval (seconds)} = \frac{4n}{r/3600} +``` + +```math +\begin{aligned} + \text{where:} \\ + n &= \text{number of repositories} \\ + r &= \text{hourly rate limit} \\ +\end{aligned} +``` + +In addition to these primary rate limits, GitHub enforces secondary rate limits +to prevent abuse and maintain API availability. The following secondary limit is +particularly relevant: + +- **Concurrent Requests Limit**: The API allows no more than 100 concurrent +requests. This limit is shared across the REST and GraphQL APIs. Since the +scraper creates a goroutine per repository, having more than 100 repositories +returned by the `search_query` will result in exceeding this limit. +It is recommended to use the `search_query` config option to limit the number of +repositories that are scraped. We recommend one instance of the receiver per +team (note: `team` is not a valid quantifier when searching repositories `topic` +is). Reminder that each instance of the receiver should have its own +corresponding token for authentication as this is what rate limits are tied to. + +In summary, we recommend the following: + +- One instance of the receiver per team +- Each instance of the receiver should have its own token +- Leverage `search_query` config option to limit repositories returned to 100 or +less per instance +- `collection_interval` should be long enough to avoid rate limiting (see above +formula). A sensible default is `300s`. + +**Additional Resources:** + +- [GitHub GraphQL Primary Rate Limit](https://docs.github.com/en/graphql/overview/rate-limits-and-node-limits-for-the-graphql-api#primary-rate-limit) +- [GitHub GraphQL Secondary Rate Limit](https://docs.github.com/en/graphql/overview/rate-limits-and-node-limits-for-the-graphql-api#secondary-rate-limit) + +[ghext]: https://github.com/liatrio/liatrio-otel-collector/tree/main/extension/githubappauthextension + +## Branch Data Limitations + + +Due to the limitations of the GitHub GraphQL and REST APIs, some data retrieved +may not be as expected. Notably there are spots in the code which link to this +section that make decisions based on these limitations. + +Queries are constructed to maximize performance without being overly complex. +Note that there are sections in the code where `BehindBy` is being used in +place of `AheadBy` and vice versa. This is a byproduct of the `getBranchData` +query which returns all the branches from a given repository and the +comparison to the default branch (trunk). Comparing it here reduces the need +to make a query that gets all the names of the branches, and then queries +against each branch. + +Another such byproduct of this method is the skipping of metric creation if the +branch is the default branch (trunk) or if no changes have been made to the +branch. This is done for three main reasons. + +1. The default branch will always be a long-lived branch and + may end up with more commits than can be possibly queried + at a given time. +2. The default is the trunk of which all changes should go + into. The intent of these metrics is to provide useful + signals helping identify cognitive overhead and + bottlenecks. +3. GitHub does not provide any means to determine when a + branch was actually created. Git the tool however does + provide a created time for each ref off the trunk. GitHub + does not expose this data via their APIs and thus we + have to calculate age based on commits added to the + branch. + +We also have to calculate the number of pages before getting the commit data. +This is because you have to know the exact number of commits added to the +branch, otherwise you'll get all commits from both trunk and the branch from +all time. From there we can evaluate the commits on each branch. To calculate +the time (age) of a branch, we have to know the commits that have been added to +the branch because GitHub does not provide the actual created date of a branch +through either of its APIs. diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/generated_graphql.go b/receiver/gitproviderreceiver/internal/scraper/githubscraper/generated_graphql.go index 5739f5743189..db5f5271fb75 100644 --- a/receiver/gitproviderreceiver/internal/scraper/githubscraper/generated_graphql.go +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/generated_graphql.go @@ -11,6 +11,278 @@ import ( "github.com/Khan/genqlient/graphql" ) +// BranchHistory includes the requested fields of the GraphQL type Ref. +// The GraphQL type's documentation follows. +// +// Represents a Git reference. +type BranchHistory struct { + // The object the ref points to. Returns null when object does not exist. + Target BranchHistoryTargetGitObject `json:"-"` +} + +// GetTarget returns BranchHistory.Target, and is useful for accessing the field via an interface. +func (v *BranchHistory) GetTarget() BranchHistoryTargetGitObject { return v.Target } + +func (v *BranchHistory) UnmarshalJSON(b []byte) error { + + if string(b) == "null" { + return nil + } + + var firstPass struct { + *BranchHistory + Target json.RawMessage `json:"target"` + graphql.NoUnmarshalJSON + } + firstPass.BranchHistory = v + + err := json.Unmarshal(b, &firstPass) + if err != nil { + return err + } + + { + dst := &v.Target + src := firstPass.Target + if len(src) != 0 && string(src) != "null" { + err = __unmarshalBranchHistoryTargetGitObject( + src, dst) + if err != nil { + return fmt.Errorf( + "unable to unmarshal BranchHistory.Target: %w", err) + } + } + } + return nil +} + +type __premarshalBranchHistory struct { + Target json.RawMessage `json:"target"` +} + +func (v *BranchHistory) MarshalJSON() ([]byte, error) { + premarshaled, err := v.__premarshalJSON() + if err != nil { + return nil, err + } + return json.Marshal(premarshaled) +} + +func (v *BranchHistory) __premarshalJSON() (*__premarshalBranchHistory, error) { + var retval __premarshalBranchHistory + + { + + dst := &retval.Target + src := v.Target + var err error + *dst, err = __marshalBranchHistoryTargetGitObject( + &src) + if err != nil { + return nil, fmt.Errorf( + "unable to marshal BranchHistory.Target: %w", err) + } + } + return &retval, nil +} + +// BranchHistoryTargetBlob includes the requested fields of the GraphQL type Blob. +// The GraphQL type's documentation follows. +// +// Represents a Git blob. +type BranchHistoryTargetBlob struct { + Typename string `json:"__typename"` +} + +// GetTypename returns BranchHistoryTargetBlob.Typename, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetBlob) GetTypename() string { return v.Typename } + +// BranchHistoryTargetCommit includes the requested fields of the GraphQL type Commit. +// The GraphQL type's documentation follows. +// +// Represents a Git commit. +type BranchHistoryTargetCommit struct { + Typename string `json:"__typename"` + Id string `json:"id"` + // The linear commit history starting from (and including) this commit, in the same order as `git log`. + History BranchHistoryTargetCommitHistoryCommitHistoryConnection `json:"history"` +} + +// GetTypename returns BranchHistoryTargetCommit.Typename, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommit) GetTypename() string { return v.Typename } + +// GetId returns BranchHistoryTargetCommit.Id, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommit) GetId() string { return v.Id } + +// GetHistory returns BranchHistoryTargetCommit.History, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommit) GetHistory() BranchHistoryTargetCommitHistoryCommitHistoryConnection { + return v.History +} + +// BranchHistoryTargetCommitHistoryCommitHistoryConnection includes the requested fields of the GraphQL type CommitHistoryConnection. +// The GraphQL type's documentation follows. +// +// The connection type for Commit. +type BranchHistoryTargetCommitHistoryCommitHistoryConnection struct { + // A list of nodes. + Nodes []CommitNode `json:"nodes"` + // Information to aid in pagination. + PageInfo BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo `json:"pageInfo"` +} + +// GetNodes returns BranchHistoryTargetCommitHistoryCommitHistoryConnection.Nodes, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommitHistoryCommitHistoryConnection) GetNodes() []CommitNode { + return v.Nodes +} + +// GetPageInfo returns BranchHistoryTargetCommitHistoryCommitHistoryConnection.PageInfo, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommitHistoryCommitHistoryConnection) GetPageInfo() BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo { + return v.PageInfo +} + +// BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo includes the requested fields of the GraphQL type PageInfo. +// The GraphQL type's documentation follows. +// +// Information about pagination in a connection. +type BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo struct { + // When paginating forwards, the cursor to continue. + EndCursor string `json:"endCursor"` + // When paginating forwards, are there more items? + HasNextPage bool `json:"hasNextPage"` +} + +// GetEndCursor returns BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo.EndCursor, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo) GetEndCursor() string { + return v.EndCursor +} + +// GetHasNextPage returns BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo.HasNextPage, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetCommitHistoryCommitHistoryConnectionPageInfo) GetHasNextPage() bool { + return v.HasNextPage +} + +// BranchHistoryTargetGitObject includes the requested fields of the GraphQL interface GitObject. +// +// BranchHistoryTargetGitObject is implemented by the following types: +// BranchHistoryTargetBlob +// BranchHistoryTargetCommit +// BranchHistoryTargetTag +// BranchHistoryTargetTree +// The GraphQL type's documentation follows. +// +// Represents a Git object. +type BranchHistoryTargetGitObject interface { + implementsGraphQLInterfaceBranchHistoryTargetGitObject() + // GetTypename returns the receiver's concrete GraphQL type-name (see interface doc for possible values). + GetTypename() string +} + +func (v *BranchHistoryTargetBlob) implementsGraphQLInterfaceBranchHistoryTargetGitObject() {} +func (v *BranchHistoryTargetCommit) implementsGraphQLInterfaceBranchHistoryTargetGitObject() {} +func (v *BranchHistoryTargetTag) implementsGraphQLInterfaceBranchHistoryTargetGitObject() {} +func (v *BranchHistoryTargetTree) implementsGraphQLInterfaceBranchHistoryTargetGitObject() {} + +func __unmarshalBranchHistoryTargetGitObject(b []byte, v *BranchHistoryTargetGitObject) error { + if string(b) == "null" { + return nil + } + + var tn struct { + TypeName string `json:"__typename"` + } + err := json.Unmarshal(b, &tn) + if err != nil { + return err + } + + switch tn.TypeName { + case "Blob": + *v = new(BranchHistoryTargetBlob) + return json.Unmarshal(b, *v) + case "Commit": + *v = new(BranchHistoryTargetCommit) + return json.Unmarshal(b, *v) + case "Tag": + *v = new(BranchHistoryTargetTag) + return json.Unmarshal(b, *v) + case "Tree": + *v = new(BranchHistoryTargetTree) + return json.Unmarshal(b, *v) + case "": + return fmt.Errorf( + "response was missing GitObject.__typename") + default: + return fmt.Errorf( + `unexpected concrete type for BranchHistoryTargetGitObject: "%v"`, tn.TypeName) + } +} + +func __marshalBranchHistoryTargetGitObject(v *BranchHistoryTargetGitObject) ([]byte, error) { + + var typename string + switch v := (*v).(type) { + case *BranchHistoryTargetBlob: + typename = "Blob" + + result := struct { + TypeName string `json:"__typename"` + *BranchHistoryTargetBlob + }{typename, v} + return json.Marshal(result) + case *BranchHistoryTargetCommit: + typename = "Commit" + + result := struct { + TypeName string `json:"__typename"` + *BranchHistoryTargetCommit + }{typename, v} + return json.Marshal(result) + case *BranchHistoryTargetTag: + typename = "Tag" + + result := struct { + TypeName string `json:"__typename"` + *BranchHistoryTargetTag + }{typename, v} + return json.Marshal(result) + case *BranchHistoryTargetTree: + typename = "Tree" + + result := struct { + TypeName string `json:"__typename"` + *BranchHistoryTargetTree + }{typename, v} + return json.Marshal(result) + case nil: + return []byte("null"), nil + default: + return nil, fmt.Errorf( + `unexpected concrete type for BranchHistoryTargetGitObject: "%T"`, v) + } +} + +// BranchHistoryTargetTag includes the requested fields of the GraphQL type Tag. +// The GraphQL type's documentation follows. +// +// Represents a Git tag. +type BranchHistoryTargetTag struct { + Typename string `json:"__typename"` +} + +// GetTypename returns BranchHistoryTargetTag.Typename, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetTag) GetTypename() string { return v.Typename } + +// BranchHistoryTargetTree includes the requested fields of the GraphQL type Tree. +// The GraphQL type's documentation follows. +// +// Represents a Git tree. +type BranchHistoryTargetTree struct { + Typename string `json:"__typename"` +} + +// GetTypename returns BranchHistoryTargetTree.Typename, and is useful for accessing the field via an interface. +func (v *BranchHistoryTargetTree) GetTypename() string { return v.Typename } + // BranchNode includes the requested fields of the GraphQL type Ref. // The GraphQL type's documentation follows. // @@ -81,6 +353,28 @@ type BranchNodeRepositoryDefaultBranchRef struct { // GetName returns BranchNodeRepositoryDefaultBranchRef.Name, and is useful for accessing the field via an interface. func (v *BranchNodeRepositoryDefaultBranchRef) GetName() string { return v.Name } +// CommitNode includes the requested fields of the GraphQL type Commit. +// The GraphQL type's documentation follows. +// +// Represents a Git commit. +type CommitNode struct { + // The datetime when this commit was committed. + CommittedDate time.Time `json:"committedDate"` + // The number of additions in this commit. + Additions int `json:"additions"` + // The number of deletions in this commit. + Deletions int `json:"deletions"` +} + +// GetCommittedDate returns CommitNode.CommittedDate, and is useful for accessing the field via an interface. +func (v *CommitNode) GetCommittedDate() time.Time { return v.CommittedDate } + +// GetAdditions returns CommitNode.Additions, and is useful for accessing the field via an interface. +func (v *CommitNode) GetAdditions() int { return v.Additions } + +// GetDeletions returns CommitNode.Deletions, and is useful for accessing the field via an interface. +func (v *CommitNode) GetDeletions() int { return v.Deletions } + // PullRequestNode includes the requested fields of the GraphQL type PullRequest. // The GraphQL type's documentation follows. // @@ -514,6 +808,34 @@ func (v *__getBranchDataInput) GetTargetBranch() string { return v.TargetBranch // GetBranchCursor returns __getBranchDataInput.BranchCursor, and is useful for accessing the field via an interface. func (v *__getBranchDataInput) GetBranchCursor() *string { return v.BranchCursor } +// __getCommitDataInput is used internally by genqlient +type __getCommitDataInput struct { + Name string `json:"name"` + Owner string `json:"owner"` + BranchFirst int `json:"branchFirst"` + CommitFirst int `json:"commitFirst"` + CommitCursor *string `json:"commitCursor"` + BranchName string `json:"branchName"` +} + +// GetName returns __getCommitDataInput.Name, and is useful for accessing the field via an interface. +func (v *__getCommitDataInput) GetName() string { return v.Name } + +// GetOwner returns __getCommitDataInput.Owner, and is useful for accessing the field via an interface. +func (v *__getCommitDataInput) GetOwner() string { return v.Owner } + +// GetBranchFirst returns __getCommitDataInput.BranchFirst, and is useful for accessing the field via an interface. +func (v *__getCommitDataInput) GetBranchFirst() int { return v.BranchFirst } + +// GetCommitFirst returns __getCommitDataInput.CommitFirst, and is useful for accessing the field via an interface. +func (v *__getCommitDataInput) GetCommitFirst() int { return v.CommitFirst } + +// GetCommitCursor returns __getCommitDataInput.CommitCursor, and is useful for accessing the field via an interface. +func (v *__getCommitDataInput) GetCommitCursor() *string { return v.CommitCursor } + +// GetBranchName returns __getCommitDataInput.BranchName, and is useful for accessing the field via an interface. +func (v *__getCommitDataInput) GetBranchName() string { return v.BranchName } + // __getPullRequestDataInput is used internally by genqlient type __getPullRequestDataInput struct { Name string `json:"name"` @@ -652,6 +974,39 @@ type getBranchDataResponse struct { // GetRepository returns getBranchDataResponse.Repository, and is useful for accessing the field via an interface. func (v *getBranchDataResponse) GetRepository() getBranchDataRepository { return v.Repository } +// getCommitDataRepository includes the requested fields of the GraphQL type Repository. +// The GraphQL type's documentation follows. +// +// A repository contains the content for a project. +type getCommitDataRepository struct { + // Fetch a list of refs from the repository + Refs getCommitDataRepositoryRefsRefConnection `json:"refs"` +} + +// GetRefs returns getCommitDataRepository.Refs, and is useful for accessing the field via an interface. +func (v *getCommitDataRepository) GetRefs() getCommitDataRepositoryRefsRefConnection { return v.Refs } + +// getCommitDataRepositoryRefsRefConnection includes the requested fields of the GraphQL type RefConnection. +// The GraphQL type's documentation follows. +// +// The connection type for Ref. +type getCommitDataRepositoryRefsRefConnection struct { + // A list of nodes. + Nodes []BranchHistory `json:"nodes"` +} + +// GetNodes returns getCommitDataRepositoryRefsRefConnection.Nodes, and is useful for accessing the field via an interface. +func (v *getCommitDataRepositoryRefsRefConnection) GetNodes() []BranchHistory { return v.Nodes } + +// getCommitDataResponse is returned by getCommitData on success. +type getCommitDataResponse struct { + // Lookup a given repository by the owner and repository name. + Repository getCommitDataRepository `json:"repository"` +} + +// GetRepository returns getCommitDataResponse.Repository, and is useful for accessing the field via an interface. +func (v *getCommitDataResponse) GetRepository() getCommitDataRepository { return v.Repository } + // getPullRequestDataRepository includes the requested fields of the GraphQL type Repository. // The GraphQL type's documentation follows. // @@ -958,6 +1313,71 @@ func getBranchData( return &data_, err_ } +// The query or mutation executed by getCommitData. +const getCommitData_Operation = ` +query getCommitData ($name: String!, $owner: String!, $branchFirst: Int!, $commitFirst: Int!, $commitCursor: String, $branchName: String!) { + repository(name: $name, owner: $owner) { + refs(refPrefix: "refs/heads/", first: $branchFirst, query: $branchName) { + nodes { + target { + __typename + ... on Commit { + id + history(first: $commitFirst, after: $commitCursor) { + nodes { + committedDate + additions + deletions + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + } + } + } +} +` + +func getCommitData( + ctx_ context.Context, + client_ graphql.Client, + name string, + owner string, + branchFirst int, + commitFirst int, + commitCursor *string, + branchName string, +) (*getCommitDataResponse, error) { + req_ := &graphql.Request{ + OpName: "getCommitData", + Query: getCommitData_Operation, + Variables: &__getCommitDataInput{ + Name: name, + Owner: owner, + BranchFirst: branchFirst, + CommitFirst: commitFirst, + CommitCursor: commitCursor, + BranchName: branchName, + }, + } + var err_ error + + var data_ getCommitDataResponse + resp_ := &graphql.Response{Data: &data_} + + err_ = client_.MakeRequest( + ctx_, + req_, + resp_, + ) + + return &data_, err_ +} + // The query or mutation executed by getPullRequestData. const getPullRequestData_Operation = ` query getPullRequestData ($name: String!, $owner: String!, $prFirst: Int!, $prCursor: String, $prStates: [PullRequestState!]) { diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/genqlient.graphql b/receiver/gitproviderreceiver/internal/scraper/githubscraper/genqlient.graphql index a6df960a4f81..7a66c245fbcb 100644 --- a/receiver/gitproviderreceiver/internal/scraper/githubscraper/genqlient.graphql +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/genqlient.graphql @@ -36,6 +36,41 @@ query checkLogin($login: String!) { } } +query getCommitData( + $name: String! + $owner: String! + $branchFirst: Int! + $commitFirst: Int! + # @genqlient(pointer: true) + $commitCursor: String + $branchName: String! +) { + repository(name: $name, owner: $owner) { + refs(refPrefix: "refs/heads/", first: $branchFirst, query: $branchName) { + # @genqlient(typename: "BranchHistory") + nodes { + target { + ... on Commit { + id + history(first: $commitFirst, after: $commitCursor) { + # @genqlient(typename: "CommitNode") + nodes { + committedDate + additions + deletions + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + } + } + } +} + query getBranchData( $name: String! $owner: String! diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper.go b/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper.go index 46c43a5bd092..80d71bcba929 100644 --- a/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper.go +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper.go @@ -98,33 +98,72 @@ func (ghs *githubScraper) scrape(ctx context.Context) (pmetric.Metrics, error) { // Get the branch count (future branch data) for each repo and record the given metrics var wg sync.WaitGroup + wg.Add(len(repos)) + var mux sync.Mutex for _, repo := range repos { repo := repo name := repo.Name trunk := repo.DefaultBranchRef.Name + now := now - wg.Add(1) go func() { defer wg.Done() - count, err := ghs.getBranches(ctx, genClient, name, trunk) + branches, count, err := ghs.getBranches(ctx, genClient, name, trunk) if err != nil { - ghs.logger.Sugar().Errorf("error getting branch count for repo %s", zap.Error(err), repo.Name) + ghs.logger.Sugar().Errorf("error getting branch count: %v", zap.Error(err)) } + + // Create a mutual exclusion lock to prevent the recordDataPoint + // SetStartTimestamp call from having a nil pointer panic + mux.Lock() ghs.mb.RecordGitRepositoryBranchCountDataPoint(now, int64(count), name) + // Iterate through the branches populating the Branch focused + // metrics + for _, branch := range branches { + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md#github-limitations + // for more information as to why we do not emit metrics for + // the default branch (trunk) nor any branch with no changes to + // it. + if branch.Name == branch.Repository.DefaultBranchRef.Name || branch.Compare.BehindBy == 0 { + continue + } + + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md#github-limitations + // for more information as to why `BehindBy` and `AheadBy` are + // swapped. + ghs.mb.RecordGitRepositoryBranchCommitAheadbyCountDataPoint(now, int64(branch.Compare.BehindBy), branch.Repository.Name, branch.Name) + ghs.mb.RecordGitRepositoryBranchCommitBehindbyCountDataPoint(now, int64(branch.Compare.AheadBy), branch.Repository.Name, branch.Name) + + var additions int + var deletions int + var age int64 + + additions, deletions, age, err = ghs.evalCommits(ctx, genClient, branch.Repository.Name, branch) + if err != nil { + ghs.logger.Sugar().Errorf("error getting commit info: %v", zap.Error(err)) + continue + } + + ghs.mb.RecordGitRepositoryBranchTimeDataPoint(now, age, branch.Repository.Name, branch.Name) + ghs.mb.RecordGitRepositoryBranchLineAdditionCountDataPoint(now, int64(additions), branch.Repository.Name, branch.Name) + ghs.mb.RecordGitRepositoryBranchLineDeletionCountDataPoint(now, int64(deletions), branch.Repository.Name, branch.Name) + + } + // Get the contributor count for each of the repositories contribs, err := ghs.getContributorCount(ctx, restClient, name) if err != nil { - ghs.logger.Sugar().Errorf("error getting contributor count for repo %s", zap.Error(err), repo.Name) + ghs.logger.Sugar().Errorf("error getting contributor count: %v", zap.Error(err)) } ghs.mb.RecordGitRepositoryContributorCountDataPoint(now, int64(contribs), name) // Get Pull Request data prs, err := ghs.getPullRequests(ctx, genClient, name) if err != nil { - ghs.logger.Sugar().Errorf("error getting pull requests for repo %s", zap.Error(err), repo.Name) + ghs.logger.Sugar().Errorf("error getting pull requests: %v", zap.Error(err)) } var merged int @@ -155,6 +194,7 @@ func (ghs *githubScraper) scrape(ctx context.Context) (pmetric.Metrics, error) { ghs.mb.RecordGitRepositoryPullRequestCountDataPoint(now, int64(open), metadata.AttributePullRequestStateOpen, name) ghs.mb.RecordGitRepositoryPullRequestCountDataPoint(now, int64(merged), metadata.AttributePullRequestStateMerged, name) + mux.Unlock() }() } diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper_test.go b/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper_test.go index 3e67d5d04acf..aa888b2567ce 100644 --- a/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper_test.go +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/github_scraper_test.go @@ -2,3 +2,201 @@ // SPDX-License-Identifier: Apache-2.0 package githubscraper // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/gitproviderreceiver" + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/google/go-github/v62/github" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/gitproviderreceiver/internal/metadata" +) + +func TestNewGitHubScraper(t *testing.T) { + factory := Factory{} + defaultConfig := factory.CreateDefaultConfig() + + s := newGitHubScraper(context.Background(), receiver.CreateSettings{}, defaultConfig.(*Config)) + + assert.NotNil(t, s) +} + +func TestScrape(t *testing.T) { + testCases := []struct { + desc string + server *http.ServeMux + testFile string + }{ + { + desc: "TestNoRepos", + server: MockServer(&responses{ + scrape: true, + checkLoginResponse: loginResponse{ + checkLogin: checkLoginResponse{ + Organization: checkLoginOrganization{ + Login: "open-telemetry", + }, + }, + responseCode: http.StatusOK, + }, + repoResponse: repoResponse{ + repos: []getRepoDataBySearchSearchSearchResultItemConnection{ + { + RepositoryCount: 0, + Nodes: []SearchNode{}, + }, + }, + responseCode: http.StatusOK, + }, + }), + testFile: "expected_no_repos.yaml", + }, + { + desc: "TestHappyPath", + server: MockServer(&responses{ + scrape: true, + checkLoginResponse: loginResponse{ + checkLogin: checkLoginResponse{ + Organization: checkLoginOrganization{ + Login: "open-telemetry", + }, + }, + responseCode: http.StatusOK, + }, + repoResponse: repoResponse{ + repos: []getRepoDataBySearchSearchSearchResultItemConnection{ + { + RepositoryCount: 1, + Nodes: []SearchNode{ + &SearchNodeRepository{ + Name: "repo1", + }, + }, + PageInfo: getRepoDataBySearchSearchSearchResultItemConnectionPageInfo{ + HasNextPage: false, + }, + }, + }, + responseCode: http.StatusOK, + }, + prResponse: prResponse{ + prs: []getPullRequestDataRepositoryPullRequestsPullRequestConnection{ + { + PageInfo: getPullRequestDataRepositoryPullRequestsPullRequestConnectionPageInfo{ + HasNextPage: false, + }, + Nodes: []PullRequestNode{ + { + Merged: false, + }, + { + Merged: true, + }, + }, + }, + }, + responseCode: http.StatusOK, + }, + branchResponse: branchResponse{ + branches: []getBranchDataRepositoryRefsRefConnection{ + { + TotalCount: 1, + Nodes: []BranchNode{ + { + Name: "main", + Compare: BranchNodeCompareComparison{ + AheadBy: 0, + BehindBy: 1, + }, + }, + }, + PageInfo: getBranchDataRepositoryRefsRefConnectionPageInfo{ + HasNextPage: false, + }, + }, + }, + responseCode: http.StatusOK, + }, + commitResponse: commitResponse{ + commits: []BranchHistoryTargetCommit{ + { + History: BranchHistoryTargetCommitHistoryCommitHistoryConnection{ + Nodes: []CommitNode{ + { + + CommittedDate: time.Now().AddDate(0, 0, -1), + Additions: 10, + Deletions: 9, + }, + }, + }, + }, + }, + responseCode: http.StatusOK, + }, + contribResponse: contribResponse{ + contribs: [][]*github.Contributor{ + { + { + ID: github.Int64(1), + }, + }, + }, + responseCode: http.StatusOK, + }, + }), + testFile: "expected_happy_path.yaml", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + server := httptest.NewServer(tc.server) + defer server.Close() + + cfg := &Config{MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig()} + + ghs := newGitHubScraper(context.Background(), receivertest.NewNopCreateSettings(), cfg) + ghs.cfg.GitHubOrg = "open-telemetry" + ghs.cfg.ClientConfig.Endpoint = server.URL + + err := ghs.start(context.Background(), componenttest.NewNopHost()) + require.NoError(t, err) + + actualMetrics, err := ghs.scrape(context.Background()) + require.NoError(t, err) + + expectedFile := filepath.Join("testdata", "scraper", tc.testFile) + + // Due to the generative nature of the code we're using through + // genqlient. The tests happy path changes, and needs to be rebuilt + // to satisfy the unit tests. When the metadata.yaml changes, and + // code is introduced, or removed. We'll need to update the metrics + // by uncommenting the below and running `make test` to generate + // it. Then we're safe to comment this out again and see happy + // tests. + // golden.WriteMetrics(t, expectedFile, actualMetrics) + + expectedMetrics, err := golden.ReadMetrics(expectedFile) + require.NoError(t, err) + require.NoError(t, pmetrictest.CompareMetrics( + expectedMetrics, + actualMetrics, + pmetrictest.IgnoreMetricDataPointsOrder(), + pmetrictest.IgnoreTimestamp(), + pmetrictest.IgnoreStartTimestamp(), + )) + + }) + } +} diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers.go b/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers.go index 4fac6a512d44..9ab92664f7c8 100644 --- a/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers.go +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers.go @@ -5,18 +5,21 @@ package githubscraper // import "github.com/open-telemetry/opentelemetry-collect import ( "context" + "errors" "fmt" + "math" "net/url" "time" "github.com/Khan/genqlient/graphql" "github.com/google/go-github/v62/github" - "go.uber.org/zap" ) const ( // The default public GitHub GraphQL Endpoint defaultGraphURL = "https://api.github.com/graphql" + // The default maximum number of items to be returned in a GraphQL query. + defaultReturnItems = 100 ) func (ghs *githubScraper) getRepos( @@ -33,7 +36,6 @@ func (ghs *githubScraper) getRepos( for next := true; next; { r, err := getRepoDataBySearch(ctx, client, searchQuery, cursor) if err != nil { - ghs.logger.Sugar().Errorf("error getting repo data", zap.Error(err)) return nil, 0, err } @@ -56,21 +58,26 @@ func (ghs *githubScraper) getBranches( client graphql.Client, repoName string, defaultBranch string, -) (int, error) { +) ([]BranchNode, int, error) { var cursor *string var count int + var branches []BranchNode for next := true; next; { - r, err := getBranchData(ctx, client, repoName, ghs.cfg.GitHubOrg, 50, defaultBranch, cursor) + // Instead of using the defaultReturnItems (100) we chose to set it to + // 50 because GitHub has been known to kill the connection server side + // when trying to get items over 80 on the getBranchData query. + items := 50 + r, err := getBranchData(ctx, client, repoName, ghs.cfg.GitHubOrg, items, defaultBranch, cursor) if err != nil { - ghs.logger.Sugar().Errorf("error getting branch data", zap.Error(err)) - return 0, err + return nil, 0, err } count = r.Repository.Refs.TotalCount cursor = &r.Repository.Refs.PageInfo.EndCursor next = r.Repository.Refs.PageInfo.HasNextPage + branches = append(branches, r.Repository.Refs.Nodes...) } - return count, nil + return branches, count, nil } // Login via the GraphQL checkLogin query in order to ensure that the user @@ -154,13 +161,12 @@ func (ghs *githubScraper) getContributorCount( // Options for Pagination support, default from GitHub was 30 // https://docs.github.com/en/rest/repos/repos#list-repository-contributors opt := &github.ListContributorsOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + ListOptions: github.ListOptions{PerPage: defaultReturnItems}, } for { contribs, resp, err := client.Repositories.ListContributors(ctx, ghs.cfg.GitHubOrg, repoName, opt) if err != nil { - ghs.logger.Sugar().Errorf("error getting contributor count", zap.Error(err)) return 0, err } @@ -181,7 +187,7 @@ func (ghs *githubScraper) getPullRequests( client graphql.Client, repoName string, ) ([]PullRequestNode, error) { - var prCursor *string + var cursor *string var pullRequests []PullRequestNode for hasNextPage := true; hasNextPage; { @@ -190,8 +196,8 @@ func (ghs *githubScraper) getPullRequests( client, repoName, ghs.cfg.GitHubOrg, - 100, - prCursor, + defaultReturnItems, + cursor, []PullRequestState{"OPEN", "MERGED"}, ) if err != nil { @@ -199,13 +205,108 @@ func (ghs *githubScraper) getPullRequests( } pullRequests = append(pullRequests, prs.Repository.PullRequests.Nodes...) - prCursor = &prs.Repository.PullRequests.PageInfo.EndCursor + cursor = &prs.Repository.PullRequests.PageInfo.EndCursor hasNextPage = prs.Repository.PullRequests.PageInfo.HasNextPage } return pullRequests, nil } +func (ghs *githubScraper) evalCommits( + ctx context.Context, + client graphql.Client, + repoName string, + branch BranchNode, +) (additions int, deletions int, age int64, err error) { + var cursor *string + items := defaultReturnItems + + // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/gitproviderreceiver/internal/scraper/githubscraper/README.md#github-limitations + // for more information as to why `BehindBy` and `AheadBy` are + // swapped. + pages := getNumPages(float64(defaultReturnItems), float64(branch.Compare.BehindBy)) + + for page := 1; page <= pages; page++ { + if page == pages { + // We need to make sure that the last page is retrieved properly + // when it's a completely full page, so if the remainder is 0 we'll + // reset to the defaultReturnItems value to ensure the items + // request sent to the getCommitData function is accurate. + items = branch.Compare.BehindBy % defaultReturnItems + if items == 0 { + items = defaultReturnItems + } + } + c, err := ghs.getCommitData(ctx, client, repoName, items, cursor, branch.Name) + if err != nil { + return 0, 0, 0, err + } + + // GraphQL could return empty commit nodes so here we confirm that + // commits were returned to prevent an index out of range error. This + // technically should never be triggered because of other preceding + // catches, but to be safe we check. + if len(c.Nodes) == 0 { + break + } + + cursor = &c.PageInfo.EndCursor + if page == pages { + node := c.GetNodes() + oldest := node[len(node)-1].GetCommittedDate() + age = int64(time.Since(oldest).Seconds()) + } + for b := 0; b < len(c.Nodes); b++ { + additions += c.Nodes[b].Additions + deletions += c.Nodes[b].Deletions + } + + } + return additions, deletions, age, nil +} + +func (ghs *githubScraper) getCommitData( + ctx context.Context, + client graphql.Client, + repoName string, + items int, + cursor *string, + branchName string, +) (*BranchHistoryTargetCommitHistoryCommitHistoryConnection, error) { + data, err := getCommitData(ctx, client, repoName, ghs.cfg.GitHubOrg, 1, items, cursor, branchName) + if err != nil { + return nil, err + } + + // This checks to ensure that the query returned a BranchHistory Node. The + // way the GraphQL query functions allows for a successful query to take + // place, but have an empty set of branches. The only time this query would + // return an empty BranchHistory Node is if the branch was deleted between + // the time the list of branches was retrieved, and the query for the + // commits on the branch. + if len(data.Repository.Refs.Nodes) == 0 { + return nil, errors.New("no branch history returned from the commit data request") + } + + tar := data.Repository.Refs.Nodes[0].GetTarget() + + // We do a sanity type check just to make sure the GraphQL response was + // indead for commits. This is a byproduct of the `... on Commit` syntax + // within the GraphQL query and then return the actual history if the + // returned Target is inded of type Commit. + if ct, ok := tar.(*BranchHistoryTargetCommit); ok { + return &ct.History, nil + } + + return nil, errors.New("GraphQL query did not return the Commit Target") +} + +func getNumPages(p float64, n float64) int { + numPages := math.Ceil(n / p) + + return int(numPages) +} + // Get the age/duration between two times in seconds. func getAge(start time.Time, end time.Time) int64 { return int64(end.Sub(start).Seconds()) diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers_test.go b/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers_test.go index b666c384cbdb..c19220c8680c 100644 --- a/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers_test.go +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/helpers_test.go @@ -26,6 +26,7 @@ type responses struct { branchResponse branchResponse checkLoginResponse loginResponse contribResponse contribResponse + commitResponse commitResponse scrape bool } @@ -47,6 +48,12 @@ type branchResponse struct { page int } +type commitResponse struct { + commits []BranchHistoryTargetCommit + responseCode int + page int +} + type loginResponse struct { checkLogin checkLoginResponse responseCode int @@ -64,7 +71,7 @@ func MockServer(responses *responses) *http.ServeMux { graphEndpoint := "/" if responses.scrape { graphEndpoint = "/api/graphql" - restEndpoint = "/api/v3/repos/liatrio/repo1/contributors" + restEndpoint = "/api/v3/repos/open-telemetry/repo1/contributors" } mux.HandleFunc(graphEndpoint, func(w http.ResponseWriter, r *http.Request) { var reqBody graphql.Request @@ -126,6 +133,26 @@ func MockServer(responses *responses) *http.ServeMux { } prResp.page++ } + case reqBody.OpName == "getCommitData": + commitResp := &responses.commitResponse + w.WriteHeader(commitResp.responseCode) + if commitResp.responseCode == http.StatusOK { + branchHistory := []BranchHistory{ + {Target: &commitResp.commits[commitResp.page]}, + } + commits := getCommitDataResponse{ + Repository: getCommitDataRepository{ + Refs: getCommitDataRepositoryRefsRefConnection{ + Nodes: branchHistory, + }, + }, + } + graphqlResponse := graphql.Response{Data: &commits} + if err := json.NewEncoder(w).Encode(graphqlResponse); err != nil { + return + } + commitResp.page++ + } } }) mux.HandleFunc(restEndpoint, func(w http.ResponseWriter, _ *http.Request) { @@ -151,6 +178,39 @@ func MockServer(responses *responses) *http.ServeMux { return &mux } +func TestGetNumPages100(t *testing.T) { + p := float64(100) + n := float64(375) + + expected := 4 + + num := getNumPages(p, n) + + assert.Equal(t, expected, num) +} + +func TestGetNumPages10(t *testing.T) { + p := float64(10) + n := float64(375) + + expected := 38 + + num := getNumPages(p, n) + + assert.Equal(t, expected, num) +} + +func TestGetNumPages1(t *testing.T) { + p := float64(10) + n := float64(1) + + expected := 1 + + num := getNumPages(p, n) + + assert.Equal(t, expected, num) +} + func TestGenDefaultSearchQueryOrg(t *testing.T) { st := "org" org := "empire" @@ -222,12 +282,12 @@ func TestCheckOwnerExists(t *testing.T) { }{ { desc: "TestOrgOwnerExists", - login: "liatrio", + login: "open-telemetry", server: MockServer(&responses{ checkLoginResponse: loginResponse{ checkLogin: checkLoginResponse{ Organization: checkLoginOrganization{ - Login: "liatrio", + Login: "open-telemetry", }, }, responseCode: http.StatusOK, @@ -237,12 +297,12 @@ func TestCheckOwnerExists(t *testing.T) { }, { desc: "TestUserOwnerExists", - login: "liatrio", + login: "open-telemetry", server: MockServer(&responses{ checkLoginResponse: loginResponse{ checkLogin: checkLoginResponse{ User: checkLoginUser{ - Login: "liatrio", + Login: "open-telemetry", }, }, responseCode: http.StatusOK, @@ -252,12 +312,12 @@ func TestCheckOwnerExists(t *testing.T) { }, { desc: "TestLoginError", - login: "liatrio", + login: "open-telemetry", server: MockServer(&responses{ checkLoginResponse: loginResponse{ checkLogin: checkLoginResponse{ Organization: checkLoginOrganization{ - Login: "liatrio", + Login: "open-telemetry", }, }, responseCode: http.StatusNotFound, @@ -598,7 +658,7 @@ func TestGetBranches(t *testing.T) { defer server.Close() client := graphql.NewClient(server.URL, ghs.client) - count, err := ghs.getBranches(context.Background(), client, "deathstarrepo", "main") + _, count, err := ghs.getBranches(context.Background(), client, "deathstarrepo", "main") assert.Equal(t, tc.expected, count) if tc.expectedErr == nil { @@ -662,3 +722,192 @@ func TestGetContributors(t *testing.T) { }) } } + +func TestEvalCommits(t *testing.T) { + testCases := []struct { + desc string + server *http.ServeMux + expectedErr error + branch BranchNode + expectedAge int64 + expectedAdditions int + expectedDeletions int + }{ + { + desc: "TestNoBranchChanges", + server: MockServer(&responses{ + scrape: false, + commitResponse: commitResponse{ + commits: []BranchHistoryTargetCommit{ + { + History: BranchHistoryTargetCommitHistoryCommitHistoryConnection{ + Nodes: []CommitNode{}, + }, + }, + }, + responseCode: http.StatusOK, + }, + }), + branch: BranchNode{ + Name: "branch1", + Compare: BranchNodeCompareComparison{ + AheadBy: 0, + BehindBy: 0, + }, + }, + expectedAge: 0, + expectedAdditions: 0, + expectedDeletions: 0, + expectedErr: nil, + }, + { + desc: "TestNoCommitsResponse", + server: MockServer(&responses{ + scrape: false, + commitResponse: commitResponse{ + commits: []BranchHistoryTargetCommit{ + { + History: BranchHistoryTargetCommitHistoryCommitHistoryConnection{ + Nodes: []CommitNode{}, + }, + }, + }, + responseCode: http.StatusOK, + }, + }), + branch: BranchNode{ + Name: "branch1", + Compare: BranchNodeCompareComparison{ + AheadBy: 0, + BehindBy: 1, + }, + }, + expectedAge: 0, + expectedAdditions: 0, + expectedDeletions: 0, + expectedErr: nil, + }, + { + desc: "TestSinglePageResponse", + server: MockServer(&responses{ + scrape: false, + commitResponse: commitResponse{ + commits: []BranchHistoryTargetCommit{ + { + History: BranchHistoryTargetCommitHistoryCommitHistoryConnection{ + Nodes: []CommitNode{ + { + + CommittedDate: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Additions: 10, + Deletions: 9, + }, + }, + }, + }, + }, + responseCode: http.StatusOK, + }, + }), + branch: BranchNode{ + Name: "branch1", + Compare: BranchNodeCompareComparison{ + AheadBy: 0, + BehindBy: 1, + }, + }, + expectedAge: int64(time.Since(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)).Seconds()), + expectedAdditions: 10, + expectedDeletions: 9, + expectedErr: nil, + }, + { + desc: "TestMultiplePageResponse", + server: MockServer(&responses{ + scrape: false, + commitResponse: commitResponse{ + commits: []BranchHistoryTargetCommit{ + { + History: BranchHistoryTargetCommitHistoryCommitHistoryConnection{ + Nodes: []CommitNode{ + { + + CommittedDate: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Additions: 10, + Deletions: 9, + }, + }, + }, + }, + { + History: BranchHistoryTargetCommitHistoryCommitHistoryConnection{ + Nodes: []CommitNode{ + { + + CommittedDate: time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC), + Additions: 1, + Deletions: 1, + }, + }, + }, + }, + }, + responseCode: http.StatusOK, + }, + }), + branch: BranchNode{ + Name: "branch1", + Compare: BranchNodeCompareComparison{ + AheadBy: 0, + BehindBy: 101, // 100 per page, so this is 2 pages + }, + }, + expectedAge: int64(time.Since(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)).Seconds()), + expectedAdditions: 11, + expectedDeletions: 10, + expectedErr: nil, + }, + { + desc: "Test404ErrorResponse", + server: MockServer(&responses{ + scrape: false, + commitResponse: commitResponse{ + responseCode: http.StatusNotFound, + }, + }), + branch: BranchNode{ + Name: "branch1", + Compare: BranchNodeCompareComparison{ + AheadBy: 0, + BehindBy: 1, + }, + }, + expectedAge: 0, + expectedAdditions: 0, + expectedDeletions: 0, + expectedErr: errors.New("returned error 404 Not Found: "), + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + factory := Factory{} + defaultConfig := factory.CreateDefaultConfig() + settings := receivertest.NewNopCreateSettings() + ghs := newGitHubScraper(context.Background(), settings, defaultConfig.(*Config)) + server := httptest.NewServer(tc.server) + defer server.Close() + client := graphql.NewClient(server.URL, ghs.client) + adds, dels, age, err := ghs.evalCommits(context.Background(), client, "repo1", tc.branch) + + assert.Equal(t, tc.expectedAge, age) + assert.Equal(t, tc.expectedDeletions, dels) + assert.Equal(t, tc.expectedAdditions, adds) + + if tc.expectedErr == nil { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr.Error()) + } + }) + } +} diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_happy_path.yaml b/receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_happy_path.yaml new file mode 100644 index 000000000000..b21b89608fe1 --- /dev/null +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_happy_path.yaml @@ -0,0 +1,165 @@ +resourceMetrics: + - resource: + attributes: + - key: git.vendor.name + value: + stringValue: github + - key: organization.name + value: + stringValue: open-telemetry + schemaUrl: https://opentelemetry.io/schemas/1.9.0 + scopeMetrics: + - metrics: + - description: The number of commits a branch is ahead of the default branch (trunk). + gauge: + dataPoints: + - asInt: "1" + attributes: + - key: branch.name + value: + stringValue: main + - key: repository.name + value: + stringValue: "" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.branch.commit.aheadby.count + unit: '{commit}' + - description: The number of commits a branch is behind the default branch (trunk). + gauge: + dataPoints: + - asInt: "0" + attributes: + - key: branch.name + value: + stringValue: main + - key: repository.name + value: + stringValue: "" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.branch.commit.behindby.count + unit: '{commit}' + - description: The number of branches in a repository. + gauge: + dataPoints: + - asInt: "1" + attributes: + - key: repository.name + value: + stringValue: repo1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.branch.count + unit: '{branch}' + - description: The number of lines added in a branch relative to the default branch (trunk). + gauge: + dataPoints: + - asInt: "10" + attributes: + - key: branch.name + value: + stringValue: main + - key: repository.name + value: + stringValue: "" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.branch.line.addition.count + unit: '{line}' + - description: The number of lines deleted in a branch relative to the default branch (trunk). + gauge: + dataPoints: + - asInt: "9" + attributes: + - key: branch.name + value: + stringValue: main + - key: repository.name + value: + stringValue: "" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.branch.line.deletion.count + unit: '{line}' + - description: Time a branch created from the default branch (trunk) has existed. + gauge: + dataPoints: + - asInt: "86400" + attributes: + - key: branch.name + value: + stringValue: main + - key: repository.name + value: + stringValue: "" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.branch.time + unit: s + - description: The number of repositories in an organization. + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.count + unit: '{repository}' + - description: The number of pull requests in a repository, categorized by their state (either open or merged). + gauge: + dataPoints: + - asInt: "1" + attributes: + - key: pull_request.state + value: + stringValue: merged + - key: repository.name + value: + stringValue: repo1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + - asInt: "1" + attributes: + - key: pull_request.state + value: + stringValue: open + - key: repository.name + value: + stringValue: repo1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.pull_request.count + unit: '{pull_request}' + - description: The amount of time a pull request has been open. + gauge: + dataPoints: + - asInt: "9223372036" + attributes: + - key: branch.name + value: + stringValue: "" + - key: repository.name + value: + stringValue: repo1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.pull_request.time_open + unit: s + - description: The amount of time it took a pull request to go from open to merged. + gauge: + dataPoints: + - asInt: "0" + attributes: + - key: branch.name + value: + stringValue: "" + - key: repository.name + value: + stringValue: repo1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.pull_request.time_to_merge + unit: s + scope: + name: otelcol/gitproviderreceiver + version: latest diff --git a/receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_no_repos.yaml b/receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_no_repos.yaml new file mode 100644 index 000000000000..774f288da763 --- /dev/null +++ b/receiver/gitproviderreceiver/internal/scraper/githubscraper/testdata/scraper/expected_no_repos.yaml @@ -0,0 +1,23 @@ +resourceMetrics: + - resource: + attributes: + - key: git.vendor.name + value: + stringValue: github + - key: organization.name + value: + stringValue: open-telemetry + schemaUrl: https://opentelemetry.io/schemas/1.9.0 + scopeMetrics: + - metrics: + - description: The number of repositories in an organization. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: git.repository.count + unit: '{repository}' + scope: + name: otelcol/gitproviderreceiver + version: latest diff --git a/receiver/gitproviderreceiver/metadata.yaml b/receiver/gitproviderreceiver/metadata.yaml index d66a8bffe667..4ec808e77771 100644 --- a/receiver/gitproviderreceiver/metadata.yaml +++ b/receiver/gitproviderreceiver/metadata.yaml @@ -41,48 +41,83 @@ attributes: metrics: git.repository.count: enabled: true - description: Number of repositories in an organization - unit: "{repository}" + description: The number of repositories in an organization. + unit: '{repository}' gauge: value_type: int attributes: [] git.repository.branch.count: enabled: true - description: Number of branches in a repository - unit: "{branch}" + description: The number of branches in a repository. + unit: '{branch}' gauge: value_type: int attributes: [repository.name] + git.repository.branch.time: + enabled: true + description: Time a branch created from the default branch (trunk) has existed. + unit: s + gauge: + value_type: int + attributes: [repository.name, branch.name] + git.repository.branch.commit.aheadby.count: + enabled: true + description: The number of commits a branch is ahead of the default branch (trunk). + unit: '{commit}' + gauge: + value_type: int + attributes: [repository.name, branch.name] + git.repository.branch.commit.behindby.count: + enabled: true + description: The number of commits a branch is behind the default branch (trunk). + unit: '{commit}' + gauge: + value_type: int + attributes: [repository.name, branch.name] + git.repository.branch.line.addition.count: + enabled: true + description: The number of lines added in a branch relative to the default branch (trunk). + unit: '{line}' + gauge: + value_type: int + attributes: [repository.name, branch.name] + git.repository.branch.line.deletion.count: + enabled: true + description: The number of lines deleted in a branch relative to the default branch (trunk). + unit: '{line}' + gauge: + value_type: int + attributes: [repository.name, branch.name] git.repository.contributor.count: enabled: false - description: Total number of unique contributors to a repository - unit: "{contributor}" + description: The number of unique contributors to a repository. + unit: '{contributor}' gauge: value_type: int attributes: [repository.name] git.repository.pull_request.time_open: enabled: true - description: The amount of time a pull request has been open + description: The amount of time a pull request has been open. unit: s gauge: value_type: int attributes: [repository.name, branch.name] git.repository.pull_request.time_to_merge: enabled: true - description: The amount of time it took a pull request to go from open to merged + description: The amount of time it took a pull request to go from open to merged. unit: s gauge: value_type: int attributes: [repository.name, branch.name] git.repository.pull_request.time_to_approval: enabled: true - description: The amount of time it took a pull request to go from open to approved + description: The amount of time it took a pull request to go from open to approved. unit: s gauge: value_type: int attributes: [repository.name, branch.name] git.repository.pull_request.count: - description: The number of pull requests in a repository, categorized by their state (either open or merged) + description: The number of pull requests in a repository, categorized by their state (either open or merged). enabled: true gauge: value_type: int