From a72713db2a0a4e089d9f59d9f008735ab19477d2 Mon Sep 17 00:00:00 2001 From: Rob Best Date: Mon, 6 Feb 2023 12:54:00 +0000 Subject: [PATCH] Bump client-go to pick up pagination fix Pagination was broken in the previous version. Also, add some very basic tests that would have caught this problem. --- go.mod | 5 +- go.sum | 7 +- internal/exporter/exporter.go | 20 +++-- internal/exporter/exporter_test.go | 118 +++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 internal/exporter/exporter_test.go diff --git a/go.mod b/go.mod index dd9693d..1a8527b 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module github.com/jetstack/dependency-track-exporter go 1.19 require ( - github.com/DependencyTrack/client-go v0.8.0 + github.com/DependencyTrack/client-go v0.8.1-0.20230206124230-e451b5e186af github.com/go-kit/log v0.2.1 + github.com/google/go-cmp v0.5.9 + github.com/google/uuid v1.3.0 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/common v0.39.0 github.com/prometheus/exporter-toolkit v0.8.2 @@ -19,7 +21,6 @@ require ( github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect diff --git a/go.sum b/go.sum index 26d4e2a..0f9c37e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/DependencyTrack/client-go v0.8.0 h1:uAukThoe45xGLtWlIUEzvwMUxg4mDP1EXQC/tWBr55U= -github.com/DependencyTrack/client-go v0.8.0/go.mod h1:joh0lDVtJfPh6G39kdtAdbtfM9+RPD/82QPeYM3ZtC4= +github.com/DependencyTrack/client-go v0.8.1-0.20230206124230-e451b5e186af h1:tNMON7PK9HqdbGRkhKDHSAELakdKIqXMW2brUqRHo5k= +github.com/DependencyTrack/client-go v0.8.1-0.20230206124230-e451b5e186af/go.mod h1:YZyJyl5WZwOB0Jjd9at3sCy/9X2ZK0sIJNgIjWPqXCQ= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= @@ -25,7 +25,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index 277a3ed..a84775a 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -187,9 +187,7 @@ func (e *Exporter) collectProjectMetrics(ctx context.Context, registry *promethe inheritedRiskScore, ) - projects, err := dtrack.FetchAll(func(po dtrack.PageOptions) (dtrack.Page[dtrack.Project], error) { - return e.Client.Project.GetAll(ctx, po) - }) + projects, err := e.fetchProjects(ctx) if err != nil { return err } @@ -265,9 +263,7 @@ func (e *Exporter) collectProjectMetrics(ctx context.Context, registry *promethe } } - violations, err := dtrack.FetchAll(func(po dtrack.PageOptions) (dtrack.Page[dtrack.PolicyViolation], error) { - return e.Client.PolicyViolation.GetAll(ctx, true, po) - }) + violations, err := e.fetchPolicyViolations(ctx) if err != nil { return err } @@ -294,3 +290,15 @@ func (e *Exporter) collectProjectMetrics(ctx context.Context, registry *promethe return nil } + +func (e *Exporter) fetchProjects(ctx context.Context) ([]dtrack.Project, error) { + return dtrack.FetchAll(func(po dtrack.PageOptions) (dtrack.Page[dtrack.Project], error) { + return e.Client.Project.GetAll(ctx, po) + }) +} + +func (e *Exporter) fetchPolicyViolations(ctx context.Context) ([]dtrack.PolicyViolation, error) { + return dtrack.FetchAll(func(po dtrack.PageOptions) (dtrack.Page[dtrack.PolicyViolation], error) { + return e.Client.PolicyViolation.GetAll(ctx, true, po) + }) +} diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go new file mode 100644 index 0000000..6d1e672 --- /dev/null +++ b/internal/exporter/exporter_test.go @@ -0,0 +1,118 @@ +package exporter + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + dtrack "github.com/DependencyTrack/client-go" + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" +) + +func TestFetchProjects_Pagination(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + var wantProjects []dtrack.Project + for i := 0; i < 468; i++ { + wantProjects = append(wantProjects, dtrack.Project{ + UUID: uuid.New(), + }) + } + + mux.HandleFunc("/api/v1/project", func(w http.ResponseWriter, r *http.Request) { + pageSize, err := strconv.Atoi(r.URL.Query().Get("pageSize")) + if err != nil { + t.Fatalf("unexpected error converting pageSize to int: %s", err) + } + pageNumber, err := strconv.Atoi(r.URL.Query().Get("pageNumber")) + if err != nil { + t.Fatalf("unexpected error converting pageNumber to int: %s", err) + } + w.Header().Set("X-Total-Count", strconv.Itoa(len(wantProjects))) + w.Header().Set("Content-type", "application/json") + var projects []dtrack.Project + for i := 0; i < pageSize; i++ { + idx := (pageSize * (pageNumber - 1)) + i + if idx >= len(wantProjects) { + break + } + projects = append(projects, wantProjects[idx]) + } + json.NewEncoder(w).Encode(projects) + }) + + client, err := dtrack.NewClient(server.URL) + if err != nil { + t.Fatalf("unexpected error setting up client: %s", err) + } + + e := &Exporter{ + Client: client, + } + + gotProjects, err := e.fetchProjects(context.Background()) + if err != nil { + t.Fatalf("unexpected error fetching projects: %s", err) + } + + if diff := cmp.Diff(wantProjects, gotProjects); diff != "" { + t.Errorf("unexpected projects:\n%s", diff) + } +} + +func TestFetchPolicyViolations_Pagination(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + var wantPolicyViolations []dtrack.PolicyViolation + for i := 0; i < 468; i++ { + wantPolicyViolations = append(wantPolicyViolations, dtrack.PolicyViolation{ + UUID: uuid.New(), + }) + } + + mux.HandleFunc("/api/v1/violation", func(w http.ResponseWriter, r *http.Request) { + pageSize, err := strconv.Atoi(r.URL.Query().Get("pageSize")) + if err != nil { + t.Fatalf("unexpected error converting pageSize to int: %s", err) + } + pageNumber, err := strconv.Atoi(r.URL.Query().Get("pageNumber")) + if err != nil { + t.Fatalf("unexpected error converting pageNumber to int: %s", err) + } + w.Header().Set("X-Total-Count", strconv.Itoa(len(wantPolicyViolations))) + w.Header().Set("Content-type", "application/json") + var policyViolations []dtrack.PolicyViolation + for i := 0; i < pageSize; i++ { + idx := (pageSize * (pageNumber - 1)) + i + if idx >= len(wantPolicyViolations) { + break + } + policyViolations = append(policyViolations, wantPolicyViolations[idx]) + } + json.NewEncoder(w).Encode(policyViolations) + }) + + client, err := dtrack.NewClient(server.URL) + if err != nil { + t.Fatalf("unexpected error setting up client: %s", err) + } + + e := &Exporter{ + Client: client, + } + + gotPolicyViolations, err := e.fetchPolicyViolations(context.Background()) + if err != nil { + t.Fatalf("unexpected error fetching projects: %s", err) + } + + if diff := cmp.Diff(wantPolicyViolations, gotPolicyViolations); diff != "" { + t.Errorf("unexpected policy violations:\n%s", diff) + } +}